For I/O BARs to work correctly on RISC-V Linux, we need to establish a
reserved memory region for them, so that drivers that wish to use I/O BARs
can issue reads and writes against a memory region that is mapped to the
host PCIe controller's I/O BAR MMIO mapping.
Signed-off-by: Yash Shah <[email protected]>
---
arch/riscv/include/asm/io.h | 7 +++++++
arch/riscv/include/asm/pgtable.h | 7 ++++++-
2 files changed, 13 insertions(+), 1 deletion(-)
diff --git a/arch/riscv/include/asm/io.h b/arch/riscv/include/asm/io.h
index fc1189a..3ba4d93 100644
--- a/arch/riscv/include/asm/io.h
+++ b/arch/riscv/include/asm/io.h
@@ -13,6 +13,7 @@
#include <linux/types.h>
#include <asm/mmiowb.h>
+#include <asm/pgtable.h>
extern void __iomem *ioremap(phys_addr_t offset, unsigned long size);
@@ -162,6 +163,12 @@ static inline u64 __raw_readq(const volatile void __iomem *addr)
#endif
/*
+ * I/O port access constants.
+ */
+#define IO_SPACE_LIMIT (PCI_IO_SIZE - 1)
+#define PCI_IOBASE ((void __iomem *)PCI_IO_START)
+
+/*
* Emulation routines for the port-mapped IO space used by some PCI drivers.
* These are defined as being "fully synchronous", but also "not guaranteed to
* be fully ordered with respect to other memory and I/O operations". We're
diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h
index 7fc5e4a..d78cc74 100644
--- a/arch/riscv/include/asm/pgtable.h
+++ b/arch/riscv/include/asm/pgtable.h
@@ -7,6 +7,7 @@
#define _ASM_RISCV_PGTABLE_H
#include <linux/mmzone.h>
+#include <linux/sizes.h>
#include <asm/pgtable-bits.h>
@@ -88,6 +89,7 @@ extern pgd_t swapper_pg_dir[];
#define VMALLOC_SIZE (KERN_VIRT_SIZE >> 1)
#define VMALLOC_END (PAGE_OFFSET - 1)
#define VMALLOC_START (PAGE_OFFSET - VMALLOC_SIZE)
+#define PCI_IO_SIZE SZ_16M
/*
* Roughly size the vmemmap space to be large enough to fit enough
@@ -102,7 +104,10 @@ extern pgd_t swapper_pg_dir[];
#define vmemmap ((struct page *)VMEMMAP_START)
-#define FIXADDR_TOP (VMEMMAP_START)
+#define PCI_IO_END VMEMMAP_START
+#define PCI_IO_START (PCI_IO_END - PCI_IO_SIZE)
+#define FIXADDR_TOP PCI_IO_START
+
#ifdef CONFIG_64BIT
#define FIXADDR_SIZE PMD_SIZE
#else
--
2.7.4
On 2019-10-24 10:51 a.m., Paul Walmsley wrote:
> On Thu, 24 Oct 2019, Logan Gunthorpe wrote:
>
>> On 2019-10-24 3:14 a.m., Yash Shah wrote:
>>> For I/O BARs to work correctly on RISC-V Linux, we need to establish a
>>> reserved memory region for them, so that drivers that wish to use I/O BARs
>>> can issue reads and writes against a memory region that is mapped to the
>>> host PCIe controller's I/O BAR MMIO mapping.
>>
>> I don't think other arches do this.
>
> $ git grep 'define PCI_IOBASE' arch/
> arch/arm/include/asm/io.h:#define PCI_IOBASE ((void __iomem *)PCI_IO_VIRT_BASE)
> arch/arm64/include/asm/io.h:#define PCI_IOBASE ((void __iomem *)PCI_IO_START)
> arch/m68k/include/asm/io_no.h:#define PCI_IOBASE ((void __iomem *) PCI_IO_PA)
> arch/microblaze/include/asm/io.h:#define PCI_IOBASE ((void __iomem *)_IO_BASE)
> arch/unicore32/include/asm/io.h:#define PCI_IOBASE PKUNITY_PCILIO_BASE
> arch/xtensa/include/asm/io.h:#define PCI_IOBASE ((void __iomem *)XCHAL_KIO_BYPASS_VADDR)
> $
>
> This is for the old x86-style, non-memory mapped I/O address space the
> legacy PCI stuff that one would use in{b,w,l}()/out{b,w,l}() for.
>
> Yash, you might consider updating your patch description to note that this
> is for "legacy I/O BARs (i.e., non-MMIO BARs)" or something similar. That
> might make things clearer.
Ah, right, yes, that would clear things up.
Logan
On 2019-10-24 3:14 a.m., Yash Shah wrote:
> For I/O BARs to work correctly on RISC-V Linux, we need to establish a
> reserved memory region for them, so that drivers that wish to use I/O BARs
> can issue reads and writes against a memory region that is mapped to the
> host PCIe controller's I/O BAR MMIO mapping.
I don't think other arches do this. ioremap() typically just uses
virtual address space in the VMALLOC region, PCI doesn't need it's own
range. As far as I know the ioremap() implementation in riscv already
does this.
In any case, 16MB for PCI bar space seems woefully inadequate.
Logan
> Signed-off-by: Yash Shah <[email protected]>
> ---
> arch/riscv/include/asm/io.h | 7 +++++++
> arch/riscv/include/asm/pgtable.h | 7 ++++++-
> 2 files changed, 13 insertions(+), 1 deletion(-)
>
> diff --git a/arch/riscv/include/asm/io.h b/arch/riscv/include/asm/io.h
> index fc1189a..3ba4d93 100644
> --- a/arch/riscv/include/asm/io.h
> +++ b/arch/riscv/include/asm/io.h
> @@ -13,6 +13,7 @@
>
> #include <linux/types.h>
> #include <asm/mmiowb.h>
> +#include <asm/pgtable.h>
>
> extern void __iomem *ioremap(phys_addr_t offset, unsigned long size);
>
> @@ -162,6 +163,12 @@ static inline u64 __raw_readq(const volatile void __iomem *addr)
> #endif
>
> /*
> + * I/O port access constants.
> + */
> +#define IO_SPACE_LIMIT (PCI_IO_SIZE - 1)
> +#define PCI_IOBASE ((void __iomem *)PCI_IO_START)
> +
> +/*
> * Emulation routines for the port-mapped IO space used by some PCI drivers.
> * These are defined as being "fully synchronous", but also "not guaranteed to
> * be fully ordered with respect to other memory and I/O operations". We're
> diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h
> index 7fc5e4a..d78cc74 100644
> --- a/arch/riscv/include/asm/pgtable.h
> +++ b/arch/riscv/include/asm/pgtable.h
> @@ -7,6 +7,7 @@
> #define _ASM_RISCV_PGTABLE_H
>
> #include <linux/mmzone.h>
> +#include <linux/sizes.h>
>
> #include <asm/pgtable-bits.h>
>
> @@ -88,6 +89,7 @@ extern pgd_t swapper_pg_dir[];
> #define VMALLOC_SIZE (KERN_VIRT_SIZE >> 1)
> #define VMALLOC_END (PAGE_OFFSET - 1)
> #define VMALLOC_START (PAGE_OFFSET - VMALLOC_SIZE)
> +#define PCI_IO_SIZE SZ_16M
>
> /*
> * Roughly size the vmemmap space to be large enough to fit enough
> @@ -102,7 +104,10 @@ extern pgd_t swapper_pg_dir[];
>
> #define vmemmap ((struct page *)VMEMMAP_START)
>
> -#define FIXADDR_TOP (VMEMMAP_START)
> +#define PCI_IO_END VMEMMAP_START
> +#define PCI_IO_START (PCI_IO_END - PCI_IO_SIZE)
> +#define FIXADDR_TOP PCI_IO_START
> +
> #ifdef CONFIG_64BIT
> #define FIXADDR_SIZE PMD_SIZE
> #else
>
On Thu, 24 Oct 2019, Logan Gunthorpe wrote:
> On 2019-10-24 3:14 a.m., Yash Shah wrote:
> > For I/O BARs to work correctly on RISC-V Linux, we need to establish a
> > reserved memory region for them, so that drivers that wish to use I/O BARs
> > can issue reads and writes against a memory region that is mapped to the
> > host PCIe controller's I/O BAR MMIO mapping.
>
> I don't think other arches do this.
$ git grep 'define PCI_IOBASE' arch/
arch/arm/include/asm/io.h:#define PCI_IOBASE ((void __iomem *)PCI_IO_VIRT_BASE)
arch/arm64/include/asm/io.h:#define PCI_IOBASE ((void __iomem *)PCI_IO_START)
arch/m68k/include/asm/io_no.h:#define PCI_IOBASE ((void __iomem *) PCI_IO_PA)
arch/microblaze/include/asm/io.h:#define PCI_IOBASE ((void __iomem *)_IO_BASE)
arch/unicore32/include/asm/io.h:#define PCI_IOBASE PKUNITY_PCILIO_BASE
arch/xtensa/include/asm/io.h:#define PCI_IOBASE ((void __iomem *)XCHAL_KIO_BYPASS_VADDR)
$
This is for the old x86-style, non-memory mapped I/O address space the
legacy PCI stuff that one would use in{b,w,l}()/out{b,w,l}() for.
Yash, you might consider updating your patch description to note that this
is for "legacy I/O BARs (i.e., non-MMIO BARs)" or something similar. That
might make things clearer.
> ioremap() typically just uses virtual address space in the VMALLOC
> region, PCI doesn't need it's own range. As far as I know the ioremap()
> implementation in riscv already does this.
>
> In any case, 16MB for PCI bar space seems woefully inadequate.
The modern MMIO PCI resources wind up in jost controller apertures, which
as you note, are usually much larger. They don't go in this legacy space.
Regarding sizing - I haven't seen any PCIe cards with more than 64KiB of
legacy I/O resources. (16MiB / 64KiB) = 256, so 16MiB sounds reasonable
from that point of view? ARM64 is using that.
- Paul
> On Thu, 24 Oct 2019, Logan Gunthorpe wrote:
>
> > On 2019-10-24 3:14 a.m., Yash Shah wrote:
> > > For I/O BARs to work correctly on RISC-V Linux, we need to establish
> > > a reserved memory region for them, so that drivers that wish to use
> > > I/O BARs can issue reads and writes against a memory region that is
> > > mapped to the host PCIe controller's I/O BAR MMIO mapping.
> >
> > I don't think other arches do this.
>
> $ git grep 'define PCI_IOBASE' arch/
> arch/arm/include/asm/io.h:#define PCI_IOBASE ((void __iomem
> *)PCI_IO_VIRT_BASE)
> arch/arm64/include/asm/io.h:#define PCI_IOBASE ((void __iomem
> *)PCI_IO_START)
> arch/m68k/include/asm/io_no.h:#define PCI_IOBASE ((void __iomem *)
> PCI_IO_PA)
> arch/microblaze/include/asm/io.h:#define PCI_IOBASE ((void __iomem
> *)_IO_BASE)
> arch/unicore32/include/asm/io.h:#define PCI_IOBASE
> PKUNITY_PCILIO_BASE
> arch/xtensa/include/asm/io.h:#define PCI_IOBASE ((void __iomem
> *)XCHAL_KIO_BYPASS_VADDR)
> $
>
> This is for the old x86-style, non-memory mapped I/O address space the
> legacy PCI stuff that one would use in{b,w,l}()/out{b,w,l}() for.
>
> Yash, you might consider updating your patch description to note that this is
> for "legacy I/O BARs (i.e., non-MMIO BARs)" or something similar. That
> might make things clearer.
Sure, will update the description and send v2.
- Yash
>
> > ioremap() typically just uses virtual address space in the VMALLOC
> > region, PCI doesn't need it's own range. As far as I know the
> > ioremap() implementation in riscv already does this.
> >
> > In any case, 16MB for PCI bar space seems woefully inadequate.
>
> The modern MMIO PCI resources wind up in jost controller apertures, which
> as you note, are usually much larger. They don't go in this legacy space.
>
> Regarding sizing - I haven't seen any PCIe cards with more than 64KiB of
> legacy I/O resources. (16MiB / 64KiB) = 256, so 16MiB sounds reasonable
> from that point of view? ARM64 is using that.
>
>
> - Paul