Received-SPF: pass (google.com: domain of linux-kernel-owner@vger.kernel.org designates 2620:137:e000::1:20 as permitted sender) client-ip=2620:137:e000::1:20;
MIME-Version: 1.0
References: <cover.1689792825.git.tjeznach@rivosinc.com> <660b7a8707e494a6bb2706e10569a7414c3640a7.1689792825.git.tjeznach@rivosinc.com>
In-Reply-To: <660b7a8707e494a6bb2706e10569a7414c3640a7.1689792825.git.tjeznach@rivosinc.com>
From:   Zong Li <zong.li@sifive.com>
Date:   Mon, 31 Jul 2023 16:02:35 +0800
Message-ID: <CANXhq0oSiv7UV+nmZppA0_oZCevEEt2UsNTz6WE3gtcCpExYqw@mail.gmail.com>
Subject: Re: [PATCH 10/11] RISC-V: drivers/iommu/riscv: Add MSI identity remapping
To:     Tomasz Jeznach <tjeznach@rivosinc.com>
Cc:     Joerg Roedel <joro@8bytes.org>, Will Deacon <will@kernel.org>,
        Robin Murphy <robin.murphy@arm.com>,
        Paul Walmsley <paul.walmsley@sifive.com>,
        Anup Patel <apatel@ventanamicro.com>,
        Albert Ou <aou@eecs.berkeley.edu>, linux@rivosinc.com,
        linux-kernel@vger.kernel.org, Sebastien Boeuf <seb@rivosinc.com>,
        iommu@lists.linux.dev, Palmer Dabbelt <palmer@dabbelt.com>,
        Nick Kossifidis <mick@ics.forth.gr>,
        linux-riscv@lists.infradead.org
Content-Type: text/plain; charset="UTF-8"
Content-Transfer-Encoding: quoted-printable
Precedence: bulk

On Thu, Jul 20, 2023 at 3:34=E2=80=AFAM Tomasz Jeznach <tjeznach@rivosinc.c=
om> wrote:
>
> This change provides basic identity mapping support to
> excercise MSI_FLAT hardware capability.
>
> Signed-off-by: Tomasz Jeznach <tjeznach@rivosinc.com>
> ---
>  drivers/iommu/riscv/iommu.c | 81 +++++++++++++++++++++++++++++++++++++
>  drivers/iommu/riscv/iommu.h |  3 ++
>  2 files changed, 84 insertions(+)
>
> diff --git a/drivers/iommu/riscv/iommu.c b/drivers/iommu/riscv/iommu.c
> index 6042c35be3ca..7b3e3e135cf6 100644
> --- a/drivers/iommu/riscv/iommu.c
> +++ b/drivers/iommu/riscv/iommu.c
> @@ -61,6 +61,9 @@ MODULE_PARM_DESC(priq_length, "Page request interface q=
ueue length.");
>  #define RISCV_IOMMU_MAX_PSCID  (1U << 20)
>  static DEFINE_IDA(riscv_iommu_pscids);
>
> +/* TODO: Enable MSI remapping */
> +#define RISCV_IMSIC_BASE       0x28000000

I'm not sure if it is appropriate to hard code the base address of
peripheral in source code, it might be depends on the layout of each
target.

> +
>  /* 1 second */
>  #define RISCV_IOMMU_TIMEOUT    riscv_timebase
>
> @@ -932,6 +935,72 @@ static irqreturn_t riscv_iommu_priq_process(int irq,=
 void *data)
>   * Endpoint management
>   */
>
> +static int riscv_iommu_enable_ir(struct riscv_iommu_endpoint *ep)
> +{
> +       struct riscv_iommu_device *iommu =3D ep->iommu;
> +       struct iommu_resv_region *entry;
> +       struct irq_domain *msi_domain;
> +       u64 val;
> +       int i;
> +
> +       /* Initialize MSI remapping */
> +       if (!ep->dc || !(iommu->cap & RISCV_IOMMU_CAP_MSI_FLAT))
> +               return 0;
> +
> +       ep->msi_root =3D (struct riscv_iommu_msi_pte *)get_zeroed_page(GF=
P_KERNEL);
> +       if (!ep->msi_root)
> +               return -ENOMEM;
> +
> +       for (i =3D 0; i < 256; i++) {
> +               ep->msi_root[i].pte =3D RISCV_IOMMU_MSI_PTE_V |
> +                   FIELD_PREP(RISCV_IOMMU_MSI_PTE_M, 3) |
> +                   phys_to_ppn(RISCV_IMSIC_BASE + i * PAGE_SIZE);
> +       }
> +
> +       entry =3D iommu_alloc_resv_region(RISCV_IMSIC_BASE, PAGE_SIZE * 2=
56, 0,
> +                                       IOMMU_RESV_SW_MSI, GFP_KERNEL);
> +       if (entry)
> +               list_add_tail(&entry->list, &ep->regions);
> +
> +       val =3D virt_to_pfn(ep->msi_root) |
> +           FIELD_PREP(RISCV_IOMMU_DC_MSIPTP_MODE, RISCV_IOMMU_DC_MSIPTP_=
MODE_FLAT);
> +       ep->dc->msiptp =3D cpu_to_le64(val);
> +
> +       /* Single page of MSIPTP, 256 IMSIC files */
> +       ep->dc->msi_addr_mask =3D cpu_to_le64(255);
> +       ep->dc->msi_addr_pattern =3D cpu_to_le64(RISCV_IMSIC_BASE >> 12);
> +       wmb();
> +
> +       /* set msi domain for the device as isolated. hack. */
> +       msi_domain =3D dev_get_msi_domain(ep->dev);
> +       if (msi_domain) {
> +               msi_domain->flags |=3D IRQ_DOMAIN_FLAG_ISOLATED_MSI;
> +       }
> +
> +       dev_dbg(ep->dev, "RV-IR enabled\n");
> +
> +       ep->ir_enabled =3D true;
> +
> +       return 0;
> +}
> +
> +static void riscv_iommu_disable_ir(struct riscv_iommu_endpoint *ep)
> +{
> +       if (!ep->ir_enabled)
> +               return;
> +
> +       ep->dc->msi_addr_pattern =3D 0ULL;
> +       ep->dc->msi_addr_mask =3D 0ULL;
> +       ep->dc->msiptp =3D 0ULL;
> +       wmb();
> +
> +       dev_dbg(ep->dev, "RV-IR disabled\n");
> +
> +       free_pages((unsigned long)ep->msi_root, 0);
> +       ep->msi_root =3D NULL;
> +       ep->ir_enabled =3D false;
> +}
> +
>  /* Endpoint features/capabilities */
>  static void riscv_iommu_disable_ep(struct riscv_iommu_endpoint *ep)
>  {
> @@ -1226,6 +1295,7 @@ static struct iommu_device *riscv_iommu_probe_devic=
e(struct device *dev)
>
>         mutex_init(&ep->lock);
>         INIT_LIST_HEAD(&ep->domain);
> +       INIT_LIST_HEAD(&ep->regions);
>
>         if (dev_is_pci(dev)) {
>                 ep->devid =3D pci_dev_id(to_pci_dev(dev));
> @@ -1248,6 +1318,7 @@ static struct iommu_device *riscv_iommu_probe_devic=
e(struct device *dev)
>         dev_iommu_priv_set(dev, ep);
>         riscv_iommu_add_device(iommu, dev);
>         riscv_iommu_enable_ep(ep);
> +       riscv_iommu_enable_ir(ep);
>
>         return &iommu->iommu;
>  }
> @@ -1279,6 +1350,7 @@ static void riscv_iommu_release_device(struct devic=
e *dev)
>                 riscv_iommu_iodir_inv_devid(iommu, ep->devid);
>         }
>
> +       riscv_iommu_disable_ir(ep);
>         riscv_iommu_disable_ep(ep);
>
>         /* Remove endpoint from IOMMU tracking structures */
> @@ -1301,6 +1373,15 @@ static struct iommu_group *riscv_iommu_device_grou=
p(struct device *dev)
>
>  static void riscv_iommu_get_resv_regions(struct device *dev, struct list=
_head *head)
>  {
> +       struct iommu_resv_region *entry, *new_entry;
> +       struct riscv_iommu_endpoint *ep =3D dev_iommu_priv_get(dev);
> +
> +       list_for_each_entry(entry, &ep->regions, list) {
> +               new_entry =3D kmemdup(entry, sizeof(*entry), GFP_KERNEL);
> +               if (new_entry)
> +                       list_add_tail(&new_entry->list, head);
> +       }
> +
>         iommu_dma_get_resv_regions(dev, head);
>  }
>
> diff --git a/drivers/iommu/riscv/iommu.h b/drivers/iommu/riscv/iommu.h
> index 83e8d00fd0f8..55418a1144fb 100644
> --- a/drivers/iommu/riscv/iommu.h
> +++ b/drivers/iommu/riscv/iommu.h
> @@ -117,14 +117,17 @@ struct riscv_iommu_endpoint {
>         struct riscv_iommu_dc *dc;              /* device context pointer=
 */
>         struct riscv_iommu_pc *pc;              /* process context root, =
valid if pasid_enabled is true */
>         struct riscv_iommu_device *iommu;       /* parent iommu device */
> +       struct riscv_iommu_msi_pte *msi_root;   /* interrupt re-mapping *=
/
>
>         struct mutex lock;
>         struct list_head domain;                /* endpoint attached mana=
ged domain */
> +       struct list_head regions;               /* reserved regions, inte=
rrupt remapping window */
>
>         /* end point info bits */
>         unsigned pasid_bits;
>         unsigned pasid_feat;
>         bool pasid_enabled;
> +       bool ir_enabled;
>  };
>
>  /* Helper functions and macros */
> --
> 2.34.1
>
>
> _______________________________________________
> linux-riscv mailing list
> linux-riscv@lists.infradead.org
> http://lists.infradead.org/mailman/listinfo/linux-riscv