Received-SPF: pass (google.com: domain of linux-kernel+bounces-70697-linux.lists.archive=gmail.com@vger.kernel.org designates 147.75.199.223 as permitted sender) client-ip=147.75.199.223;
Precedence: bulk
MIME-Version: 1.0
References: <20240127161753.114685-1-apatel@ventanamicro.com>
 <20240127161753.114685-20-apatel@ventanamicro.com> <87eddccgo7.ffs@tglx>
In-Reply-To: <87eddccgo7.ffs@tglx>
From: Anup Patel <apatel@ventanamicro.com>
Date: Mon, 19 Feb 2024 09:40:35 +0530
Message-ID: <CAK9=C2VK-sOtyhxnDH83WwMNQbetC-zLssZdYG399pBghU+yUw@mail.gmail.com>
Subject: Re: [PATCH v12 19/25] irqchip/riscv-imsic: Add device MSI domain
 support for platform devices
To: Thomas Gleixner <tglx@linutronix.de>
Cc: Palmer Dabbelt <palmer@dabbelt.com>, Paul Walmsley <paul.walmsley@sifive.com>, 
	Rob Herring <robh+dt@kernel.org>, 
	Krzysztof Kozlowski <krzysztof.kozlowski+dt@linaro.org>, Frank Rowand <frowand.list@gmail.com>, 
	Conor Dooley <conor+dt@kernel.org>, Marc Zyngier <maz@kernel.org>, =?UTF-8?B?QmrDtnJuIFTDtnBlbA==?= <bjorn@kernel.org>, 
	Atish Patra <atishp@atishpatra.org>, Andrew Jones <ajones@ventanamicro.com>, 
	Sunil V L <sunilvl@ventanamicro.com>, Saravana Kannan <saravanak@google.com>, 
	Anup Patel <anup@brainfault.org>, linux-riscv@lists.infradead.org, 
	linux-arm-kernel@lists.infradead.org, linux-kernel@vger.kernel.org, 
	devicetree@vger.kernel.org
Content-Type: text/plain; charset="UTF-8"
Content-Transfer-Encoding: quoted-printable

On Sat, Feb 17, 2024 at 1:42=E2=80=AFAM Thomas Gleixner <tglx@linutronix.de=
> wrote:
>
> On Sat, Jan 27 2024 at 21:47, Anup Patel wrote:
> > +static int imsic_cpu_page_phys(unsigned int cpu,
> > +                            unsigned int guest_index,
> > +                            phys_addr_t *out_msi_pa)
> > +{
> > +     struct imsic_global_config *global;
> > +     struct imsic_local_config *local;
> > +
> > +     global =3D &imsic->global;
> > +     local =3D per_cpu_ptr(global->local, cpu);
> > +
> > +     if (BIT(global->guest_index_bits) <=3D guest_index)
> > +             return -EINVAL;
>
> As the callsite does not care about the return value, just make this
> function boolean and return true on success.

Okay, I will update.

>
> > +     if (out_msi_pa)
> > +             *out_msi_pa =3D local->msi_pa +
> > +                           (guest_index * IMSIC_MMIO_PAGE_SZ);
> > +
> > +     return 0;
> > +}
> > +
> > +static void imsic_irq_mask(struct irq_data *d)
> > +{
> > +     imsic_vector_mask(irq_data_get_irq_chip_data(d));
> > +}
> > +
> > +static void imsic_irq_unmask(struct irq_data *d)
> > +{
> > +     imsic_vector_unmask(irq_data_get_irq_chip_data(d));
> > +}
> > +
> > +static int imsic_irq_retrigger(struct irq_data *d)
> > +{
> > +     struct imsic_vector *vec =3D irq_data_get_irq_chip_data(d);
> > +     struct imsic_local_config *local;
> > +
> > +     if (WARN_ON(vec =3D=3D NULL))
> > +             return -ENOENT;
> > +
> > +     local =3D per_cpu_ptr(imsic->global.local, vec->cpu);
> > +     writel(vec->local_id, local->msi_va);
> > +     return 0;
> > +}
> > +
> > +static void imsic_irq_compose_vector_msg(struct imsic_vector *vec,
> > +                                      struct msi_msg *msg)
> > +{
> > +     phys_addr_t msi_addr;
> > +     int err;
> > +
> > +     if (WARN_ON(vec =3D=3D NULL))
> > +             return;
> > +
> > +     err =3D imsic_cpu_page_phys(vec->cpu, 0, &msi_addr);
> > +     if (WARN_ON(err))
> > +             return;
>
>         if (WARN_ON(!imsic_cpu_page_phys(...)))
>                 return
> Hmm?

Okay, I will update like you suggested.

>
> > +
> > +     msg->address_hi =3D upper_32_bits(msi_addr);
> > +     msg->address_lo =3D lower_32_bits(msi_addr);
> > +     msg->data =3D vec->local_id;
> > +}
> > +
> > +static void imsic_irq_compose_msg(struct irq_data *d, struct msi_msg *=
msg)
> > +{
> > +     imsic_irq_compose_vector_msg(irq_data_get_irq_chip_data(d), msg);
> > +}
> > +
> > +#ifdef CONFIG_SMP
> > +static void imsic_msi_update_msg(struct irq_data *d, struct imsic_vect=
or *vec)
> > +{
> > +     struct msi_msg msg[2] =3D { [1] =3D { }, };
> > +
> > +     imsic_irq_compose_vector_msg(vec, msg);
> > +     irq_data_get_irq_chip(d)->irq_write_msi_msg(d, msg);
> > +}
> > +
> > +static int imsic_irq_set_affinity(struct irq_data *d,
> > +                               const struct cpumask *mask_val,
> > +                               bool force)
> > +{
> > +     struct imsic_vector *old_vec, *new_vec;
> > +     struct irq_data *pd =3D d->parent_data;
> > +
> > +     old_vec =3D irq_data_get_irq_chip_data(pd);
> > +     if (WARN_ON(old_vec =3D=3D NULL))
> > +             return -ENOENT;
> > +
> > +     /* Get a new vector on the desired set of CPUs */
> > +     new_vec =3D imsic_vector_alloc(old_vec->hwirq, mask_val);
> > +     if (!new_vec)
> > +             return -ENOSPC;
> > +
> > +     /* If old vector belongs to the desired CPU then do nothing */
> > +     if (old_vec->cpu =3D=3D new_vec->cpu) {
> > +             imsic_vector_free(new_vec);
> > +             return IRQ_SET_MASK_OK_DONE;
> > +     }
>
> You can spare that exercise by checking it before the allocation:
>
>         if (cpumask_test_cpu(old_vec->cpu, mask_val))
>                 return IRQ_SET_MASK_OK_DONE;

Okay, I will update.

>
> > +
> > +     /* Point device to the new vector */
> > +     imsic_msi_update_msg(d, new_vec);
>
> > +static int imsic_irq_domain_alloc(struct irq_domain *domain,
> > +                               unsigned int virq, unsigned int nr_irqs=
,
> > +                               void *args)
> > +{
> > +     struct imsic_vector *vec;
> > +     int hwirq;
> > +
> > +     /* Legacy-MSI or multi-MSI not supported yet. */
>
> What's legacy MSI in that context?

The legacy-MSI is the MSI support in PCI v2.2 where
number of MSIs allocated by device were either 1, 2, 4,
8, 16, or 32 and the data written is <data_word> + <irqnum>.

>
> > +     if (nr_irqs > 1)
> > +             return -ENOTSUPP;
> > +
> > +     hwirq =3D imsic_hwirq_alloc();
> > +     if (hwirq < 0)
> > +             return hwirq;
> > +
> > +     vec =3D imsic_vector_alloc(hwirq, cpu_online_mask);
> > +     if (!vec) {
> > +             imsic_hwirq_free(hwirq);
> > +             return -ENOSPC;
> > +     }
> > +
> > +     irq_domain_set_info(domain, virq, hwirq,
> > +                         &imsic_irq_base_chip, vec,
> > +                         handle_simple_irq, NULL, NULL);
> > +     irq_set_noprobe(virq);
> > +     irq_set_affinity(virq, cpu_online_mask);
> > +
> > +     /*
> > +      * IMSIC does not implement irq_disable() so Linux interrupt
> > +      * subsystem will take a lazy approach for disabling an IMSIC
> > +      * interrupt. This means IMSIC interrupts are left unmasked
> > +      * upon system suspend and interrupts are not processed
> > +      * immediately upon system wake up. To tackle this, we disable
> > +      * the lazy approach for all IMSIC interrupts.
>
> Why? Lazy works perfectly fine even w/o an irq_disable() callback.

This was suggested by SiFive folks. I am also not sure why we
need this. For now, I will drop this and bring it back as a separate
patch if required.

>
> > +      */
> > +     irq_set_status_flags(virq, IRQ_DISABLE_UNLAZY);
>
> > +
> > +#define MATCH_PLATFORM_MSI           BIT(DOMAIN_BUS_PLATFORM_MSI)
>
> You really love macro indirections :)

This is to be consistent with MATCH_PCI_MSI introduced by the
subsequent patch.

Also, this is inspired from your ARM GIC patches.
https://lore.kernel.org/linux-arm-kernel/20221121140049.038269899@linutroni=
x.de/
https://lore.kernel.org/linux-arm-kernel/20221121140049.112451419@linutroni=
x.de/
https://lore.kernel.org/linux-arm-kernel/20221121140049.237988384@linutroni=
x.de/
https://lore.kernel.org/linux-arm-kernel/20221121140049.941784867@linutroni=
x.de/

>
> > +static const struct msi_parent_ops imsic_msi_parent_ops =3D {
> > +     .supported_flags        =3D MSI_GENERIC_FLAGS_MASK,
> > +     .required_flags         =3D MSI_FLAG_USE_DEF_DOM_OPS |
> > +                               MSI_FLAG_USE_DEF_CHIP_OPS,
> > +     .bus_select_token       =3D DOMAIN_BUS_NEXUS,
> > +     .bus_select_mask        =3D MATCH_PLATFORM_MSI,
> > +     .init_dev_msi_info      =3D imsic_init_dev_msi_info,
> > +};
> > +
> > +int imsic_irqdomain_init(void)
> > +{
> > +     struct imsic_global_config *global;
> > +
> > +     if (!imsic || !imsic->fwnode) {
> > +             pr_err("early driver not probed\n");
> > +             return -ENODEV;
> > +     }
> > +
> > +     if (imsic->base_domain) {
> > +             pr_err("%pfwP: irq domain already created\n", imsic->fwno=
de);
> > +             return -ENODEV;
> > +     }
> > +
> > +     global =3D &imsic->global;
>
> Please move that assignment down to the usage site. Here it's just a
> distraction.

Okay, I will update.

>
> > +     /* Create Base IRQ domain */
> > +     imsic->base_domain =3D irq_domain_create_tree(imsic->fwnode,
> > +                                     &imsic_base_domain_ops, imsic);
> > +     if (!imsic->base_domain) {
> > +             pr_err("%pfwP: failed to create IMSIC base domain\n",
> > +                     imsic->fwnode);
> > +             return -ENOMEM;
> > +     }
> > +     imsic->base_domain->flags |=3D IRQ_DOMAIN_FLAG_MSI_PARENT;
> > +     imsic->base_domain->msi_parent_ops =3D &imsic_msi_parent_ops;
>

Regards,
Anup