Date:   Thu, 09 Feb 2023 08:07:05 +0000
Message-ID: <874jrv1cuu.wl-maz@kernel.org>
From:   Marc Zyngier <maz@kernel.org>
To:     Mason Huo <mason.huo@starfivetech.com>
Cc:     Thomas Gleixner <tglx@linutronix.de>,
        Palmer Dabbelt <palmer@dabbelt.com>,
        Paul Walmsley <paul.walmsley@sifive.com>,
        <linux-kernel@vger.kernel.org>, <linux-riscv@lists.infradead.org>,
        Ley Foon Tan <leyfoon.tan@starfivetech.com>,
        Sia Jee Heng <jeeheng.sia@starfivetech.com>
Subject: Re: [PATCH v2] irqchip/irq-sifive-plic: Add syscore callbacks for hibernation
In-Reply-To: <20230209034322.69943-1-mason.huo@starfivetech.com>
References: <20230209034322.69943-1-mason.huo@starfivetech.com>
User-Agent: Wanderlust/2.15.9 (Almost Unreal) SEMI-EPG/1.14.7 (Harue)
 FLIM-LB/1.14.9 (=?UTF-8?B?R29qxY0=?=) APEL-LB/10.8 EasyPG/1.0.0 Emacs/27.1
 (x86_64-pc-linux-gnu) MULE/6.0 (HANACHIRUSATO)
MIME-Version: 1.0 (generated by SEMI-EPG 1.14.7 - "Harue")
Content-Type: text/plain; charset=US-ASCII
Precedence: bulk

On Thu, 09 Feb 2023 03:43:22 +0000,
Mason Huo <mason.huo@starfivetech.com> wrote:
> 
> The priority and enable registers of plic will be reset
> during hibernation power cycle in poweroff mode,
> add the syscore callbacks to save/restore those registers.
> 
> Signed-off-by: Mason Huo <mason.huo@starfivetech.com>
> Reviewed-by: Ley Foon Tan <leyfoon.tan@starfivetech.com>
> Reviewed-by: Sia Jee Heng <jeeheng.sia@starfivetech.com>
> ---
>  drivers/irqchip/irq-sifive-plic.c | 94 ++++++++++++++++++++++++++++++-
>  1 file changed, 92 insertions(+), 2 deletions(-)
> 
> diff --git a/drivers/irqchip/irq-sifive-plic.c b/drivers/irqchip/irq-sifive-plic.c
> index ff47bd0dec45..4683e49d90ad 100644
> --- a/drivers/irqchip/irq-sifive-plic.c
> +++ b/drivers/irqchip/irq-sifive-plic.c
> @@ -17,6 +17,7 @@
>  #include <linux/of_irq.h>
>  #include <linux/platform_device.h>
>  #include <linux/spinlock.h>
> +#include <linux/syscore_ops.h>
>  #include <asm/smp.h>
>  
>  /*
> @@ -67,6 +68,8 @@ struct plic_priv {
>  	struct irq_domain *irqdomain;
>  	void __iomem *regs;
>  	unsigned long plic_quirks;
> +	unsigned int nr_irqs;
> +	unsigned long *priority_reg;

This isn't a pointer to registers. This is a save area for the
values. Please fix the naming.

>  };
>  
>  struct plic_handler {
> @@ -78,11 +81,13 @@ struct plic_handler {
>  	 */
>  	raw_spinlock_t		enable_lock;
>  	void __iomem		*enable_base;
> +	u32			*enable_reg;

Same thing here.

>  	struct plic_priv	*priv;
>  };
>  static int plic_parent_irq __ro_after_init;
>  static bool plic_cpuhp_setup_done __ro_after_init;
>  static DEFINE_PER_CPU(struct plic_handler, plic_handlers);
> +static struct plic_priv *priv_data;
>  
>  static int plic_irq_set_type(struct irq_data *d, unsigned int type);
>  
> @@ -229,6 +234,68 @@ static int plic_irq_set_type(struct irq_data *d, unsigned int type)
>  	return IRQ_SET_MASK_OK;
>  }
>  
> +static int plic_irq_suspend(void)
> +{
> +	unsigned int i, cpu;
> +	u32 __iomem *reg;
> +
> +	for (i = 0; i < priv_data->nr_irqs; i++)
> +		if (readl(priv_data->regs + PRIORITY_BASE + i * PRIORITY_PER_ID))
> +			__set_bit(i, priv_data->priority_reg);
> +		else
> +			__clear_bit(i, priv_data->priority_reg);
> +
> +	for_each_cpu(cpu, cpu_present_mask) {
> +		struct plic_handler *handler = per_cpu_ptr(&plic_handlers, cpu);
> +
> +		if (!handler->present)
> +			continue;
> +
> +		raw_spin_lock(&handler->enable_lock);
> +		for (i = 0; i < DIV_ROUND_UP(priv_data->nr_irqs, 32); i++) {
> +			reg = handler->enable_base + i * sizeof(u32);
> +			handler->enable_reg[i] = readl(reg);
> +		}
> +		raw_spin_unlock(&handler->enable_lock);
> +	}
> +
> +	return 0;
> +}
> +
> +static void plic_irq_resume(void)
> +{
> +	unsigned int i, cpu;
> +	u32 __iomem *reg;
> +
> +	for (i = 0; i < priv_data->nr_irqs; i++)
> +		writel(test_bit(i, priv_data->priority_reg),
> +				priv_data->regs + PRIORITY_BASE + i * PRIORITY_PER_ID);

I suggest you write the priority value instead of the result of
test_bit(). Yes, they are the same for now. They may change in the
future.

> +
> +	for_each_cpu(cpu, cpu_present_mask) {
> +		struct plic_handler *handler = per_cpu_ptr(&plic_handlers, cpu);
> +
> +		if (!handler->present)
> +			continue;
> +
> +		raw_spin_lock(&handler->enable_lock);
> +		for (i = 0; i < DIV_ROUND_UP(priv_data->nr_irqs, 32); i++) {
> +			reg = handler->enable_base + i * sizeof(u32);
> +			writel(handler->enable_reg[i], reg);
> +		}
> +		raw_spin_unlock(&handler->enable_lock);
> +	}
> +}
> +
> +static struct syscore_ops plic_irq_syscore_ops = {
> +	.suspend	= plic_irq_suspend,
> +	.resume		= plic_irq_resume,
> +};
> +
> +static void plic_irq_pm_init(void)
> +{
> +	register_syscore_ops(&plic_irq_syscore_ops);
> +}

I think we can live without this single line helper.

> +
>  static int plic_irqdomain_map(struct irq_domain *d, unsigned int irq,
>  			      irq_hw_number_t hwirq)
>  {
> @@ -345,12 +412,14 @@ static int __init __plic_init(struct device_node *node,
>  	u32 nr_irqs;
>  	struct plic_priv *priv;
>  	struct plic_handler *handler;
> +	unsigned int cpu;
>  
>  	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
>  	if (!priv)
>  		return -ENOMEM;
>  
>  	priv->plic_quirks = plic_quirks;
> +	priv_data = priv;

And what happens if you have more than a single PLIC in the system, as
described in [1]?

You also already have this pointer in each per-CPU handler structure.
Why do you need a global one?

>  
>  	priv->regs = of_iomap(node, 0);
>  	if (WARN_ON(!priv->regs)) {
> @@ -363,15 +432,23 @@ static int __init __plic_init(struct device_node *node,
>  	if (WARN_ON(!nr_irqs))
>  		goto out_iounmap;
>  
> +	priv->nr_irqs = nr_irqs;
> +
> +	priv->priority_reg = kcalloc(DIV_ROUND_UP(nr_irqs,
> +		sizeof(unsigned long) * 8),
> +		sizeof(unsigned long), GFP_KERNEL);

Is this trying to be a substitute to bitmap_alloc()?

> +	if (!priv->priority_reg)
> +		goto out_free_priority_reg;
> +
>  	nr_contexts = of_irq_count(node);
>  	if (WARN_ON(!nr_contexts))
> -		goto out_iounmap;
> +		goto out_free_priority_reg;
>  
>  	error = -ENOMEM;
>  	priv->irqdomain = irq_domain_add_linear(node, nr_irqs + 1,
>  			&plic_irqdomain_ops, priv);
>  	if (WARN_ON(!priv->irqdomain))
> -		goto out_iounmap;
> +		goto out_free_priority_reg;
>  
>  	for (i = 0; i < nr_contexts; i++) {
>  		struct of_phandle_args parent;
> @@ -441,6 +518,11 @@ static int __init __plic_init(struct device_node *node,
>  		handler->enable_base = priv->regs + CONTEXT_ENABLE_BASE +
>  			i * CONTEXT_ENABLE_SIZE;
>  		handler->priv = priv;
> +
> +		handler->enable_reg =  kcalloc(DIV_ROUND_UP(nr_irqs, 32),
> +			32, GFP_KERNEL);

nit: either you write this on a single line, or you align the second
line with the opening bracket of the previous one.

> +		if (!handler->enable_reg)
> +			goto out_free_enable_reg;
>  done:
>  		for (hwirq = 1; hwirq <= nr_irqs; hwirq++) {
>  			plic_toggle(handler, hwirq, 0);
> @@ -461,11 +543,19 @@ static int __init __plic_init(struct device_node *node,
>  				  plic_starting_cpu, plic_dying_cpu);
>  		plic_cpuhp_setup_done = true;
>  	}
> +	plic_irq_pm_init();
>
>  	pr_info("%pOFP: mapped %d interrupts with %d handlers for"
>  		" %d contexts.\n", node, nr_irqs, nr_handlers, nr_contexts);
>  	return 0;
>  
> +out_free_enable_reg:
> +	for_each_cpu(cpu, cpu_present_mask) {
> +		handler = per_cpu_ptr(&plic_handlers, cpu);
> +		kfree(handler->enable_reg);
> +	}
> +out_free_priority_reg:
> +	kfree(priv->priority_reg);
>  out_iounmap:
>  	iounmap(priv->regs);
>  out_free_priv:

	M.

[1] https://patchwork.kernel.org/project/linux-riscv/patch/20200302231146.15530-3-atish.patra@wdc.com/

-- 
Without deviation from the norm, progress is not possible.