pch_uart_interrupt() takes priv->port.lock which leads to two recursive
spinlock calls if low_latency==1 or CONFIG_PREEMPT_RT_FULL=y (one
otherwise):
pch_uart_interrupt
spin_lock_irqsave(priv->port.lock, flags)
case PCH_UART_IID_RDR_TO (data ready)
handle_rx_to
push_rx
tty_port_tty_get
spin_lock_irqsave(&port->lock, flags) <--- already hold this lock
...
tty_flip_buffer_push
...
flush_to_ldisc
spin_lock_irqsave(&tty->buf.lock)
spin_lock_irqsave(&tty->buf.lock)
disc->ops->receive_buf(tty, char_buf)
n_tty_receive_buf
tty->ops->flush_chars()
uart_flush_chars
uart_start
spin_lock_irqsave(&port->lock) <--- already hold this lock
Avoid this by using a dedicated lock to protect the eg20t_port structure
and IO access to its membase. This is more consistent with the 8250
driver. Ensure priv->lock is always take prior to priv->port.lock when
taken at the same time.
Tomoya, I have attempted to protect the eg20t_port structure and the membase
with the new eg20t_port.lock field, however I believe the current locking to be
a bit coarse. I imagine the priv->lock could be held a bit less. What are your
thoughts?
Signed-off-by: Darren Hart <[email protected]>
CC: Tomoya MORINAGA <[email protected]>
CC: Feng Tang <[email protected]>
CC: Alexander Stein <[email protected]>
CC: Greg Kroah-Hartman <[email protected]>
CC: Alan Cox <[email protected]>
CC: [email protected]
---
drivers/tty/serial/pch_uart.c | 22 ++++++++++++++++------
1 files changed, 16 insertions(+), 6 deletions(-)
diff --git a/drivers/tty/serial/pch_uart.c b/drivers/tty/serial/pch_uart.c
index 4fdec6a..c90a8cd 100644
--- a/drivers/tty/serial/pch_uart.c
+++ b/drivers/tty/serial/pch_uart.c
@@ -253,6 +253,9 @@ struct eg20t_port {
dma_addr_t rx_buf_dma;
struct dentry *debugfs;
+
+ /* protect the eg20t_port private structure and io access to membase */
+ spinlock_t lock;
};
/**
@@ -1058,7 +1061,7 @@ static irqreturn_t pch_uart_interrupt(int irq, void *dev_id)
int next = 1;
u8 msr;
- spin_lock_irqsave(&priv->port.lock, flags);
+ spin_lock_irqsave(&priv->lock, flags);
handled = 0;
while (next) {
iid = pch_uart_hal_get_iid(priv);
@@ -1116,7 +1119,7 @@ static irqreturn_t pch_uart_interrupt(int irq, void *dev_id)
handled |= (unsigned int)ret;
}
- spin_unlock_irqrestore(&priv->port.lock, flags);
+ spin_unlock_irqrestore(&priv->lock, flags);
return IRQ_RETVAL(handled);
}
@@ -1226,9 +1229,9 @@ static void pch_uart_break_ctl(struct uart_port *port, int ctl)
unsigned long flags;
priv = container_of(port, struct eg20t_port, port);
- spin_lock_irqsave(&port->lock, flags);
+ spin_lock_irqsave(&priv->lock, flags);
pch_uart_hal_set_break(priv, ctl);
- spin_unlock_irqrestore(&port->lock, flags);
+ spin_unlock_irqrestore(&priv->lock, flags);
}
/* Grab any interrupt resources and initialise any low level driver state. */
@@ -1376,7 +1379,8 @@ static void pch_uart_set_termios(struct uart_port *port,
baud = uart_get_baud_rate(port, termios, old, 0, port->uartclk / 16);
- spin_lock_irqsave(&port->lock, flags);
+ spin_lock_irqsave(&priv->lock, flags);
+ spin_lock(&port->lock);
uart_update_timeout(port, termios->c_cflag, baud);
rtn = pch_uart_hal_set_line(priv, baud, parity, bits, stb);
@@ -1389,7 +1393,8 @@ static void pch_uart_set_termios(struct uart_port *port,
tty_termios_encode_baud_rate(termios, baud, baud);
out:
- spin_unlock_irqrestore(&port->lock, flags);
+ spin_unlock(&port->lock);
+ spin_unlock_irqrestore(&priv->lock, flags);
}
static const char *pch_uart_type(struct uart_port *port)
@@ -1546,6 +1551,7 @@ pch_console_write(struct console *co, const char *s, unsigned int count)
touch_nmi_watchdog();
local_irq_save(flags);
+ spin_lock(&priv->lock);
if (priv->port.sysrq) {
/* serial8250_handle_port() already took the lock */
locked = 0;
@@ -1572,7 +1578,9 @@ pch_console_write(struct console *co, const char *s, unsigned int count)
if (locked)
spin_unlock(&priv->port.lock);
+ spin_unlock(&priv->lock);
local_irq_restore(flags);
+
}
static int __init pch_console_setup(struct console *co, char *options)
@@ -1669,6 +1677,8 @@ static struct eg20t_port *pch_uart_init_port(struct pci_dev *pdev,
pci_enable_msi(pdev);
pci_set_master(pdev);
+ spin_lock_init(&priv->lock);
+
iobase = pci_resource_start(pdev, 0);
mapbase = pci_resource_start(pdev, 1);
priv->mapbase = mapbase;
--
1.7.5.4
On Thu, May 31, 2012 at 5:54 PM, Darren Hart <[email protected]> wrote:
> @@ -1376,7 +1379,8 @@ static void pch_uart_set_termios(struct uart_port *port,
>
> ? ? ? ?baud = uart_get_baud_rate(port, termios, old, 0, port->uartclk / 16);
>
> - ? ? ? spin_lock_irqsave(&port->lock, flags);
> + ? ? ? spin_lock_irqsave(&priv->lock, flags);
> + ? ? ? spin_lock(&port->lock);
>
> ? ? ? ?uart_update_timeout(port, termios->c_cflag, baud);
> ? ? ? ?rtn = pch_uart_hal_set_line(priv, baud, parity, bits, stb);
> @@ -1389,7 +1393,8 @@ static void pch_uart_set_termios(struct uart_port *port,
> ? ? ? ? ? ? ? ?tty_termios_encode_baud_rate(termios, baud, baud);
>
> ?out:
> - ? ? ? spin_unlock_irqrestore(&port->lock, flags);
> + ? ? ? spin_unlock(&port->lock);
> + ? ? ? spin_unlock_irqrestore(&priv->lock, flags);
> ?}
Are both port->lock and priv->lock really necessary ?
> @@ -1572,7 +1578,9 @@ pch_console_write(struct console *co, const char *s, unsigned int count)
>
> ? ? ? ?if (locked)
> ? ? ? ? ? ? ? ?spin_unlock(&priv->port.lock);
> + ? ? ? spin_unlock(&priv->lock);
> ? ? ? ?local_irq_restore(flags);
> +
> ?}
Looks spare blank line.
thanks.
--
ROHM Co., Ltd.
tomoya
On 06/01/2012 01:30 AM, Tomoya MORINAGA wrote:
> On Thu, May 31, 2012 at 5:54 PM, Darren Hart <[email protected]> wrote:
>> @@ -1376,7 +1379,8 @@ static void pch_uart_set_termios(struct uart_port *port,
>>
>> baud = uart_get_baud_rate(port, termios, old, 0, port->uartclk / 16);
>>
>> - spin_lock_irqsave(&port->lock, flags);
>> + spin_lock_irqsave(&priv->lock, flags);
>> + spin_lock(&port->lock);
>>
>> uart_update_timeout(port, termios->c_cflag, baud);
>> rtn = pch_uart_hal_set_line(priv, baud, parity, bits, stb);
>> @@ -1389,7 +1393,8 @@ static void pch_uart_set_termios(struct uart_port *port,
>> tty_termios_encode_baud_rate(termios, baud, baud);
>>
>> out:
>> - spin_unlock_irqrestore(&port->lock, flags);
>> + spin_unlock(&port->lock);
>> + spin_unlock_irqrestore(&priv->lock, flags);
>> }
>
> Are both port->lock and priv->lock really necessary ?
The priv lock protects the pch_uart_hal* calls and the io access.
The port lock protects the uart_update_timeout call. I'm assuming the
8250.c driver is correct in holding the port lock before making this
call and making other modifcations to the port struct.
So yes, I believe both are required. The port->lock was used as the lock
to protect the private data in the interrupt handler,
pch_uart_interrupt. If we could avoid holding that lock across the
entire function, limiting it to just around the pch_uart_hal calls
(perhaps by adding it to the hal calls and adding lockless __pch_uart*
calls) we could avoid the recursive lock that occurs with handle_rx. I
still think a priv-lock is a good idea though, even if just to clarify
what is being protected.
Thoughts?
>
>
>> @@ -1572,7 +1578,9 @@ pch_console_write(struct console *co, const char *s, unsigned int count)
>>
>> if (locked)
>> spin_unlock(&priv->port.lock);
>> + spin_unlock(&priv->lock);
>> local_irq_restore(flags);
>> +
>> }
>
> Looks spare blank line.
Thanks, will fix for V2 after this discussion wraps up.
>
> thanks.
--
Darren Hart
Intel Open Source Technology Center
Yocto Project - Linux Kernel
On 06/01/2012 11:36 AM, Darren Hart wrote:
>
>
> On 06/01/2012 01:30 AM, Tomoya MORINAGA wrote:
>> On Thu, May 31, 2012 at 5:54 PM, Darren Hart <[email protected]> wrote:
>>> @@ -1376,7 +1379,8 @@ static void pch_uart_set_termios(struct uart_port *port,
>>>
>>> baud = uart_get_baud_rate(port, termios, old, 0, port->uartclk / 16);
>>>
>>> - spin_lock_irqsave(&port->lock, flags);
>>> + spin_lock_irqsave(&priv->lock, flags);
>>> + spin_lock(&port->lock);
>>>
>>> uart_update_timeout(port, termios->c_cflag, baud);
>>> rtn = pch_uart_hal_set_line(priv, baud, parity, bits, stb);
>>> @@ -1389,7 +1393,8 @@ static void pch_uart_set_termios(struct uart_port *port,
>>> tty_termios_encode_baud_rate(termios, baud, baud);
>>>
>>> out:
>>> - spin_unlock_irqrestore(&port->lock, flags);
>>> + spin_unlock(&port->lock);
>>> + spin_unlock_irqrestore(&priv->lock, flags);
>>> }
>>
>> Are both port->lock and priv->lock really necessary ?
>
> The priv lock protects the pch_uart_hal* calls and the io access.
>
> The port lock protects the uart_update_timeout call. I'm assuming the
> 8250.c driver is correct in holding the port lock before making this
> call and making other modifcations to the port struct.
>
> So yes, I believe both are required. The port->lock was used as the lock
> to protect the private data in the interrupt handler,
> pch_uart_interrupt. If we could avoid holding that lock across the
> entire function, limiting it to just around the pch_uart_hal calls
> (perhaps by adding it to the hal calls and adding lockless __pch_uart*
> calls) we could avoid the recursive lock that occurs with handle_rx. I
> still think a priv-lock is a good idea though, even if just to clarify
> what is being protected.
>
> Thoughts?
Are there still concerns about the additional lock? I'll resend V2
tomorrow with the single whitespace fix if I don't hear anything back today.
Thanks,
Darren
>
>>
>>
>>> @@ -1572,7 +1578,9 @@ pch_console_write(struct console *co, const char *s, unsigned int count)
>>>
>>> if (locked)
>>> spin_unlock(&priv->port.lock);
>>> + spin_unlock(&priv->lock);
>>> local_irq_restore(flags);
>>> +
>>> }
>>
>> Looks spare blank line.
>
> Thanks, will fix for V2 after this discussion wraps up.
>
>>
>> thanks.
>
--
Darren Hart
Intel Open Source Technology Center
Yocto Project - Linux Kernel
On Wed, Jun 6, 2012 at 7:07 AM, Darren Hart <[email protected]> wrote:
> Are there still concerns about the additional lock? I'll resend V2
> tomorrow with the single whitespace fix if I don't hear anything back today.
I understand your saying. Looks good.
However, I am not expert of linux-uart core system.
So, I'd like UART maintainer to give us your opinion.
thanks.
tomoya
On 06/05/2012 04:48 PM, Tomoya MORINAGA wrote:
> On Wed, Jun 6, 2012 at 7:07 AM, Darren Hart <[email protected]> wrote:
>> Are there still concerns about the additional lock? I'll resend V2
>> tomorrow with the single whitespace fix if I don't hear anything back today.
>
> I understand your saying. Looks good.
> However, I am not expert of linux-uart core system.
> So, I'd like UART maintainer to give us your opinion.
Greg, Alan,
any concerns with the locking approach I've adopted in the patch?
Thanks,
--
Darren Hart
Intel Open Source Technology Center
Yocto Project - Linux Kernel
On Mon, Jun 18, 2012 at 02:41:46PM -0700, Darren Hart wrote:
>
>
> On 06/05/2012 04:48 PM, Tomoya MORINAGA wrote:
> > On Wed, Jun 6, 2012 at 7:07 AM, Darren Hart <[email protected]> wrote:
> >> Are there still concerns about the additional lock? I'll resend V2
> >> tomorrow with the single whitespace fix if I don't hear anything back today.
> >
> > I understand your saying. Looks good.
> > However, I am not expert of linux-uart core system.
> > So, I'd like UART maintainer to give us your opinion.
>
> Greg, Alan,
>
> any concerns with the locking approach I've adopted in the patch?
Care to resend the patch, as it was a RFC one, it's no longer in my
queue.
thanks,
greg k-h
On Mon, 18 Jun 2012 14:41:46 -0700
Darren Hart <[email protected]> wrote:
>
>
> On 06/05/2012 04:48 PM, Tomoya MORINAGA wrote:
> > On Wed, Jun 6, 2012 at 7:07 AM, Darren Hart <[email protected]> wrote:
> >> Are there still concerns about the additional lock? I'll resend V2
> >> tomorrow with the single whitespace fix if I don't hear anything back today.
> >
> > I understand your saying. Looks good.
> > However, I am not expert of linux-uart core system.
> > So, I'd like UART maintainer to give us your opinion.
>
> Greg, Alan,
>
> any concerns with the locking approach I've adopted in the patch?
Only the one I noted in my reply the first time around which is that you
can't permit tty->low_latency=1 unless your tty receive path is not an
IRQ path. From a locking point of view the change makes sense anyway.
Going back over it your console locking also needs care - an oops or
printk within the areas the private lock covers will hang the box. That
should also probably be a trylock style lock as with the other lock on
that path
Alan
On 06/19/2012 02:14 AM, Alan Cox wrote:
> On Mon, 18 Jun 2012 14:41:46 -0700
> Darren Hart <[email protected]> wrote:
>
>>
>>
>> On 06/05/2012 04:48 PM, Tomoya MORINAGA wrote:
>>> On Wed, Jun 6, 2012 at 7:07 AM, Darren Hart <[email protected]> wrote:
>>>> Are there still concerns about the additional lock? I'll resend V2
>>>> tomorrow with the single whitespace fix if I don't hear anything back today.
>>>
>>> I understand your saying. Looks good.
>>> However, I am not expert of linux-uart core system.
>>> So, I'd like UART maintainer to give us your opinion.
>>
>> Greg, Alan,
>>
>> any concerns with the locking approach I've adopted in the patch?
>
> Only the one I noted in my reply the first time around
Hi Alan,
I've hunted, but I can't seem to find this reply. :-/
> which is that you
> can't permit tty->low_latency=1 unless your tty receive path is not an
> IRQ path. From a locking point of view the change makes sense anyway.
I ran into this on the PREEMPT_RT kernel which always uses
tty->low_latency and converts the interrupt handler into a thread.
There is a follow-on patch for RT only to address a sleeping while
atomic bug in pch_console_write(), but I felt _this_ locking structure
change was appropriate for mainline.
>
> Going back over it your console locking also needs care - an oops or
> printk within the areas the private lock covers will hang the box. That
> should also probably be a trylock style lock as with the other lock on
> that path
I presume you are referring to pch_console_write()?
> static void
> pch_console_write(struct console *co, const char *s, unsigned int count)
> {
> struct eg20t_port *priv;
> unsigned long flags;
> u8 ier;
> int locked = 1;
>
> priv = pch_uart_ports[co->index];
>
> touch_nmi_watchdog();
>
> local_irq_save(flags);
> spin_lock(&priv->lock);
> if (priv->port.sysrq) {
> /* serial8250_handle_port() already took the lock */
> locked = 0;
> } else if (oops_in_progress) {
> locked = spin_trylock(&priv->port.lock);
> } else
> spin_lock(&priv->port.lock);
I see, the oops_in_progress test right? My thinking was that the
oops_in_progress was only relevant to the port.lock as that could be
taken outside of the pch_uart driver, while the priv.lock is only used
within the driver. But, as the oops uses the pch_console_write itself, I
can see the recursive spinlock failure case there.
As for the printk, it seems the 8250 driver would also suffer from that
in the serial8250_console_write function on the port.lock, and it does
not make any allowances for printk.
I would like to hold the priv.lock for a smaller window, but ordering
requires that I take it prior to the port.lock.
So I can test for oops_in_progress on the priv->lock too, but that won't
address the printk issue. Is the oops the bigger concern?
--
Darren Hart
Intel Open Source Technology Center
Yocto Project - Linux Kernel
> I see, the oops_in_progress test right? My thinking was that the
> oops_in_progress was only relevant to the port.lock as that could be
> taken outside of the pch_uart driver, while the priv.lock is only used
> within the driver. But, as the oops uses the pch_console_write itself, I
> can see the recursive spinlock failure case there.
Until your driver crashes...
> As for the printk, it seems the 8250 driver would also suffer from that
> in the serial8250_console_write function on the port.lock, and it does
> not make any allowances for printk.
I think 8250 probably wants fixing too then!
>
> I would like to hold the priv.lock for a smaller window, but ordering
> requires that I take it prior to the port.lock.
>
> So I can test for oops_in_progress on the priv->lock too, but that won't
> address the printk issue. Is the oops the bigger concern?
the oops is the main one - a printk would have to be in driver as a
screwup, and you can force an oops on a stall so pick it up later
Alan