2024-05-20 15:21:57

by Steven Sistare

Subject: [PATCH V3 3/3] vdpa_sim: flush workers on suspend

Flush to guarantee no workers are running when suspend returns.
Add a lock to enforce ordering between clearing running, flushing,
and posting new work in vdpasim_kick_vq. It must be a spin lock
because vdpasim_kick_vq may be reached va eventfd_write.

Signed-off-by: Steve Sistare <[email protected]>
---
drivers/vdpa/vdpa_sim/vdpa_sim.c | 16 ++++++++++++++--
drivers/vdpa/vdpa_sim/vdpa_sim.h | 1 +
2 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/drivers/vdpa/vdpa_sim/vdpa_sim.c b/drivers/vdpa/vdpa_sim/vdpa_sim.c
index 8ffea8430f95..67ed49d95bf0 100644
--- a/drivers/vdpa/vdpa_sim/vdpa_sim.c
+++ b/drivers/vdpa/vdpa_sim/vdpa_sim.c
@@ -322,7 +322,7 @@ static u16 vdpasim_get_vq_size(struct vdpa_device *vdpa, u16 idx)
return VDPASIM_QUEUE_MAX;
}

-static void vdpasim_kick_vq(struct vdpa_device *vdpa, u16 idx)
+static void vdpasim_do_kick_vq(struct vdpa_device *vdpa, u16 idx)
{
struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];
@@ -337,6 +337,15 @@ static void vdpasim_kick_vq(struct vdpa_device *vdpa, u16 idx)
vdpasim_schedule_work(vdpasim);
}

+static void vdpasim_kick_vq(struct vdpa_device *vdpa, u16 idx)
+{
+ struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
+
+ spin_lock(&vdpasim->kick_lock);
+ vdpasim_do_kick_vq(vdpa, idx);
+ spin_unlock(&vdpasim->kick_lock);
+}
+
static void vdpasim_set_vq_cb(struct vdpa_device *vdpa, u16 idx,
struct vdpa_callback *cb)
{
@@ -520,8 +529,11 @@ static int vdpasim_suspend(struct vdpa_device *vdpa)
struct vdpasim *vdpasim = vdpa_to_sim(vdpa);

mutex_lock(&vdpasim->mutex);
+ spin_lock(&vdpasim->kick_lock);
vdpasim->running = false;
+ spin_unlock(&vdpasim->kick_lock);
mutex_unlock(&vdpasim->mutex);
+ kthread_flush_work(&vdpasim->work);

return 0;
}
@@ -537,7 +549,7 @@ static int vdpasim_resume(struct vdpa_device *vdpa)
if (vdpasim->pending_kick) {
/* Process pending descriptors */
for (i = 0; i < vdpasim->dev_attr.nvqs; ++i)
- vdpasim_kick_vq(vdpa, i);
+ vdpasim_do_kick_vq(vdpa, i);

vdpasim->pending_kick = false;
}
diff --git a/drivers/vdpa/vdpa_sim/vdpa_sim.h b/drivers/vdpa/vdpa_sim/vdpa_sim.h
index bb137e479763..5eb6ca9c5ec5 100644
--- a/drivers/vdpa/vdpa_sim/vdpa_sim.h
+++ b/drivers/vdpa/vdpa_sim/vdpa_sim.h
@@ -75,6 +75,7 @@ struct vdpasim {
bool pending_kick;
/* spinlock to synchronize iommu table */
spinlock_t iommu_lock;
+ spinlock_t kick_lock;
};

struct vdpasim *vdpasim_create(struct vdpasim_dev_attr *attr,
--
2.39.3

2024-05-21 02:32:57

by Jason Wang

Subject: Re: [PATCH V3 3/3] vdpa_sim: flush workers on suspend

On Mon, May 20, 2024 at 11:21 PM Steve Sistare
<[email protected]> wrote:
>
> Flush to guarantee no workers are running when suspend returns.
> Add a lock to enforce ordering between clearing running, flushing,
> and posting new work in vdpasim_kick_vq. It must be a spin lock
> because vdpasim_kick_vq may be reached va eventfd_write.
>
> Signed-off-by: Steve Sistare <[email protected]>
> ---
> drivers/vdpa/vdpa_sim/vdpa_sim.c | 16 ++++++++++++++--
> drivers/vdpa/vdpa_sim/vdpa_sim.h | 1 +
> 2 files changed, 15 insertions(+), 2 deletions(-)
>
> diff --git a/drivers/vdpa/vdpa_sim/vdpa_sim.c b/drivers/vdpa/vdpa_sim/vdpa_sim.c
> index 8ffea8430f95..67ed49d95bf0 100644
> --- a/drivers/vdpa/vdpa_sim/vdpa_sim.c
> +++ b/drivers/vdpa/vdpa_sim/vdpa_sim.c
> @@ -322,7 +322,7 @@ static u16 vdpasim_get_vq_size(struct vdpa_device *vdpa, u16 idx)
> return VDPASIM_QUEUE_MAX;
> }
>
> -static void vdpasim_kick_vq(struct vdpa_device *vdpa, u16 idx)
> +static void vdpasim_do_kick_vq(struct vdpa_device *vdpa, u16 idx)
> {
> struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
> struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];
> @@ -337,6 +337,15 @@ static void vdpasim_kick_vq(struct vdpa_device *vdpa, u16 idx)
> vdpasim_schedule_work(vdpasim);
> }
>
> +static void vdpasim_kick_vq(struct vdpa_device *vdpa, u16 idx)
> +{
> + struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
> +
> + spin_lock(&vdpasim->kick_lock);
> + vdpasim_do_kick_vq(vdpa, idx);
> + spin_unlock(&vdpasim->kick_lock);
> +}
> +
> static void vdpasim_set_vq_cb(struct vdpa_device *vdpa, u16 idx,
> struct vdpa_callback *cb)
> {
> @@ -520,8 +529,11 @@ static int vdpasim_suspend(struct vdpa_device *vdpa)
> struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
>
> mutex_lock(&vdpasim->mutex);
> + spin_lock(&vdpasim->kick_lock);
> vdpasim->running = false;
> + spin_unlock(&vdpasim->kick_lock);
> mutex_unlock(&vdpasim->mutex);
> + kthread_flush_work(&vdpasim->work);
>
> return 0;
> }
> @@ -537,7 +549,7 @@ static int vdpasim_resume(struct vdpa_device *vdpa)
> if (vdpasim->pending_kick) {
> /* Process pending descriptors */
> for (i = 0; i < vdpasim->dev_attr.nvqs; ++i)
> - vdpasim_kick_vq(vdpa, i);
> + vdpasim_do_kick_vq(vdpa, i);
>
> vdpasim->pending_kick = false;
> }
> diff --git a/drivers/vdpa/vdpa_sim/vdpa_sim.h b/drivers/vdpa/vdpa_sim/vdpa_sim.h
> index bb137e479763..5eb6ca9c5ec5 100644
> --- a/drivers/vdpa/vdpa_sim/vdpa_sim.h
> +++ b/drivers/vdpa/vdpa_sim/vdpa_sim.h
> @@ -75,6 +75,7 @@ struct vdpasim {
> bool pending_kick;
> /* spinlock to synchronize iommu table */
> spinlock_t iommu_lock;
> + spinlock_t kick_lock;

It looks to me this is not initialized?

Thanks

> };
>
> struct vdpasim *vdpasim_create(struct vdpasim_dev_attr *attr,
> --
> 2.39.3
>

2024-05-21 13:40:08

by Steven Sistare

Subject: Re: [PATCH V3 3/3] vdpa_sim: flush workers on suspend

On 5/20/2024 10:32 PM, Jason Wang wrote:
> On Mon, May 20, 2024 at 11:21 PM Steve Sistare
> <[email protected]> wrote:
>>
>> Flush to guarantee no workers are running when suspend returns.
>> Add a lock to enforce ordering between clearing running, flushing,
>> and posting new work in vdpasim_kick_vq. It must be a spin lock
>> because vdpasim_kick_vq may be reached va eventfd_write.
>>
>> Signed-off-by: Steve Sistare <[email protected]>
>> ---
>> drivers/vdpa/vdpa_sim/vdpa_sim.c | 16 ++++++++++++++--
>> drivers/vdpa/vdpa_sim/vdpa_sim.h | 1 +
>> 2 files changed, 15 insertions(+), 2 deletions(-)
>>
>> diff --git a/drivers/vdpa/vdpa_sim/vdpa_sim.c b/drivers/vdpa/vdpa_sim/vdpa_sim.c
>> index 8ffea8430f95..67ed49d95bf0 100644
>> --- a/drivers/vdpa/vdpa_sim/vdpa_sim.c
>> +++ b/drivers/vdpa/vdpa_sim/vdpa_sim.c
>> @@ -322,7 +322,7 @@ static u16 vdpasim_get_vq_size(struct vdpa_device *vdpa, u16 idx)
>> return VDPASIM_QUEUE_MAX;
>> }
>>
>> -static void vdpasim_kick_vq(struct vdpa_device *vdpa, u16 idx)
>> +static void vdpasim_do_kick_vq(struct vdpa_device *vdpa, u16 idx)
>> {
>> struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
>> struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];
>> @@ -337,6 +337,15 @@ static void vdpasim_kick_vq(struct vdpa_device *vdpa, u16 idx)
>> vdpasim_schedule_work(vdpasim);
>> }
>>
>> +static void vdpasim_kick_vq(struct vdpa_device *vdpa, u16 idx)
>> +{
>> + struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
>> +
>> + spin_lock(&vdpasim->kick_lock);
>> + vdpasim_do_kick_vq(vdpa, idx);
>> + spin_unlock(&vdpasim->kick_lock);
>> +}
>> +
>> static void vdpasim_set_vq_cb(struct vdpa_device *vdpa, u16 idx,
>> struct vdpa_callback *cb)
>> {
>> @@ -520,8 +529,11 @@ static int vdpasim_suspend(struct vdpa_device *vdpa)
>> struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
>>
>> mutex_lock(&vdpasim->mutex);
>> + spin_lock(&vdpasim->kick_lock);
>> vdpasim->running = false;
>> + spin_unlock(&vdpasim->kick_lock);
>> mutex_unlock(&vdpasim->mutex);
>> + kthread_flush_work(&vdpasim->work);
>>
>> return 0;
>> }
>> @@ -537,7 +549,7 @@ static int vdpasim_resume(struct vdpa_device *vdpa)
>> if (vdpasim->pending_kick) {
>> /* Process pending descriptors */
>> for (i = 0; i < vdpasim->dev_attr.nvqs; ++i)
>> - vdpasim_kick_vq(vdpa, i);
>> + vdpasim_do_kick_vq(vdpa, i);
>>
>> vdpasim->pending_kick = false;
>> }
>> diff --git a/drivers/vdpa/vdpa_sim/vdpa_sim.h b/drivers/vdpa/vdpa_sim/vdpa_sim.h
>> index bb137e479763..5eb6ca9c5ec5 100644
>> --- a/drivers/vdpa/vdpa_sim/vdpa_sim.h
>> +++ b/drivers/vdpa/vdpa_sim/vdpa_sim.h
>> @@ -75,6 +75,7 @@ struct vdpasim {
>> bool pending_kick;
>> /* spinlock to synchronize iommu table */
>> spinlock_t iommu_lock;
>> + spinlock_t kick_lock;
>
> It looks to me this is not initialized?

Yup, I lost that line while fiddling with different locking schemes.
Thanks, will fix in V4.

@@ -236,6 +236,7 @@ struct vdpasim *vdpasim_create(struct vdpasim_dev_attr
*dev_attr,

mutex_init(&vdpasim->mutex);
spin_lock_init(&vdpasim->iommu_lock);
+ spin_lock_init(&vdpasim->kick_lock);

With that fix, does this patch earn your RB?

- Steve

>> };
>>
>> struct vdpasim *vdpasim_create(struct vdpasim_dev_attr *attr,
>> --
>> 2.39.3
>>
>

2024-05-22 03:40:09

by Jason Wang

Subject: Re: [PATCH V3 3/3] vdpa_sim: flush workers on suspend

On Tue, May 21, 2024 at 9:39 PM Steven Sistare
<[email protected]> wrote:
>
> On 5/20/2024 10:32 PM, Jason Wang wrote:
> > On Mon, May 20, 2024 at 11:21 PM Steve Sistare
> > <[email protected]> wrote:
> >>
> >> Flush to guarantee no workers are running when suspend returns.
> >> Add a lock to enforce ordering between clearing running, flushing,
> >> and posting new work in vdpasim_kick_vq. It must be a spin lock
> >> because vdpasim_kick_vq may be reached va eventfd_write.
> >>
> >> Signed-off-by: Steve Sistare <[email protected]>
> >> ---
> >> drivers/vdpa/vdpa_sim/vdpa_sim.c | 16 ++++++++++++++--
> >> drivers/vdpa/vdpa_sim/vdpa_sim.h | 1 +
> >> 2 files changed, 15 insertions(+), 2 deletions(-)
> >>
> >> diff --git a/drivers/vdpa/vdpa_sim/vdpa_sim.c b/drivers/vdpa/vdpa_sim/vdpa_sim.c
> >> index 8ffea8430f95..67ed49d95bf0 100644
> >> --- a/drivers/vdpa/vdpa_sim/vdpa_sim.c
> >> +++ b/drivers/vdpa/vdpa_sim/vdpa_sim.c
> >> @@ -322,7 +322,7 @@ static u16 vdpasim_get_vq_size(struct vdpa_device *vdpa, u16 idx)
> >> return VDPASIM_QUEUE_MAX;
> >> }
> >>
> >> -static void vdpasim_kick_vq(struct vdpa_device *vdpa, u16 idx)
> >> +static void vdpasim_do_kick_vq(struct vdpa_device *vdpa, u16 idx)
> >> {
> >> struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
> >> struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];
> >> @@ -337,6 +337,15 @@ static void vdpasim_kick_vq(struct vdpa_device *vdpa, u16 idx)
> >> vdpasim_schedule_work(vdpasim);
> >> }
> >>
> >> +static void vdpasim_kick_vq(struct vdpa_device *vdpa, u16 idx)
> >> +{
> >> + struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
> >> +
> >> + spin_lock(&vdpasim->kick_lock);
> >> + vdpasim_do_kick_vq(vdpa, idx);
> >> + spin_unlock(&vdpasim->kick_lock);
> >> +}
> >> +
> >> static void vdpasim_set_vq_cb(struct vdpa_device *vdpa, u16 idx,
> >> struct vdpa_callback *cb)
> >> {
> >> @@ -520,8 +529,11 @@ static int vdpasim_suspend(struct vdpa_device *vdpa)
> >> struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
> >>
> >> mutex_lock(&vdpasim->mutex);
> >> + spin_lock(&vdpasim->kick_lock);
> >> vdpasim->running = false;
> >> + spin_unlock(&vdpasim->kick_lock);
> >> mutex_unlock(&vdpasim->mutex);
> >> + kthread_flush_work(&vdpasim->work);
> >>
> >> return 0;
> >> }
> >> @@ -537,7 +549,7 @@ static int vdpasim_resume(struct vdpa_device *vdpa)
> >> if (vdpasim->pending_kick) {
> >> /* Process pending descriptors */
> >> for (i = 0; i < vdpasim->dev_attr.nvqs; ++i)
> >> - vdpasim_kick_vq(vdpa, i);
> >> + vdpasim_do_kick_vq(vdpa, i);
> >>
> >> vdpasim->pending_kick = false;
> >> }
> >> diff --git a/drivers/vdpa/vdpa_sim/vdpa_sim.h b/drivers/vdpa/vdpa_sim/vdpa_sim.h
> >> index bb137e479763..5eb6ca9c5ec5 100644
> >> --- a/drivers/vdpa/vdpa_sim/vdpa_sim.h
> >> +++ b/drivers/vdpa/vdpa_sim/vdpa_sim.h
> >> @@ -75,6 +75,7 @@ struct vdpasim {
> >> bool pending_kick;
> >> /* spinlock to synchronize iommu table */
> >> spinlock_t iommu_lock;
> >> + spinlock_t kick_lock;
> >
> > It looks to me this is not initialized?
>
> Yup, I lost that line while fiddling with different locking schemes.
> Thanks, will fix in V4.
>
> @@ -236,6 +236,7 @@ struct vdpasim *vdpasim_create(struct vdpasim_dev_attr
> *dev_attr,
>
> mutex_init(&vdpasim->mutex);
> spin_lock_init(&vdpasim->iommu_lock);
> + spin_lock_init(&vdpasim->kick_lock);
>
> With that fix, does this patch earn your RB?

Yes.

Thanks

>
> - Steve
>
> >> };
> >>
> >> struct vdpasim *vdpasim_create(struct vdpasim_dev_attr *attr,
> >> --
> >> 2.39.3
> >>
> >
>