Subject: Re: [Suspected-Phishing]Re: [PATCH V3 1/2] nvme: split resetting
 state into reset_prepate and resetting
From: Max Gurtovoy <maxg@mellanox.com>
To: Sagi Grimberg <sagi@grimberg.me>,
        Jianchao Wang <jianchao.w.wang@oracle.com>,
        <keith.busch@intel.com>, <axboe@fb.com>, <hch@lst.de>
CC: <linux-kernel@vger.kernel.org>, <linux-nvme@lists.infradead.org>
References: <1515647268-1717-1-git-send-email-jianchao.w.wang@oracle.com>
 <1515647268-1717-2-git-send-email-jianchao.w.wang@oracle.com>
 <b502ea5a-d319-3a03-0204-e2b6068bb55e@grimberg.me>
 <de132a4d-eb8b-10e8-7c96-7836edf1b52a@mellanox.com>
Message-ID: <1c001532-234f-bc56-7fb4-bcd08142842e@mellanox.com>
Date: Mon, 15 Jan 2018 15:36:48 +0200
User-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64; rv:52.0) Gecko/20100101
 Thunderbird/52.5.2
MIME-Version: 1.0
In-Reply-To: <de132a4d-eb8b-10e8-7c96-7836edf1b52a@mellanox.com>
Content-Type: text/plain; charset="windows-1255"; format=flowed
Content-Language: he
Content-Transfer-Encoding: 8bit
SpamDiagnosticOutput: 1:99
SpamDiagnosticMetadata: NSPM
X-MS-Exchange-CrossTenant-OriginalArrivalTime: 15 Jan 2018 13:36:51.0865
 (UTC)
X-MS-Exchange-CrossTenant-Network-Message-Id: ca129664-f741-40a9-2620-08d55c1d08d0
X-MS-Exchange-CrossTenant-Id: a652971c-7d2e-4d9b-a6a4-d149256f461b
X-MS-Exchange-CrossTenant-OriginalAttributedTenantConnectingIp: TenantId=a652971c-7d2e-4d9b-a6a4-d149256f461b;Ip=[193.47.165.134];Helo=[mtlcas13.mtl.com]
X-MS-Exchange-CrossTenant-FromEntityHeader: HybridOnPrem
X-MS-Exchange-Transport-CrossTenantHeadersStamped: DB6PR0501MB2264
Sender: linux-kernel-owner@vger.kernel.org
List-ID: <linux-kernel.vger.kernel.org>
X-Mailing-List: linux-kernel@vger.kernel.org
Return-Path: <linux-kernel-owner@vger.kernel.org>


On 1/15/2018 3:28 PM, Max Gurtovoy wrote:
> 
> 
> On 1/14/2018 11:48 AM, Sagi Grimberg wrote:
>>
>>> Currently, the ctrl->state will be changed to NVME_CTRL_RESETTING
>>> before queue the reset work. This is not so strict. There could be
>>> a big gap before the reset_work callback is invoked. In addition,
>>> there is some disable work in the reset_work callback, strictly
>>> speaking, not part of reset work, and could lead to some confusion.
>>>
>>> This patch splits the NVME_CTRL_RESETTING into NVME_CTRL_RESET_PREPARE
>>> and NVME_CTRL_RESETTING. Before queue the reset work, changes state
>>> to NVME_CTRL_RESET_PREPARE, after disable work completes, changes
>>> state to NVME_CTRL_RESETTING.
>>
>> What guarantees that nvme_timeout will do the right thing? If it
>> is relying on the fact that nvme-pci sets the state to RESETTING
>> only _after_ quiescing the requests queues it needs to be documented
>> as its a real delicate dependency.
>>
>>
>>
>>>
>>> Suggested-by: Christoph Hellwig <hch@lst.de>
>>> Signed-off-by: Jianchao Wang <jianchao.w.wang@oracle.com>
>>> ---
>>> ? drivers/nvme/host/core.c?? | 17 +++++++++++++++--
>>> ? drivers/nvme/host/fc.c???? |? 2 ++
>>> ? drivers/nvme/host/nvme.h?? |? 1 +
>>> ? drivers/nvme/host/pci.c??? | 28 ++++++++++++++++++++++------
>>> ? drivers/nvme/host/rdma.c?? |? 8 ++++++++
>>> ? drivers/nvme/target/loop.c |? 5 +++++
>>> ? 6 files changed, 53 insertions(+), 8 deletions(-)
>>>
>>> diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c
>>> index 1e46e60..106a437 100644
>>> --- a/drivers/nvme/host/core.c
>>> +++ b/drivers/nvme/host/core.c
>>> @@ -87,7 +87,7 @@ static __le32 nvme_get_log_dw10(u8 lid, size_t size)
>>> ? int nvme_reset_ctrl(struct nvme_ctrl *ctrl)
>>> ? {
>>> -??? if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING))
>>> +??? if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_RESET_PREPARE))
>>> ????????? return -EBUSY;
>>> ????? if (!queue_work(nvme_wq, &ctrl->reset_work))
>>> ????????? return -EBUSY;
>>> @@ -243,7 +243,7 @@ bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl,
>>> ????????????? break;
>>> ????????? }
>>> ????????? break;
>>> -??? case NVME_CTRL_RESETTING:
>>> +??? case NVME_CTRL_RESET_PREPARE:
>>> ????????? switch (old_state) {
>>> ????????? case NVME_CTRL_NEW:
>>> ????????? case NVME_CTRL_LIVE:
>>> @@ -253,10 +253,21 @@ bool nvme_change_ctrl_state(struct nvme_ctrl 
>>> *ctrl,
>>> ????????????? break;
>>> ????????? }
>>> ????????? break;
>>> +
>>> +??? case NVME_CTRL_RESETTING:
>>> +??????? switch (old_state) {
>>> +??????? case NVME_CTRL_RESET_PREPARE:
>>> +??????????? changed = true;
>>> +??????????? /* FALLTHRU */
>>> +??????? default:
>>> +??????????? break;
>>> +??????? }
>>> +??????? break;
>>> ????? case NVME_CTRL_RECONNECTING:
>>> ????????? switch (old_state) {
>>> ????????? case NVME_CTRL_LIVE:
>>> ????????? case NVME_CTRL_RESETTING:
>>> +??????? case NVME_CTRL_RESET_PREPARE:

I forget to add that we shouldn't move from RESET_PREPARE to 
RECONNECTING (with my suggestion to rdma.c).
Also need to consider adding another check in nvmf_check_init_req 
(/drivers/nvme/host/fabrics.h) for the new state.


>>> ????????????? changed = true;
>>> ????????????? /* FALLTHRU */
>>> ????????? default:
>>> @@ -267,6 +278,7 @@ bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl,
>>> ????????? switch (old_state) {
>>> ????????? case NVME_CTRL_LIVE:
>>> ????????? case NVME_CTRL_RESETTING:
>>> +??????? case NVME_CTRL_RESET_PREPARE:
>>> ????????? case NVME_CTRL_RECONNECTING:
>>> ????????????? changed = true;
>>> ????????????? /* FALLTHRU */
>>> @@ -2603,6 +2615,7 @@ static ssize_t nvme_sysfs_show_state(struct 
>>> device *dev,
>>> ????????? [NVME_CTRL_NEW]??????? = "new",
>>> ????????? [NVME_CTRL_LIVE]??? = "live",
>>> ????????? [NVME_CTRL_RESETTING]??? = "resetting",
>>> +??????? [NVME_CTRL_RESET_PREPARE]??? = "reset-prepare",
>>> ????????? [NVME_CTRL_RECONNECTING]= "reconnecting",
>>> ????????? [NVME_CTRL_DELETING]??? = "deleting",
>>> ????????? [NVME_CTRL_DEAD]??? = "dead",
>>> diff --git a/drivers/nvme/host/fc.c b/drivers/nvme/host/fc.c
>>> index 794e66e..516c1ea 100644
>>> --- a/drivers/nvme/host/fc.c
>>> +++ b/drivers/nvme/host/fc.c
>>> @@ -547,6 +547,7 @@ nvme_fc_resume_controller(struct nvme_fc_ctrl *ctrl)
>>> ????????? break;
>>> ????? case NVME_CTRL_RESETTING:
>>> +??? case NVME_CTRL_RESET_PREPARE:
>>> ????????? /*
>>> ?????????? * Controller is already in the process of terminating the
>>> ?????????? * association. No need to do anything further. The reconnect
>>> @@ -790,6 +791,7 @@ nvme_fc_ctrl_connectivity_loss(struct 
>>> nvme_fc_ctrl *ctrl)
>>> ????????? break;
>>> ????? case NVME_CTRL_RESETTING:
>>> +??? case NVME_CTRL_RESET_PREPARE:
>>> ????????? /*
>>> ?????????? * Controller is already in the process of terminating the
>>> ?????????? * association.? No need to do anything further. The reconnect
>>
>> James, can you take a look at this?
>>
>>> diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c
>>> index f5800c3..e477c35 100644
>>> --- a/drivers/nvme/host/pci.c
>>> +++ b/drivers/nvme/host/pci.c
>>> @@ -1141,8 +1141,13 @@ static bool nvme_should_reset(struct nvme_dev 
>>> *dev, u32 csts)
>>> ????? bool nssro = dev->subsystem && (csts & NVME_CSTS_NSSRO);
>>> ????? /* If there is a reset ongoing, we shouldn't reset again. */
>>> -??? if (dev->ctrl.state == NVME_CTRL_RESETTING)
>>> +??? switch (dev->ctrl.state) {
>>> +??? case NVME_CTRL_RESETTING:
>>> +??? case NVME_CTRL_RESET_PREPARE:
>>> ????????? return false;
>>> +??? default:
>>> +??????? break;
>>> +??? }
>>
>> Isn't the indentation off? also in other places...
>>
>>> @@ -2292,7 +2303,7 @@ static void nvme_reset_work(struct work_struct 
>>> *work)
>>> ????? bool was_suspend = !!(dev->ctrl.ctrl_config & NVME_CC_SHN_NORMAL);
>>> ????? int result = -ENODEV;
>>> -??? if (WARN_ON(dev->ctrl.state != NVME_CTRL_RESETTING))
>>> +??? if (WARN_ON(dev->ctrl.state != NVME_CTRL_RESET_PREPARE))
>>> ????????? goto out;
>>> ????? /*
>>> @@ -2302,6 +2313,11 @@ static void nvme_reset_work(struct work_struct 
>>> *work)
>>> ????? if (dev->ctrl.ctrl_config & NVME_CC_ENABLE)
>>> ????????? nvme_dev_disable(dev, false);
>>> +??? if (!nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_RESETTING)) {
>>> +??????? WARN_ON_ONCE(dev->ctrl.state != NVME_CTRL_DELETING);
>>> +??????? goto out;
>>> +??? }
>>> +
>>
>> This needs documentation on _why_ here.
>>
>>> -??? nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_RESETTING);
>>> +??? nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_RESET_PREPARE);
>>> ????? dev_info(dev->ctrl.device, "pci function %s\n", 
>>> dev_name(&pdev->dev));
>>
>> Can you rebase on top of my commit:
>> 4caff8fc19f1 nvme-pci: don't open-code nvme_reset_ctrl
>>
>>> diff --git a/drivers/nvme/host/rdma.c b/drivers/nvme/host/rdma.c
>>> index 37af565..8ae073e 100644
>>> --- a/drivers/nvme/host/rdma.c
>>> +++ b/drivers/nvme/host/rdma.c
>>> @@ -1753,6 +1753,14 @@ static void nvme_rdma_reset_ctrl_work(struct 
>>> work_struct *work)
>>> ????? nvme_stop_ctrl(&ctrl->ctrl);
>>> ????? nvme_rdma_shutdown_ctrl(ctrl, false);
>>> +??? changed = nvme_change_ctrl_state(&ctrl->ctrl,
>>> +??????????? NVME_CTRL_RESET_PREPARE);
>>> +??? if (!changed) {
>>> +??????? /* state change failure is ok if we're in DELETING state */
>>> +??????? WARN_ON_ONCE(ctrl->ctrl.state != NVME_CTRL_DELETING);
>>> +??????? return;
>>> +??? }
>>> +
>>
>> setting RESET_PREPARE here??
>>
>> Also, the error recovery code is mutually excluded from reset_work
>> by trying to set the same state which is protected by the ctrl state
>> machine, so a similar change is needed there.
> 
> Sagi,
> Do you mean to add this (if so, I agree):
> 
> 
> diff --git a/drivers/nvme/host/rdma.c b/drivers/nvme/host/rdma.c
> index d06641b..44ef52a 100644
> --- a/drivers/nvme/host/rdma.c
> +++ b/drivers/nvme/host/rdma.c
> @@ -957,6 +957,12 @@ static void nvme_rdma_error_recovery_work(struct 
> work_struct *work)
>  ??? struct nvme_rdma_ctrl *ctrl = container_of(work,
>  ??????????? struct nvme_rdma_ctrl, err_work);
> 
> +?? if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_RESETING)) {
> +?????? /* state change failure should never happen */
> +?????? WARN_ON_ONCE(1);
> +?????? return;
> +?? }
> +
>  ??? nvme_stop_keep_alive(&ctrl->ctrl);
> 
>  ??? if (ctrl->ctrl.queue_count > 1) {
> @@ -989,7 +995,7 @@ static void nvme_rdma_error_recovery_work(struct 
> work_struct *work)
> 
>  ?static void nvme_rdma_error_recovery(struct nvme_rdma_ctrl *ctrl)
>  ?{
> -?? if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_RESETTING))
> +?? if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_RESET_PREPARE))
>  ??????? return;
> 
>  ??? queue_work(nvme_wq, &ctrl->err_work);
> @@ -1760,6 +1766,12 @@ static void nvme_rdma_reset_ctrl_work(struct 
> work_struct *work)
>  ??? int ret;
>  ??? bool changed;
> 
> +?? if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_RESETING)) {
> +?????? /* state change failure should never happen */
> +?????? WARN_ON_ONCE(1);
> +?????? return;
> +?? }
> +
>  ??? nvme_stop_ctrl(&ctrl->ctrl);
>  ??? nvme_rdma_shutdown_ctrl(ctrl, false);
> 
> 
> 
> 
> 
> 
>>
>> So in order not to break rdma ctrl reset (again) please look at:
>> bcdacf7fc3d8 nvme-rdma: fix concurrent reset and reconnect
>>
>> _______________________________________________
>> Linux-nvme mailing list
>> Linux-nvme@lists.infradead.org
>> http://lists.infradead.org/mailman/listinfo/linux-nvme
> 
> _______________________________________________
> Linux-nvme mailing list
> Linux-nvme@lists.infradead.org
> http://lists.infradead.org/mailman/listinfo/linux-nvme