Received-SPF: pass (google.com: domain of linux-kernel-owner@vger.kernel.org designates 23.128.96.18 as permitted sender) client-ip=23.128.96.18;
Sender: cang=codeaurora.org@mg.codeaurora.org
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8;
 format=flowed
Content-Transfer-Encoding: 8bit
Date:   Mon, 15 Mar 2021 17:36:28 +0800
From:   Can Guo <cang@codeaurora.org>
To:     Avri Altman <avri.altman@wdc.com>
Cc:     "James E . J . Bottomley" <jejb@linux.vnet.ibm.com>,
        "Martin K . Petersen" <martin.petersen@oracle.com>,
        linux-scsi@vger.kernel.org, linux-kernel@vger.kernel.org,
        gregkh@linuxfoundation.org, Bart Van Assche <bvanassche@acm.org>,
        yongmyung lee <ymhungry.lee@samsung.com>,
        Daejun Park <daejun7.park@samsung.com>,
        alim.akhtar@samsung.com, asutoshd@codeaurora.org,
        Zang Leigang <zangleigang@hisilicon.com>,
        Avi Shchislowski <avi.shchislowski@wdc.com>,
        Bean Huo <beanhuo@micron.com>, stanley.chu@mediatek.com
Subject: Re: [PATCH v5 07/10] scsi: ufshpb: Add "Cold" regions timer
In-Reply-To: <20210302132503.224670-8-avri.altman@wdc.com>
References: <20210302132503.224670-1-avri.altman@wdc.com>
 <20210302132503.224670-8-avri.altman@wdc.com>
Message-ID: <be5c0c390d7c0cf13e67f51cdc7dd8c2@codeaurora.org>
User-Agent: Roundcube Webmail/1.3.9
Precedence: bulk

On 2021-03-02 21:25, Avri Altman wrote:
> In order not to hang on to “cold” regions, we shall inactivate a
> region that has no READ access for a predefined amount of time -
> READ_TO_MS. For that purpose we shall monitor the active regions list,
> polling it on every POLLING_INTERVAL_MS. On timeout expiry we shall add
> the region to the "to-be-inactivated" list, unless it is clean and did
> not exhaust its READ_TO_EXPIRIES - another parameter.
> 
> All this does not apply to pinned regions.
> 
> Signed-off-by: Avri Altman <avri.altman@wdc.com>
> ---
>  drivers/scsi/ufs/ufshpb.c | 65 +++++++++++++++++++++++++++++++++++++++
>  drivers/scsi/ufs/ufshpb.h |  6 ++++
>  2 files changed, 71 insertions(+)
> 
> diff --git a/drivers/scsi/ufs/ufshpb.c b/drivers/scsi/ufs/ufshpb.c
> index 0034fa03fdc6..89a930e72cff 100644
> --- a/drivers/scsi/ufs/ufshpb.c
> +++ b/drivers/scsi/ufs/ufshpb.c
> @@ -18,6 +18,9 @@
> 
>  #define ACTIVATION_THRESHOLD 4 /* 4 IOs */
>  #define EVICTION_THRESHOLD (ACTIVATION_THRESHOLD << 6) /* 256 IOs */
> +#define READ_TO_MS 1000
> +#define READ_TO_EXPIRIES 100
> +#define POLLING_INTERVAL_MS 200
> 
>  /* memory management */
>  static struct kmem_cache *ufshpb_mctx_cache;
> @@ -1024,12 +1027,61 @@ static int
> ufshpb_check_srgns_issue_state(struct ufshpb_lu *hpb,
>  	return 0;
>  }
> 
> +static void ufshpb_read_to_handler(struct work_struct *work)
> +{
> +	struct delayed_work *dwork = to_delayed_work(work);
> +	struct ufshpb_lu *hpb;
> +	struct victim_select_info *lru_info;
> +	struct ufshpb_region *rgn;
> +	unsigned long flags;
> +	LIST_HEAD(expired_list);
> +
> +	hpb = container_of(dwork, struct ufshpb_lu, ufshpb_read_to_work);
> +
> +	spin_lock_irqsave(&hpb->rgn_state_lock, flags);
> +
> +	lru_info = &hpb->lru_info;
> +
> +	list_for_each_entry(rgn, &lru_info->lh_lru_rgn, list_lru_rgn) {
> +		bool timedout = ktime_after(ktime_get(), rgn->read_timeout);
> +
> +		if (timedout) {
> +			rgn->read_timeout_expiries--;
> +			if (is_rgn_dirty(rgn) ||
> +			    rgn->read_timeout_expiries == 0)
> +				list_add(&rgn->list_expired_rgn, &expired_list);
> +			else
> +				rgn->read_timeout = ktime_add_ms(ktime_get(),
> +							 READ_TO_MS);
> +		}
> +	}
> +
> +	spin_unlock_irqrestore(&hpb->rgn_state_lock, flags);
> +
> +	list_for_each_entry(rgn, &expired_list, list_expired_rgn) {

Here can be problematic - since you don't have the native expired_list 
initialized
before use, if above loop did not insert anything to expired_list, it 
shall become
a dead loop here.

And, which lock is protecting rgn->list_expired_rgn? If two 
read_to_handler works
are running in parallel, one can be inserting it to its expired_list 
while another
can be deleting it.

Can Guo.

> +		list_del_init(&rgn->list_expired_rgn);
> +		spin_lock_irqsave(&hpb->rsp_list_lock, flags);
> +		ufshpb_update_inactive_info(hpb, rgn->rgn_idx);
> +		hpb->stats.rb_inactive_cnt++;
> +		spin_unlock_irqrestore(&hpb->rsp_list_lock, flags);
> +	}
> +
> +	ufshpb_kick_map_work(hpb);
> +
> +	schedule_delayed_work(&hpb->ufshpb_read_to_work,
> +			      msecs_to_jiffies(POLLING_INTERVAL_MS));
> +}
> +
>  static void ufshpb_add_lru_info(struct victim_select_info *lru_info,
>  				struct ufshpb_region *rgn)
>  {
>  	rgn->rgn_state = HPB_RGN_ACTIVE;
>  	list_add_tail(&rgn->list_lru_rgn, &lru_info->lh_lru_rgn);
>  	atomic_inc(&lru_info->active_cnt);
> +	if (rgn->hpb->is_hcm) {
> +		rgn->read_timeout = ktime_add_ms(ktime_get(), READ_TO_MS);
> +		rgn->read_timeout_expiries = READ_TO_EXPIRIES;
> +	}
>  }
> 
>  static void ufshpb_hit_lru_info(struct victim_select_info *lru_info,
> @@ -1813,6 +1865,7 @@ static int ufshpb_alloc_region_tbl(struct
> ufs_hba *hba, struct ufshpb_lu *hpb)
> 
>  		INIT_LIST_HEAD(&rgn->list_inact_rgn);
>  		INIT_LIST_HEAD(&rgn->list_lru_rgn);
> +		INIT_LIST_HEAD(&rgn->list_expired_rgn);
> 
>  		if (rgn_idx == hpb->rgns_per_lu - 1) {
>  			srgn_cnt = ((hpb->srgns_per_lu - 1) %
> @@ -1834,6 +1887,7 @@ static int ufshpb_alloc_region_tbl(struct
> ufs_hba *hba, struct ufshpb_lu *hpb)
>  		}
> 
>  		rgn->rgn_flags = 0;
> +		rgn->hpb = hpb;
>  	}
> 
>  	return 0;
> @@ -2053,6 +2107,8 @@ static int ufshpb_lu_hpb_init(struct ufs_hba
> *hba, struct ufshpb_lu *hpb)
>  			  ufshpb_normalization_work_handler);
>  		INIT_WORK(&hpb->ufshpb_lun_reset_work,
>  			  ufshpb_reset_work_handler);
> +		INIT_DELAYED_WORK(&hpb->ufshpb_read_to_work,
> +				  ufshpb_read_to_handler);
>  	}
> 
>  	hpb->map_req_cache = kmem_cache_create("ufshpb_req_cache",
> @@ -2087,6 +2143,10 @@ static int ufshpb_lu_hpb_init(struct ufs_hba
> *hba, struct ufshpb_lu *hpb)
>  	ufshpb_stat_init(hpb);
>  	ufshpb_param_init(hpb);
> 
> +	if (hpb->is_hcm)
> +		schedule_delayed_work(&hpb->ufshpb_read_to_work,
> +				      msecs_to_jiffies(POLLING_INTERVAL_MS));
> +
>  	return 0;
> 
>  release_pre_req_mempool:
> @@ -2154,6 +2214,7 @@ static void ufshpb_discard_rsp_lists(struct
> ufshpb_lu *hpb)
>  static void ufshpb_cancel_jobs(struct ufshpb_lu *hpb)
>  {
>  	if (hpb->is_hcm) {
> +		cancel_delayed_work_sync(&hpb->ufshpb_read_to_work);
>  		cancel_work_sync(&hpb->ufshpb_lun_reset_work);
>  		cancel_work_sync(&hpb->ufshpb_normalization_work);
>  	}
> @@ -2264,6 +2325,10 @@ void ufshpb_resume(struct ufs_hba *hba)
>  			continue;
>  		ufshpb_set_state(hpb, HPB_PRESENT);
>  		ufshpb_kick_map_work(hpb);
> +		if (hpb->is_hcm)
> +			schedule_delayed_work(&hpb->ufshpb_read_to_work,
> +				msecs_to_jiffies(POLLING_INTERVAL_MS));
> +
>  	}
>  }
> 
> diff --git a/drivers/scsi/ufs/ufshpb.h b/drivers/scsi/ufs/ufshpb.h
> index 37c1b0ea0c0a..b49e9a34267f 100644
> --- a/drivers/scsi/ufs/ufshpb.h
> +++ b/drivers/scsi/ufs/ufshpb.h
> @@ -109,6 +109,7 @@ struct ufshpb_subregion {
>  };
> 
>  struct ufshpb_region {
> +	struct ufshpb_lu *hpb;
>  	struct ufshpb_subregion *srgn_tbl;
>  	enum HPB_RGN_STATE rgn_state;
>  	int rgn_idx;
> @@ -126,6 +127,10 @@ struct ufshpb_region {
>  	/* region reads - for host mode */
>  	spinlock_t rgn_lock;
>  	unsigned int reads;
> +	/* region "cold" timer - for host mode */
> +	ktime_t read_timeout;
> +	unsigned int read_timeout_expiries;
> +	struct list_head list_expired_rgn;
>  };
> 
>  #define for_each_sub_region(rgn, i, srgn)				\
> @@ -219,6 +224,7 @@ struct ufshpb_lu {
>  	struct victim_select_info lru_info;
>  	struct work_struct ufshpb_normalization_work;
>  	struct work_struct ufshpb_lun_reset_work;
> +	struct delayed_work ufshpb_read_to_work;
> 
>  	/* pinned region information */
>  	u32 lu_pinned_start;