1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3 * Universal Flash Storage Host Performance Booster
4 *
5 * Copyright (C) 2017-2021 Samsung Electronics Co., Ltd.
6 *
7 * Authors:
8 * Yongmyung Lee <ymhungry.lee@samsung.com>
9 * Jinyoung Choi <j-young.choi@samsung.com>
10 */
11
12 #ifndef _UFSHPB_H_
13 #define _UFSHPB_H_
14
15 /* hpb response UPIU macro */
16 #define HPB_RSP_NONE 0x0
17 #define HPB_RSP_REQ_REGION_UPDATE 0x1
18 #define HPB_RSP_DEV_RESET 0x2
19 #define MAX_ACTIVE_NUM 2
20 #define MAX_INACTIVE_NUM 2
21 #define DEV_DATA_SEG_LEN 0x14
22 #define DEV_SENSE_SEG_LEN 0x12
23 #define DEV_DES_TYPE 0x80
24 #define DEV_ADDITIONAL_LEN 0x10
25
26 /* hpb map & entries macro */
27 #define HPB_RGN_SIZE_UNIT 512
28 #define HPB_ENTRY_BLOCK_SIZE 4096
29 #define HPB_ENTRY_SIZE 0x8
30 #define PINNED_NOT_SET U32_MAX
31
32 /* hpb support chunk size */
33 #define HPB_LEGACY_CHUNK_HIGH 1
34 #define HPB_MULTI_CHUNK_LOW 7
35 #define HPB_MULTI_CHUNK_HIGH 255
36
37 /* hpb vender defined opcode */
38 #define UFSHPB_READ 0xF8
39 #define UFSHPB_READ_BUFFER 0xF9
40 #define UFSHPB_READ_BUFFER_ID 0x01
41 #define UFSHPB_WRITE_BUFFER 0xFA
42 #define UFSHPB_WRITE_BUFFER_INACT_SINGLE_ID 0x01
43 #define UFSHPB_WRITE_BUFFER_PREFETCH_ID 0x02
44 #define UFSHPB_WRITE_BUFFER_INACT_ALL_ID 0x03
45 #define HPB_WRITE_BUFFER_CMD_LENGTH 10
46 #define MAX_HPB_READ_ID 0x7F
47 #define HPB_READ_BUFFER_CMD_LENGTH 10
48 #define LU_ENABLED_HPB_FUNC 0x02
49
50 #define HPB_RESET_REQ_RETRIES 10
51 #define HPB_MAP_REQ_RETRIES 5
52 #define HPB_REQUEUE_TIME_MS 0
53
54 #define HPB_SUPPORT_VERSION 0x200
55 #define HPB_SUPPORT_LEGACY_VERSION 0x100
56 #define HPB_MAJOR_VERSION_MASK 0xFF00
57
58 enum UFSHPB_MODE {
59 HPB_HOST_CONTROL,
60 HPB_DEVICE_CONTROL,
61 };
62
63 enum UFSHPB_STATE {
64 HPB_INIT = 0,
65 HPB_PRESENT = 1,
66 HPB_SUSPEND,
67 HPB_FAILED,
68 HPB_RESET,
69 };
70
71 enum HPB_RGN_STATE {
72 HPB_RGN_INACTIVE,
73 HPB_RGN_ACTIVE,
74 /* pinned regions are always active */
75 HPB_RGN_PINNED,
76 };
77
78 enum HPB_SRGN_STATE {
79 HPB_SRGN_UNUSED,
80 HPB_SRGN_INVALID,
81 HPB_SRGN_VALID,
82 HPB_SRGN_ISSUED,
83 };
84
85 /**
86 * struct ufshpb_lu_info - UFSHPB logical unit related info
87 * @num_blocks: the number of logical block
88 * @pinned_start: the start region number of pinned region
89 * @num_pinned: the number of pinned regions
90 * @max_active_rgns: maximum number of active regions
91 */
92 struct ufshpb_lu_info {
93 int num_blocks;
94 int pinned_start;
95 int num_pinned;
96 int max_active_rgns;
97 };
98
99 struct ufshpb_map_ctx {
100 struct page **m_page;
101 unsigned long *ppn_dirty;
102 };
103
104 struct ufshpb_subregion {
105 struct ufshpb_map_ctx *mctx;
106 enum HPB_SRGN_STATE srgn_state;
107 int rgn_idx;
108 int srgn_idx;
109 bool is_last;
110
111 /* subregion reads - for host mode */
112 unsigned int reads;
113
114 /* below information is used by rsp_list */
115 struct list_head list_act_srgn;
116 };
117
118 struct ufshpb_region {
119 struct ufshpb_lu *hpb;
120 struct ufshpb_subregion *srgn_tbl;
121 enum HPB_RGN_STATE rgn_state;
122 int rgn_idx;
123 int srgn_cnt;
124
125 /* below information is used by rsp_list */
126 struct list_head list_inact_rgn;
127
128 /* below information is used by lru */
129 struct list_head list_lru_rgn;
130 unsigned long rgn_flags;
131 #define RGN_FLAG_DIRTY 0
132 #define RGN_FLAG_UPDATE 1
133
134 /* region reads - for host mode */
135 spinlock_t rgn_lock;
136 unsigned int reads;
137 /* region "cold" timer - for host mode */
138 ktime_t read_timeout;
139 unsigned int read_timeout_expiries;
140 struct list_head list_expired_rgn;
141 };
142
143 #define for_each_sub_region(rgn, i, srgn) \
144 for ((i) = 0; \
145 ((i) < (rgn)->srgn_cnt) && ((srgn) = &(rgn)->srgn_tbl[i]); \
146 (i)++)
147
148 /**
149 * struct ufshpb_req - HPB related request structure (write/read buffer)
150 * @req: block layer request structure
151 * @bio: bio for this request
152 * @hpb: ufshpb_lu structure that related to
153 * @list_req: ufshpb_req mempool list
154 * @sense: store its sense data
155 * @mctx: L2P map information
156 * @rgn_idx: target region index
157 * @srgn_idx: target sub-region index
158 * @lun: target logical unit number
159 * @m_page: L2P map information data for pre-request
160 * @len: length of host-side cached L2P map in m_page
161 * @lpn: start LPN of L2P map in m_page
162 */
163 struct ufshpb_req {
164 struct request *req;
165 struct bio *bio;
166 struct ufshpb_lu *hpb;
167 struct list_head list_req;
168 union {
169 struct {
170 struct ufshpb_map_ctx *mctx;
171 unsigned int rgn_idx;
172 unsigned int srgn_idx;
173 unsigned int lun;
174 } rb;
175 struct {
176 struct page *m_page;
177 unsigned int len;
178 unsigned long lpn;
179 } wb;
180 };
181 };
182
183 struct victim_select_info {
184 struct list_head lh_lru_rgn; /* LRU list of regions */
185 int max_lru_active_cnt; /* supported hpb #region - pinned #region */
186 atomic_t active_cnt;
187 };
188
189 /**
190 * ufshpb_params - ufs hpb parameters
191 * @requeue_timeout_ms - requeue threshold of wb command (0x2)
192 * @activation_thld - min reads [IOs] to activate/update a region
193 * @normalization_factor - shift right the region's reads
194 * @eviction_thld_enter - min reads [IOs] for the entering region in eviction
195 * @eviction_thld_exit - max reads [IOs] for the exiting region in eviction
196 * @read_timeout_ms - timeout [ms] from the last read IO to the region
197 * @read_timeout_expiries - amount of allowable timeout expireis
198 * @timeout_polling_interval_ms - frequency in which timeouts are checked
199 * @inflight_map_req - number of inflight map requests
200 */
201 struct ufshpb_params {
202 unsigned int requeue_timeout_ms;
203 unsigned int activation_thld;
204 unsigned int normalization_factor;
205 unsigned int eviction_thld_enter;
206 unsigned int eviction_thld_exit;
207 unsigned int read_timeout_ms;
208 unsigned int read_timeout_expiries;
209 unsigned int timeout_polling_interval_ms;
210 unsigned int inflight_map_req;
211 };
212
213 struct ufshpb_stats {
214 u64 hit_cnt;
215 u64 miss_cnt;
216 u64 rb_noti_cnt;
217 u64 rb_active_cnt;
218 u64 rb_inactive_cnt;
219 u64 map_req_cnt;
220 u64 pre_req_cnt;
221 u64 umap_req_cnt;
222 };
223
224 struct ufshpb_lu {
225 int lun;
226 struct scsi_device *sdev_ufs_lu;
227
228 spinlock_t rgn_state_lock; /* for protect rgn/srgn state */
229 struct ufshpb_region *rgn_tbl;
230
231 atomic_t hpb_state;
232
233 spinlock_t rsp_list_lock;
234 struct list_head lh_act_srgn; /* hold rsp_list_lock */
235 struct list_head lh_inact_rgn; /* hold rsp_list_lock */
236
237 /* pre request information */
238 struct ufshpb_req *pre_req;
239 int num_inflight_pre_req;
240 int throttle_pre_req;
241 int num_inflight_map_req; /* hold param_lock */
242 spinlock_t param_lock;
243
244 struct list_head lh_pre_req_free;
245 int cur_read_id;
246 int pre_req_min_tr_len;
247 int pre_req_max_tr_len;
248
249 /* cached L2P map management worker */
250 struct work_struct map_work;
251
252 /* for selecting victim */
253 struct victim_select_info lru_info;
254 struct work_struct ufshpb_normalization_work;
255 struct delayed_work ufshpb_read_to_work;
256 unsigned long work_data_bits;
257 #define TIMEOUT_WORK_RUNNING 0
258
259 /* pinned region information */
260 u32 lu_pinned_start;
261 u32 lu_pinned_end;
262
263 /* HPB related configuration */
264 u32 rgns_per_lu;
265 u32 srgns_per_lu;
266 u32 last_srgn_entries;
267 int srgns_per_rgn;
268 u32 srgn_mem_size;
269 u32 entries_per_rgn_mask;
270 u32 entries_per_rgn_shift;
271 u32 entries_per_srgn;
272 u32 entries_per_srgn_mask;
273 u32 entries_per_srgn_shift;
274 u32 pages_per_srgn;
275
276 bool is_hcm;
277
278 struct ufshpb_stats stats;
279 struct ufshpb_params params;
280
281 struct kmem_cache *map_req_cache;
282 struct kmem_cache *m_page_cache;
283
284 struct list_head list_hpb_lu;
285 };
286
287 struct ufs_hba;
288 struct ufshcd_lrb;
289
290 #ifndef CONFIG_SCSI_UFS_HPB
ufshpb_prep(struct ufs_hba * hba,struct ufshcd_lrb * lrbp)291 static int ufshpb_prep(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) { return 0; }
ufshpb_rsp_upiu(struct ufs_hba * hba,struct ufshcd_lrb * lrbp)292 static void ufshpb_rsp_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) {}
ufshpb_resume(struct ufs_hba * hba)293 static void ufshpb_resume(struct ufs_hba *hba) {}
ufshpb_suspend(struct ufs_hba * hba)294 static void ufshpb_suspend(struct ufs_hba *hba) {}
ufshpb_reset(struct ufs_hba * hba)295 static void ufshpb_reset(struct ufs_hba *hba) {}
ufshpb_reset_host(struct ufs_hba * hba)296 static void ufshpb_reset_host(struct ufs_hba *hba) {}
ufshpb_init(struct ufs_hba * hba)297 static void ufshpb_init(struct ufs_hba *hba) {}
ufshpb_init_hpb_lu(struct ufs_hba * hba,struct scsi_device * sdev)298 static void ufshpb_init_hpb_lu(struct ufs_hba *hba, struct scsi_device *sdev) {}
ufshpb_destroy_lu(struct ufs_hba * hba,struct scsi_device * sdev)299 static void ufshpb_destroy_lu(struct ufs_hba *hba, struct scsi_device *sdev) {}
ufshpb_remove(struct ufs_hba * hba)300 static void ufshpb_remove(struct ufs_hba *hba) {}
ufshpb_is_allowed(struct ufs_hba * hba)301 static bool ufshpb_is_allowed(struct ufs_hba *hba) { return false; }
ufshpb_get_geo_info(struct ufs_hba * hba,u8 * geo_buf)302 static void ufshpb_get_geo_info(struct ufs_hba *hba, u8 *geo_buf) {}
ufshpb_get_dev_info(struct ufs_hba * hba,u8 * desc_buf)303 static void ufshpb_get_dev_info(struct ufs_hba *hba, u8 *desc_buf) {}
ufshpb_is_legacy(struct ufs_hba * hba)304 static bool ufshpb_is_legacy(struct ufs_hba *hba) { return false; }
305 #else
306 int ufshpb_prep(struct ufs_hba *hba, struct ufshcd_lrb *lrbp);
307 void ufshpb_rsp_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp);
308 void ufshpb_resume(struct ufs_hba *hba);
309 void ufshpb_suspend(struct ufs_hba *hba);
310 void ufshpb_reset(struct ufs_hba *hba);
311 void ufshpb_reset_host(struct ufs_hba *hba);
312 void ufshpb_init(struct ufs_hba *hba);
313 void ufshpb_init_hpb_lu(struct ufs_hba *hba, struct scsi_device *sdev);
314 void ufshpb_destroy_lu(struct ufs_hba *hba, struct scsi_device *sdev);
315 void ufshpb_remove(struct ufs_hba *hba);
316 bool ufshpb_is_allowed(struct ufs_hba *hba);
317 void ufshpb_get_geo_info(struct ufs_hba *hba, u8 *geo_buf);
318 void ufshpb_get_dev_info(struct ufs_hba *hba, u8 *desc_buf);
319 bool ufshpb_is_legacy(struct ufs_hba *hba);
320 extern struct attribute_group ufs_sysfs_hpb_stat_group;
321 extern struct attribute_group ufs_sysfs_hpb_param_group;
322 #endif
323
324 #endif /* End of Header */
325