36 static struct hpio *__iotab_search_get(struct hp_iotab *iotab, u32 eid)  in __iotab_search_get()
38 	struct hpio *hpio = NULL;  in __iotab_search_get()  local
40 	list_for_each_entry(hpio, &iotab->io_list, list)  in __iotab_search_get()
41 		if (hpio->eid == eid && kref_get_unless_zero(&hpio->refcnt))  in __iotab_search_get()
42 			return hpio;  in __iotab_search_get()
47 static struct hpio *iotab_search_get(struct hp_iotab *iotab, u32 eid)  in iotab_search_get()
49 	struct hpio *hpio = NULL;  in iotab_search_get()  local
53 	hpio = __iotab_search_get(iotab, eid);  in iotab_search_get()
56 	pr_info("find hpio %p for eid %u.\n", hpio, eid);  in iotab_search_get()
58 	return hpio;  in iotab_search_get()
65 static struct hpio *iotab_insert(struct hp_iotab *iotab, struct hpio *hpio)  in iotab_insert()  argument
67 	struct hpio *dup = NULL;  in iotab_insert()
71 	dup = __iotab_search_get(iotab, hpio->eid);  in iotab_insert()
74 				dup, hpio->eid, hpio);  in iotab_insert()
77 	list_add(&hpio->list, &iotab->io_list);  in iotab_insert()
79 	pr_info("insert new hpio %p for eid %u.\n", hpio, hpio->eid);  in iotab_insert()
86 static void iotab_delete(struct hp_iotab *iotab, struct hpio *hpio)  in iotab_delete()  argument
91 	list_del(&hpio->list);  in iotab_delete()
97 	pr_info("delete hpio %p for eid %u from iotab.\n", hpio, hpio->eid);  in iotab_delete()
100 static void hpio_clear_pages(struct hpio *hpio)  in hpio_clear_pages()  argument
104 	if (!hpio->pages)  in hpio_clear_pages()
107 	for (i = 0; i < hpio->nr_page; i++)  in hpio_clear_pages()
108 		if (hpio->pages[i]) {  in hpio_clear_pages()
109 			put_page(hpio->pages[i]);  in hpio_clear_pages()
112 	kfree(hpio->pages);  in hpio_clear_pages()
113 	atomic64_sub(sizeof(struct page *) * hpio->nr_page, &hpio_mem);  in hpio_clear_pages()
114 	hpio->nr_page = 0;  in hpio_clear_pages()
115 	hpio->pages = NULL;  in hpio_clear_pages()
121 static bool hpio_fill_pages(struct hpio *hpio, u32 nr_page, gfp_t gfp, bool new_page)  in hpio_fill_pages()  argument
125 	BUG_ON(hpio->pages);  in hpio_fill_pages()
126 	hpio->nr_page = nr_page;  in hpio_fill_pages()
127 	hpio->pages = kcalloc(hpio->nr_page, sizeof(struct page *), gfp);  in hpio_fill_pages()
128 	if (!hpio->pages)  in hpio_fill_pages()
130 	atomic64_add(sizeof(struct page *) * hpio->nr_page, &hpio_mem);  in hpio_fill_pages()
134 	for (i = 0; i < hpio->nr_page; i++) {  in hpio_fill_pages()
135 		hpio->pages[i] = alloc_page(gfp);  in hpio_fill_pages()
136 		if (!hpio->pages[i])  in hpio_fill_pages()
143 	hpio_clear_pages(hpio);  in hpio_fill_pages()
148 void hpio_free(struct hpio *hpio)  in hpio_free()  argument
150 	if (!hpio)  in hpio_free()
153 	pr_info("free hpio = %p.\n", hpio);  in hpio_free()
155 	hpio_clear_pages(hpio);  in hpio_free()
156 	kfree(hpio);  in hpio_free()
157 	atomic64_sub(sizeof(struct hpio), &hpio_mem);  in hpio_free()
160 struct hpio *hpio_alloc(u32 nr_page, gfp_t gfp, unsigned int op, bool new_page)  in hpio_alloc()
162 	struct hpio *hpio = NULL;  in hpio_alloc()  local
164 	hpio = kzalloc(sizeof(struct hpio), gfp);  in hpio_alloc()
165 	if (!hpio)  in hpio_alloc()
167 	atomic64_add(sizeof(struct hpio), &hpio_mem);  in hpio_alloc()
168 	if (!hpio_fill_pages(hpio, nr_page, gfp, new_page))  in hpio_alloc()
170 	hpio->op = op;  in hpio_alloc()
171 	atomic_set(&hpio->state, HPIO_INIT);  in hpio_alloc()
172 	kref_init(&hpio->refcnt);  in hpio_alloc()
173 	init_completion(&hpio->wait);  in hpio_alloc()
175 	return hpio;  in hpio_alloc()
177 	hpio_free(hpio);  in hpio_alloc()
182 struct hpio *hpio_get(u32 eid)  in hpio_get()
187 struct hpio *hpio_get_alloc(u32 eid, u32 nr_page, gfp_t gfp, unsigned int op)  in hpio_get_alloc()
189 	struct hpio *hpio = NULL;  in hpio_get_alloc()  local
190 	struct hpio *dup = NULL;  in hpio_get_alloc()
192 	hpio = iotab_search_get(&iotab, eid);  in hpio_get_alloc()
193 	if (hpio) {  in hpio_get_alloc()
194 		pr_info("find exist hpio %p for eid %u.\n", hpio, eid);  in hpio_get_alloc()
197 	hpio = hpio_alloc(nr_page, gfp, op, true);  in hpio_get_alloc()
198 	if (!hpio)  in hpio_get_alloc()
200 	hpio->eid = eid;  in hpio_get_alloc()
202 	pr_info("alloc hpio %p for eid %u.\n", hpio, eid);  in hpio_get_alloc()
204 	dup = iotab_insert(&iotab, hpio);  in hpio_get_alloc()
206 		hpio_free(hpio);  in hpio_get_alloc()
207 		hpio = dup;  in hpio_get_alloc()
210 	return hpio;  in hpio_get_alloc()
215 	struct hpio *hpio = container_of(kref, struct hpio, refcnt);  in hpio_release()  local
217 	iotab_delete(&iotab, hpio);  in hpio_release()
218 	if (hpio->free_extent)  in hpio_release()
219 		hpio->free_extent(hpio->eid);  in hpio_release()
220 	hpio_free(hpio);  in hpio_release()
223 bool hpio_put(struct hpio *hpio)  in hpio_put()  argument
225 	pr_info("put hpio %p for eid %u, ref = %u.\n", hpio, hpio->eid, kref_read(&hpio->refcnt));  in hpio_put()
226 	return kref_put(&hpio->refcnt, hpio_release);  in hpio_put()
229 void hpio_complete(struct hpio *hpio)  in hpio_complete()  argument
231 	pr_info("complete hpio %p for eid %u.\n", hpio, hpio->eid);  in hpio_complete()
232 	complete_all(&hpio->wait);  in hpio_complete()
235 void hpio_wait(struct hpio *hpio)  in hpio_wait()  argument
237 	wait_for_completion(&hpio->wait);  in hpio_wait()
240 enum hpio_state hpio_get_state(struct hpio *hpio)  in hpio_get_state()  argument
242 	return atomic_read(&hpio->state);  in hpio_get_state()
245 void hpio_set_state(struct hpio *hpio, enum hpio_state state)  in hpio_set_state()  argument
247 	atomic_set(&hpio->state, state);  in hpio_set_state()
250 bool hpio_change_state(struct hpio *hpio, enum hpio_state from, enum hpio_state to)  in hpio_change_state()  argument
252 	return atomic_cmpxchg(&hpio->state, from, to) == from;  in hpio_change_state()
257 	struct hpio *hpio = NULL;  in dump_iotab()  local
262 	list_for_each_entry(hpio, &iotab->io_list, list)  in dump_iotab()
263 		pr_info("hpio %p for eid %u is inflight.\n", hpio, hpio->eid);  in dump_iotab()