1 /*
2 * Copyright (c) 2023 Institute of Parallel And Distributed Systems (IPADS), Shanghai Jiao Tong University (SJTU)
3 * Licensed under the Mulan PSL v2.
4 * You can use this software according to the terms and conditions of the Mulan PSL v2.
5 * You may obtain a copy of Mulan PSL v2 at:
6 * http://license.coscl.org.cn/MulanPSL2
7 * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
8 * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
9 * PURPOSE.
10 * See the Mulan PSL v2 for more details.
11 */
12 #include <malloc.h>
13 #include <pthread.h>
14 #include <stdlib.h>
15 #include <chcore/defs.h>
16 #include <chcore/memory.h>
17 #include <chcore/syscall.h>
18 #include <chcore-internal/fs_debug.h>
19
20 #include "fs_page_cache.h"
21 #include "fs_wrapper_defs.h"
22
23 static struct fs_page_cache page_cache = {0};
24
25 /* +++++++++++++++++++++++++++ Helper functions +++++++++++++++++++++++++++++ */
26
27 /*
28 * if page_block_idx == -1, check a page's dirty flag,
29 * else check a block's dirty flag.
30 */
is_block_or_page_dirty(struct cached_page * p,int page_block_idx)31 static bool is_block_or_page_dirty(struct cached_page *p, int page_block_idx)
32 {
33 int i;
34
35 if (page_block_idx != -1)
36 return p->dirty[page_block_idx];
37 else
38 for (i = 0; i < BLOCK_PER_PAGE; ++i)
39 if (p->dirty[i])
40 return true;
41 return false;
42 }
43
44 /*
45 * if page_block_idx == -1, set a page's dirty flag as @is_dirty,
46 * else set a block's dirty flag as @is_dirtyy.
47 */
set_block_or_page_dirty(struct cached_page * p,int page_block_idx,bool is_dirty)48 static void set_block_or_page_dirty(struct cached_page *p, int page_block_idx,
49 bool is_dirty)
50 {
51 int i;
52
53 if (page_block_idx != -1)
54 p->dirty[page_block_idx] = is_dirty;
55 else
56 for (i = 0; i < BLOCK_PER_PAGE; ++i)
57 p->dirty[i] = is_dirty;
58 }
59
60 /* ++++++++++++++++++++++++++ Page list operations ++++++++++++++++++++++++++ */
cached_pages_list_init(struct cached_pages_list * list)61 static inline void cached_pages_list_init(struct cached_pages_list *list)
62 {
63 init_list_head(&list->queue);
64 list->size = 0;
65 }
66
67 /* Return the head of the list. */
68 static struct cached_page *
cached_pages_list_top_node(struct cached_pages_list * list,PAGE_CACHE_LIST_TYPE list_type)69 cached_pages_list_top_node(struct cached_pages_list *list,
70 PAGE_CACHE_LIST_TYPE list_type)
71 {
72 if (list_empty(&list->queue))
73 return NULL;
74 if (list_type == ACTIVE_LIST || list_type == INACTIVE_LIST)
75 return list_entry(list->queue.next, struct cached_page, two_list_node);
76 else if (list_type == INODE_PAGES_LIST)
77 return list_entry(
78 list->queue.next, struct cached_page, inode_pages_node);
79 else if (list_type == PINNED_PAGES_LIST)
80 return list_entry(
81 list->queue.next, struct cached_page, pinned_pages_node);
82
83 BUG("Unknown list type.");
84 return NULL;
85 }
86
87 /* Find corresponding page from the page_cache_entity_of_inode. */
88 static inline struct cached_page *
get_node_from_page_cache_entity(struct page_cache_entity_of_inode * pce,pidx_t file_page_idx)89 get_node_from_page_cache_entity(struct page_cache_entity_of_inode *pce,
90 pidx_t file_page_idx)
91 {
92 return radix_get(&pce->idx2page, file_page_idx);
93 }
94
95 /*
96 * Append node to the tail of the list.
97 */
cached_pages_list_append_node(struct cached_pages_list * list,struct cached_page * p,PAGE_CACHE_LIST_TYPE list_type)98 static void cached_pages_list_append_node(struct cached_pages_list *list,
99 struct cached_page *p,
100 PAGE_CACHE_LIST_TYPE list_type)
101 {
102 switch (list_type) {
103 case ACTIVE_LIST:
104 case INACTIVE_LIST:
105 list_append(&p->two_list_node, &list->queue);
106 break;
107 case INODE_PAGES_LIST:
108 list_append(&p->inode_pages_node, &list->queue);
109 break;
110 case PINNED_PAGES_LIST:
111 list_append(&p->pinned_pages_node, &list->queue);
112 break;
113 default:
114 BUG("Unknown list type.");
115 break;
116 }
117
118 p->in_which_list = list_type;
119 list->size++;
120 }
121
122 /*
123 * Remove head node, don't forget to free the data outside this function.
124 */
cached_pages_list_delete_node(struct cached_pages_list * list,struct cached_page * p,PAGE_CACHE_LIST_TYPE list_type)125 static void cached_pages_list_delete_node(struct cached_pages_list *list,
126 struct cached_page *p,
127 PAGE_CACHE_LIST_TYPE list_type)
128 {
129 switch (list_type) {
130 case ACTIVE_LIST:
131 case INACTIVE_LIST:
132 list_del(&p->two_list_node);
133 p->in_which_list = INODE_PAGES_LIST;
134 break;
135 case INODE_PAGES_LIST:
136 list_del(&p->inode_pages_node);
137 p->in_which_list = UNKNOWN_LIST;
138 break;
139 case PINNED_PAGES_LIST:
140 list_del(&p->pinned_pages_node);
141 p->in_which_list = INODE_PAGES_LIST;
142 break;
143 default:
144 BUG("Unknown list type.");
145 break;
146 }
147
148 list->size--;
149 BUG_ON(list->size < 0);
150 }
151
152 /* +++++++++++++++++++++++++ cached_page operations +++++++++++++++++++++++++ */
153 /* The caller should guarantee page_block_idx is in legal range. */
flush_single_block(struct cached_page * p,int page_block_idx)154 static int flush_single_block(struct cached_page *p, int page_block_idx)
155 {
156 int ret = 0;
157
158 BUG_ON(page_block_idx < 0);
159
160 if (is_block_or_page_dirty(p, page_block_idx)) {
161 page_cache_debug(
162 "[write_back_routine] %d:%d:%d in active list is dirty, write it back.\n",
163 p->owner->host_idx,
164 p->file_page_idx,
165 page_block_idx);
166 if (page_cache.user_func.file_write(
167 p->content + page_block_idx * CACHED_BLOCK_SIZE,
168 p->file_page_idx,
169 page_block_idx,
170 p->owner->private_data)
171 <= 0) {
172 BUG("[write_back_all_pages] file_write failed\n");
173 ret = -1;
174 goto out;
175 }
176 set_block_or_page_dirty(p, page_block_idx, false);
177 }
178
179 out:
180 return ret;
181 }
182
flush_single_page(struct cached_page * p)183 static int flush_single_page(struct cached_page *p)
184 {
185 int i, ret = 0;
186
187 BUG_ON(p == NULL);
188
189 for (i = 0; i < BLOCK_PER_PAGE; ++i) {
190 if (flush_single_block(p, i) != 0) {
191 ret = -1;
192 goto out;
193 }
194 }
195 out:
196 return ret;
197 }
198
free_page(struct cached_page * p)199 static void free_page(struct cached_page *p)
200 {
201 BUG_ON(p == NULL);
202
203 free(p->content);
204
205 /* Delete from ACTIVE_LIST, INACTIVE_LIST or PINNED_PAGES_LIST. */
206 switch (p->in_which_list) {
207 case ACTIVE_LIST:
208 cached_pages_list_delete_node(&page_cache.active_list, p, ACTIVE_LIST);
209 break;
210 case INACTIVE_LIST:
211 cached_pages_list_delete_node(
212 &page_cache.inactive_list, p, INACTIVE_LIST);
213 break;
214 case PINNED_PAGES_LIST:
215 cached_pages_list_delete_node(
216 &page_cache.pinned_pages_list, p, PINNED_PAGES_LIST);
217 break;
218 default:
219 BUG("Try to free a page that is not in ACTIVE_LIST, INACTIVE_LIST or PINNED_PAGES_LIST.\n");
220 }
221
222 /* Delete from INODE_PAGES_LIST. */
223 cached_pages_list_delete_node(&p->owner->pages, p, INODE_PAGES_LIST);
224 radix_del(&p->owner->idx2page, p->file_page_idx, 0);
225
226 /* Once pages_cnt in a pce reached 0, trigger handler */
227 if (--(p->owner->pages_cnt) == 0
228 && page_cache.user_func.handler_pce_turns_empty) {
229 page_cache.user_func.handler_pce_turns_empty(p->owner->private_data);
230 }
231
232 pthread_rwlock_destroy(&p->page_rwlock);
233
234 free(p);
235 }
236
flush_and_free_page(struct cached_page * p)237 static void flush_and_free_page(struct cached_page *p)
238 {
239 BUG_ON(p == NULL);
240
241 /* Can't free a page that's being used. */
242 pthread_rwlock_wrlock(&p->page_rwlock);
243
244 /*
245 * pages with empty contents is marked not dirty.
246 * So it won't be flushed.
247 */
248 if (is_block_or_page_dirty(p, -1))
249 flush_single_page(p);
250
251 free_page(p);
252 }
253
new_page(struct page_cache_entity_of_inode * owner,pidx_t file_page_idx)254 static struct cached_page *new_page(struct page_cache_entity_of_inode *owner,
255 pidx_t file_page_idx)
256 {
257 struct cached_page *p;
258
259 p = (struct cached_page *)calloc(1, sizeof(struct cached_page));
260 if (!p)
261 return NULL;
262
263 p->owner = owner;
264 p->file_page_idx = file_page_idx;
265
266 init_list_head(&p->two_list_node);
267 init_list_head(&p->inode_pages_node);
268 init_list_head(&p->pinned_pages_node);
269
270 pthread_rwlock_init(&p->page_rwlock, NULL);
271
272 p->content = aligned_alloc(CACHED_PAGE_SIZE, CACHED_PAGE_SIZE);
273 BUG_ON(!p->content);
274 memset(p->content, 0, CACHED_PAGE_SIZE);
275
276 /* Append to INODE_PAGES_LIST. */
277 cached_pages_list_append_node(&owner->pages, p, INODE_PAGES_LIST);
278 radix_add(&owner->idx2page, file_page_idx, p);
279
280 /* Contribute 1 ref to pce for each cached_page */
281 if (owner->pages_cnt++ == 0
282 && page_cache.user_func.handler_pce_turns_nonempty) {
283 page_cache.user_func.handler_pce_turns_nonempty(owner->private_data);
284 }
285
286 return p;
287 }
288
evict_head_page(struct cached_pages_list * list,PAGE_CACHE_LIST_TYPE list_type)289 static inline void evict_head_page(struct cached_pages_list *list,
290 PAGE_CACHE_LIST_TYPE list_type)
291 {
292 struct cached_page *p;
293
294 /* Evict head page. */
295 if (list_type == ACTIVE_LIST || list_type == INACTIVE_LIST) {
296 p = cached_pages_list_top_node(list, list_type);
297 page_cache_debug("[evict_head_page] Evict %d:%d in list %d.\n",
298 p->owner->host_idx,
299 p->file_page_idx,
300 list_type);
301 flush_and_free_page(p);
302 } else {
303 BUG("Try to evict a page that is not in two list.\n");
304 }
305 }
306
307 /*
308 * Boost a node in active list. Remove the node and append it at tail.
309 */
boost_active_page(struct cached_page * p)310 static inline void boost_active_page(struct cached_page *p)
311 {
312 /* Delete node from active list. */
313 cached_pages_list_delete_node(&page_cache.active_list, p, ACTIVE_LIST);
314
315 /* Append it to the tail. */
316 cached_pages_list_append_node(&page_cache.active_list, p, ACTIVE_LIST);
317 }
318
319 /*
320 * Boost a node in inactive list.
321 * Remove the node and append it to the tail of the active list.
322 */
boost_inactive_page(struct cached_page * p)323 static inline void boost_inactive_page(struct cached_page *p)
324 {
325 struct cached_page *head_page;
326 /* Delete node from inactive list */
327 cached_pages_list_delete_node(&page_cache.inactive_list, p, INACTIVE_LIST);
328
329 /* Add node to active list */
330 cached_pages_list_append_node(&page_cache.active_list, p, ACTIVE_LIST);
331
332 if (page_cache.active_list.size > ACTIVE_LIST_MAX) {
333 /*
334 * If active list is full,
335 * move head unpinned page to inactive list.
336 */
337 head_page =
338 cached_pages_list_top_node(&page_cache.active_list, ACTIVE_LIST);
339 /**
340 * It should be impossible for head_page == NULL but klocwork
341 * reported a bug here, maybe due to precision of static
342 * analysis
343 */
344 if (head_page == NULL) {
345 WARN("head_page == NULL");
346 return;
347 }
348 cached_pages_list_delete_node(
349 &page_cache.active_list, head_page, ACTIVE_LIST);
350 cached_pages_list_append_node(
351 &page_cache.inactive_list, head_page, INACTIVE_LIST);
352
353 if (page_cache.inactive_list.size > INACTIVE_LIST_MAX) {
354 /*
355 * if inactive list is full,
356 * evict head unpinned page.
357 */
358 evict_head_page(&page_cache.inactive_list, INACTIVE_LIST);
359 }
360 }
361 }
362
363 /* Caller should hold per page_cache mutex lock to protect list operations */
boost_page(struct cached_page * p)364 static inline void boost_page(struct cached_page *p)
365 {
366 if (p->in_which_list == ACTIVE_LIST)
367 boost_active_page(p);
368 else if (p->in_which_list == INACTIVE_LIST)
369 boost_inactive_page(p);
370 else
371 BUG("Try to boost page that is not in two lists!\n");
372 }
373
374 /*
375 * Find a page from active_list, inactive_list and pinned_pages_list, if page
376 * not found, create a new one and insert it to inactive list and pages list.
377 * Notice: The caller should hold per page_cache mutex lock!
378 * return value:
379 * @which_list: ACTIVE_LIST or INACTIVE_LIST or PINNED_PAGES_LIST.
380 * @is_new: if page is newly allocated, is_new = true, else is_new = false.
381 */
find_or_new_page(struct page_cache_entity_of_inode * pce,pidx_t file_page_idx,bool * is_new)382 struct cached_page *find_or_new_page(struct page_cache_entity_of_inode *pce,
383 pidx_t file_page_idx, bool *is_new)
384 {
385 struct cached_page *p;
386 bool temp_is_new;
387
388 p = get_node_from_page_cache_entity(pce, file_page_idx);
389 if (p == NULL) {
390 #ifdef TEST_COUNT_PAGE_CACHE
391 count.miss = count.miss + 1;
392 #endif
393 /*
394 * Cache miss, create a new page
395 * and fill it with corresponding contents.
396 */
397 page_cache_debug(
398 "[find_or_new_page] page cache %d:%d not found， alloc a new one\n",
399 pce->host_idx,
400 file_page_idx);
401 p = new_page(pce, file_page_idx);
402 if (p == NULL) {
403 BUG("[find_or_new_page] new_page failed\n");
404 goto fail;
405 }
406
407 /*
408 * According to two list strategy, a new page should
409 * be inserted into inactive list.
410 */
411 cached_pages_list_append_node(
412 &page_cache.inactive_list, p, INACTIVE_LIST);
413 if (page_cache.inactive_list.size > INACTIVE_LIST_MAX) {
414 /*
415 * if inactive list is full,
416 * evict head unpinned page.
417 */
418 evict_head_page(&page_cache.inactive_list, INACTIVE_LIST);
419 }
420
421 /* Fill it with corresponding contents. */
422 page_cache.user_func.file_read(
423 p->content, file_page_idx, pce->private_data);
424
425 temp_is_new = true;
426 } else {
427 /* Cache hit. */
428 #ifdef TEST_COUNT_PAGE_CACHE
429 count.hit = count.hit + 1;
430 #endif
431 page_cache_debug("[find_or_new_page] page cache %d:%d found in %d\n",
432 pce->host_idx,
433 file_page_idx,
434 p->in_which_list);
435 temp_is_new = false;
436 }
437
438 if (is_new)
439 *is_new = temp_is_new;
440 return p;
441
442 fail:
443 if (p)
444 flush_and_free_page(p);
445 return NULL;
446 }
447
448 /* The caller should hold per page_cache mutex lock! */
write_back_all_pages(void)449 void write_back_all_pages(void)
450 {
451 struct cached_page *p;
452
453 /* Write back pages in active list. */
454 for_each_in_list (
455 p, struct cached_page, two_list_node, &page_cache.active_list.queue) {
456 pthread_rwlock_rdlock(&p->page_rwlock);
457 flush_single_page(p);
458 pthread_rwlock_unlock(&p->page_rwlock);
459 }
460 /* Write back pages in inactive list. */
461 for_each_in_list (
462 p, struct cached_page, two_list_node, &page_cache.inactive_list.queue) {
463 pthread_rwlock_rdlock(&p->page_rwlock);
464 flush_single_page(p);
465 pthread_rwlock_unlock(&p->page_rwlock);
466 }
467 /* Write back pages in pinned pages list. */
468 for_each_in_list (p,
469 struct cached_page,
470 pinned_pages_node,
471 &page_cache.pinned_pages_list.queue) {
472 pthread_rwlock_rdlock(&p->page_rwlock);
473 flush_single_page(p);
474 pthread_rwlock_unlock(&p->page_rwlock);
475 }
476 }
477
478 /* Write back all dirty pages periodically. */
write_back_routine(void * args)479 void *write_back_routine(void *args)
480 {
481 struct timespec ts;
482
483 while (1) {
484 pthread_mutex_lock(&page_cache.page_cache_lock);
485 if (page_cache.cache_strategy == WRITE_BACK) {
486 page_cache_debug(
487 "[write_back_routine] write back routine started.\n");
488 write_back_all_pages();
489 page_cache_debug(
490 "[write_back_routine] write back routine completed.\n");
491 }
492 pthread_mutex_unlock(&page_cache.page_cache_lock);
493
494 ts.tv_sec = WRITE_BACK_CYCLE;
495 ts.tv_nsec = 0;
496 nanosleep(&ts, NULL);
497 }
498
499 return NULL;
500 }
501
502 /* +++++++++++++++++++++++++ Exposed Functions+++++++++++++++++++++++++++ */
503
fs_page_cache_init(PAGE_CACHE_STRATEGY strategy,struct user_defined_funcs * uf)504 void fs_page_cache_init(PAGE_CACHE_STRATEGY strategy,
505 struct user_defined_funcs *uf)
506 {
507 pthread_t thread;
508
509 memcpy(&page_cache.user_func, uf, sizeof(*uf));
510
511 cached_pages_list_init(&page_cache.active_list);
512 cached_pages_list_init(&page_cache.inactive_list);
513 cached_pages_list_init(&page_cache.pinned_pages_list);
514
515 page_cache.cache_strategy = strategy;
516 pthread_create(&thread, 0, write_back_routine, NULL);
517
518 pthread_mutex_init(&page_cache.page_cache_lock, NULL);
519
520 page_cache_debug("fs page cache init finished.\n");
521 }
522
523 struct page_cache_entity_of_inode *
new_page_cache_entity_of_inode(ino_t host_idx,void * private_data)524 new_page_cache_entity_of_inode(ino_t host_idx, void *private_data)
525 {
526 struct page_cache_entity_of_inode *pce;
527
528 pce = (struct page_cache_entity_of_inode *)calloc(
529 1, sizeof(struct page_cache_entity_of_inode));
530 if (pce == NULL)
531 goto out;
532
533 pce->host_idx = host_idx;
534 pce->private_data = private_data;
535
536 cached_pages_list_init(&pce->pages);
537 init_radix(&pce->idx2page);
538
539 out:
540 return pce;
541 }
542
page_cache_switch_strategy(PAGE_CACHE_STRATEGY new_strategy)543 int page_cache_switch_strategy(PAGE_CACHE_STRATEGY new_strategy)
544 {
545 if (page_cache.cache_strategy == new_strategy)
546 return 0;
547
548 pthread_mutex_lock(&page_cache.page_cache_lock);
549
550 if (page_cache.cache_strategy == WRITE_BACK) {
551 /* Write back all the pages. */
552 write_back_all_pages();
553 }
554
555 page_cache_debug(
556 "[page_cache_switch_strategy] switch cache strategy from %d to %d\n",
557 page_cache.cache_strategy,
558 new_strategy);
559 page_cache.cache_strategy = new_strategy;
560
561 pthread_mutex_unlock(&page_cache.page_cache_lock);
562
563 return 0;
564 }
565
page_cache_check_page(struct page_cache_entity_of_inode * pce,pidx_t file_page_idx)566 int page_cache_check_page(struct page_cache_entity_of_inode *pce,
567 pidx_t file_page_idx)
568 {
569 struct cached_page *p;
570
571 pthread_mutex_lock(&page_cache.page_cache_lock);
572
573 /* Find corresponding page from pce list. */
574 p = get_node_from_page_cache_entity(pce, file_page_idx);
575
576 pthread_mutex_unlock(&page_cache.page_cache_lock);
577
578 if (p == NULL)
579 return 0;
580 else
581 return 1;
582 }
583
584 /* Only boost_page in page_cache_get_block_or_page. */
page_cache_put_block_or_page(struct page_cache_entity_of_inode * pce,pidx_t file_page_idx,int page_block_idx,PAGE_CACHE_OPERATION_TYPE op_type)585 void page_cache_put_block_or_page(struct page_cache_entity_of_inode *pce,
586 pidx_t file_page_idx, int page_block_idx,
587 PAGE_CACHE_OPERATION_TYPE op_type)
588 {
589 struct cached_page *p;
590 bool is_new;
591
592 BUG_ON(page_block_idx < -1 || page_block_idx >= BLOCK_PER_PAGE);
593 BUG_ON(op_type != READ && op_type != WRITE);
594
595 pthread_mutex_lock(&page_cache.page_cache_lock);
596
597 p = find_or_new_page(pce, file_page_idx, &is_new);
598 if (p == NULL)
599 goto out;
600
601 /* Target page must be existed. */
602 BUG_ON(is_new == true);
603
604 /* Nothing need to be done when handling read operations. */
605 if (op_type == READ) {
606 pthread_rwlock_unlock(&p->page_rwlock);
607 goto out;
608 }
609
610 /* Handling write operation. */
611 switch (page_cache.cache_strategy) {
612 case DIRECT:
613 /* Directly write block or page to disk. */
614 if (page_block_idx != -1) {
615 set_block_or_page_dirty(p, page_block_idx, true);
616 if (flush_single_block(p, page_block_idx) != 0)
617 BUG("[pc_set_block_or_page] file_write failed.\n");
618 } else {
619 set_block_or_page_dirty(p, -1, true);
620 if (flush_single_page(p) != 0)
621 BUG("[pc_set_block_or_page] file_write failed.\n");
622 }
623
624 /* Must unlock before freeing. */
625 pthread_rwlock_unlock(&p->page_rwlock);
626
627 /*
628 * Invalidate and free outdated cache.
629 * There won't be any use-after-free error because
630 * page_cache_lock protect us from this error.
631 */
632 flush_and_free_page(p);
633 break;
634
635 case WRITE_BACK:
636 /* Mark this block or page as dirty. */
637 set_block_or_page_dirty(p, page_block_idx, true);
638
639 pthread_rwlock_unlock(&p->page_rwlock);
640 break;
641
642 case WRITE_THROUGH:
643 /* Directly write block or page to disk. */
644 if (page_block_idx != -1) {
645 set_block_or_page_dirty(p, page_block_idx, true);
646 if (flush_single_block(p, page_block_idx) != 0)
647 BUG("[pc_set_block_or_page] file_write failed.\n");
648 } else {
649 set_block_or_page_dirty(p, -1, true);
650 if (flush_single_page(p) != 0)
651 BUG("[pc_set_block_or_page] file_write failed.\n");
652 }
653
654 pthread_rwlock_unlock(&p->page_rwlock);
655 break;
656 }
657
658 out:
659 pthread_mutex_unlock(&page_cache.page_cache_lock);
660 }
661
662 /* Only boost_page when calling page_cache_get_block_or_page. */
page_cache_get_block_or_page(struct page_cache_entity_of_inode * pce,pidx_t file_page_idx,int page_block_idx,PAGE_CACHE_OPERATION_TYPE op_type)663 char *page_cache_get_block_or_page(struct page_cache_entity_of_inode *pce,
664 pidx_t file_page_idx, int page_block_idx,
665 PAGE_CACHE_OPERATION_TYPE op_type)
666 {
667 struct cached_page *p;
668 bool is_new;
669 char *buf;
670
671 BUG_ON(page_block_idx < -1 || page_block_idx >= BLOCK_PER_PAGE);
672 BUG_ON(op_type != READ && op_type != WRITE);
673
674 pthread_mutex_lock(&page_cache.page_cache_lock);
675
676 p = find_or_new_page(pce, file_page_idx, &is_new);
677 if (p == NULL) {
678 buf = NULL;
679 goto out;
680 }
681
682 /* Grab read or write lock of this page. */
683 if (op_type == READ)
684 pthread_rwlock_rdlock(&p->page_rwlock);
685 else
686 pthread_rwlock_wrlock(&p->page_rwlock);
687
688 /* Read from corresponding cached page. */
689 if (page_block_idx != -1)
690 buf = p->content + page_block_idx * CACHED_BLOCK_SIZE;
691 else
692 buf = p->content;
693
694 /* New allocated pages should not be boosted. */
695 if (is_new == false)
696 boost_page(p);
697
698 out:
699 pthread_mutex_unlock(&page_cache.page_cache_lock);
700 return buf;
701 }
702
page_cache_flush_block_or_page(struct page_cache_entity_of_inode * pce,pidx_t file_page_idx,int page_block_idx)703 int page_cache_flush_block_or_page(struct page_cache_entity_of_inode *pce,
704 pidx_t file_page_idx, int page_block_idx)
705 {
706 struct cached_page *p;
707 int ret = 0;
708
709 BUG_ON(page_block_idx < -1 || page_block_idx >= BLOCK_PER_PAGE);
710
711 pthread_mutex_lock(&page_cache.page_cache_lock);
712
713 p = get_node_from_page_cache_entity(pce, file_page_idx);
714 if (p == NULL) {
715 page_cache_debug("Corresponding page cache does not exist!\n");
716 ret = -1;
717 goto out;
718 }
719
720 pthread_rwlock_rdlock(&p->page_rwlock);
721 if (page_block_idx != -1)
722 flush_single_block(p, page_block_idx);
723 else
724 flush_single_page(p);
725
726 out:
727 if (p)
728 pthread_rwlock_unlock(&p->page_rwlock);
729 pthread_mutex_unlock(&page_cache.page_cache_lock);
730 return ret;
731 }
732
page_cache_flush_pages_of_inode(struct page_cache_entity_of_inode * pce)733 int page_cache_flush_pages_of_inode(struct page_cache_entity_of_inode *pce)
734 {
735 struct cached_page *p;
736 int ret = 0;
737
738 pthread_mutex_lock(&page_cache.page_cache_lock);
739
740 page_cache_debug(
741 "[page_cache_flush_pages_of_inode] write back inode pages started.\n");
742
743 /* Write back pages that belongs to an inode. */
744 for_each_in_list (
745 p, struct cached_page, inode_pages_node, &pce->pages.queue) {
746 pthread_rwlock_rdlock(&p->page_rwlock);
747 if (flush_single_page(p) != 0) {
748 ret = -1;
749 goto fail_unlock;
750 }
751 pthread_rwlock_unlock(&p->page_rwlock);
752 }
753
754 page_cache_debug(
755 "[page_cache_flush_pages_of_inode] write back inode pages finished.\n");
756 goto out;
757
758 fail_unlock:
759 pthread_rwlock_unlock(&p->page_rwlock);
760 out:
761 pthread_mutex_unlock(&page_cache.page_cache_lock);
762 return ret;
763 }
764
page_cache_flush_all_pages(void)765 int page_cache_flush_all_pages(void)
766 {
767 pthread_mutex_lock(&page_cache.page_cache_lock);
768
769 page_cache_debug("[page_cache_flush_all_pages] flush all pages started.\n");
770
771 write_back_all_pages();
772
773 page_cache_debug(
774 "[page_cache_flush_all_pages] flush all pages finished.\n");
775
776 pthread_mutex_unlock(&page_cache.page_cache_lock);
777 return 0;
778 }
779
page_cache_pin_single_page(struct page_cache_entity_of_inode * pce,pidx_t file_page_idx)780 int page_cache_pin_single_page(struct page_cache_entity_of_inode *pce,
781 pidx_t file_page_idx)
782 {
783 struct cached_page *p;
784 int ret = 0;
785
786 pthread_mutex_lock(&page_cache.page_cache_lock);
787
788 /* Check max pinned pages. */
789 if (page_cache.pinned_pages_list.size == MAX_PINNED_PAGE) {
790 page_cache_debug("pinned pages number reach limits.\n");
791 ret = -1;
792 goto out;
793 }
794
795 /* Find target page, if not found, create one. */
796 p = find_or_new_page(pce, file_page_idx, NULL);
797 if (p == NULL) {
798 ret = -1;
799 goto out;
800 }
801
802 /* Remove this page from two lists. */
803 if (p->in_which_list == ACTIVE_LIST) {
804 cached_pages_list_delete_node(&page_cache.active_list, p, ACTIVE_LIST);
805 } else if (p->in_which_list == INACTIVE_LIST) {
806 cached_pages_list_delete_node(
807 &page_cache.inactive_list, p, INACTIVE_LIST);
808 } else if (p->in_which_list == PINNED_PAGES_LIST) {
809 /* Page already pinned. */
810 ret = 0;
811 goto out;
812 } else {
813 BUG("Invalid list type.\n");
814 ret = -1;
815 goto out;
816 }
817
818 /* Insert this page to pinned pages list. */
819 cached_pages_list_append_node(
820 &page_cache.pinned_pages_list, p, PINNED_PAGES_LIST);
821
822 out:
823 pthread_mutex_unlock(&page_cache.page_cache_lock);
824 return ret;
825 }
826
page_cache_unpin_single_page(struct page_cache_entity_of_inode * pce,pidx_t file_page_idx)827 int page_cache_unpin_single_page(struct page_cache_entity_of_inode *pce,
828 pidx_t file_page_idx)
829 {
830 struct cached_page *p;
831 int ret = 0;
832
833 pthread_mutex_lock(&page_cache.page_cache_lock);
834
835 /* Ensuring target page is already existed. */
836 p = get_node_from_page_cache_entity(pce, file_page_idx);
837 if (p == NULL) {
838 page_cache_debug("page not found.\n");
839 ret = -1;
840 goto out;
841 }
842
843 /* Ensuring target page is already pinned. */
844 if (p->in_which_list != PINNED_PAGES_LIST) {
845 page_cache_debug("page not pinned.\n");
846 ret = -1;
847 goto out;
848 }
849
850 /*
851 * Because we are unlikely to access this page again after we unpin it,
852 * we just flush and free this page.
853 */
854 flush_and_free_page(p);
855
856 out:
857 pthread_mutex_unlock(&page_cache.page_cache_lock);
858 return ret;
859 }
860
page_cache_pin_multiple_pages(struct page_cache_entity_of_inode * pce,pidx_t file_page_idx,u64 page_num)861 int page_cache_pin_multiple_pages(struct page_cache_entity_of_inode *pce,
862 pidx_t file_page_idx, u64 page_num)
863 {
864 u64 i;
865
866 for (i = file_page_idx; i < file_page_idx + page_num; ++i)
867 if (page_cache_pin_single_page(pce, i) != 0)
868 return -1;
869
870 return 0;
871 }
872
page_cache_unpin_multiple_pages(struct page_cache_entity_of_inode * pce,pidx_t file_page_idx,u64 page_num)873 int page_cache_unpin_multiple_pages(struct page_cache_entity_of_inode *pce,
874 pidx_t file_page_idx, u64 page_num)
875 {
876 u64 i;
877
878 for (i = file_page_idx; i < file_page_idx + page_num; ++i)
879 if (page_cache_unpin_single_page(pce, i) != 0)
880 return -1;
881
882 return 0;
883 }
884
page_cache_evict_single_page(struct page_cache_entity_of_inode * pce,pidx_t file_page_idx)885 int page_cache_evict_single_page(struct page_cache_entity_of_inode *pce,
886 pidx_t file_page_idx)
887 {
888 struct cached_page *p;
889 int ret = 0;
890
891 pthread_mutex_lock(&page_cache.page_cache_lock);
892
893 /* Ensuring target page is already existing. */
894 p = get_node_from_page_cache_entity(pce, file_page_idx);
895 if (p == NULL) {
896 page_cache_debug(
897 "[page_cache_evict_single_page] page does not exist.\n");
898 ret = -1;
899 goto out;
900 }
901
902 page_cache_debug("Evict %d:%d in list %d\n",
903 p->owner->host_idx,
904 p->file_page_idx,
905 p->in_which_list);
906 flush_and_free_page(p);
907
908 out:
909 pthread_mutex_unlock(&page_cache.page_cache_lock);
910 return ret;
911 }
912
page_cache_evict_pages_of_inode(struct page_cache_entity_of_inode * pce)913 int page_cache_evict_pages_of_inode(struct page_cache_entity_of_inode *pce)
914 {
915 struct cached_page **p_array;
916 struct cached_page *p;
917 int queue_size, i = 0;
918
919 pthread_mutex_lock(&page_cache.page_cache_lock);
920
921 queue_size = pce->pages.size;
922 p_array = (struct cached_page **)calloc(pce->pages.size,
923 sizeof(struct cached_page *));
924 BUG_ON(p_array == NULL);
925
926 /* Find all target pages. */
927 for_each_in_list (
928 p, struct cached_page, inode_pages_node, &pce->pages.queue)
929 p_array[i++] = p;
930 BUG_ON(i != queue_size);
931
932 for (i = 0; i < queue_size; ++i) {
933 page_cache_debug("Evict %d:%d in list %d\n",
934 p_array[i]->owner->host_idx,
935 p_array[i]->file_page_idx,
936 p_array[i]->in_which_list);
937 flush_and_free_page(p_array[i]);
938 }
939
940 free(p_array);
941
942 pthread_mutex_unlock(&page_cache.page_cache_lock);
943 return 0;
944 }
945
page_cache_delete_single_page(struct page_cache_entity_of_inode * pce,pidx_t file_page_idx)946 int page_cache_delete_single_page(struct page_cache_entity_of_inode *pce,
947 pidx_t file_page_idx)
948 {
949 struct cached_page *p;
950 int ret = 0;
951
952 pthread_mutex_lock(&page_cache.page_cache_lock);
953
954 /* Ensuring target page is already existing. */
955 p = get_node_from_page_cache_entity(pce, file_page_idx);
956 if (p == NULL) {
957 page_cache_debug(
958 "[page_cache_delete_single_page] page does not exist.\n");
959 ret = -1;
960 goto out;
961 }
962
963 page_cache_debug("Delete %d:%d in list %d\n",
964 p->owner->host_idx,
965 p->file_page_idx,
966 p->in_which_list);
967 free_page(p);
968
969 out:
970 pthread_mutex_unlock(&page_cache.page_cache_lock);
971 return ret;
972 }
973
page_cache_delete_pages_of_inode(struct page_cache_entity_of_inode * pce)974 int page_cache_delete_pages_of_inode(struct page_cache_entity_of_inode *pce)
975 {
976 struct cached_page **p_array;
977 struct cached_page *p;
978 int queue_size, i = 0;
979
980 pthread_mutex_lock(&page_cache.page_cache_lock);
981
982 queue_size = pce->pages.size;
983 p_array = (struct cached_page **)calloc(pce->pages.size,
984 sizeof(struct cached_page *));
985 BUG_ON(p_array == NULL);
986
987 /* Find all target pages. */
988 for_each_in_list (
989 p, struct cached_page, inode_pages_node, &pce->pages.queue)
990 p_array[i++] = p;
991 BUG_ON(i != queue_size);
992
993 for (i = 0; i < queue_size; ++i) {
994 page_cache_debug("Delete %d:%d in list %d\n",
995 p_array[i]->owner->host_idx,
996 p_array[i]->file_page_idx,
997 p_array[i]->in_which_list);
998 free_page(p_array[i]);
999 }
1000
1001 free(p_array);
1002
1003 pthread_mutex_unlock(&page_cache.page_cache_lock);
1004 return 0;
1005 }
1006