1 #pragma once 2 #ifndef IWFSMFILE_H 3 #define IWFSMFILE_H 4 5 /************************************************************************************************** 6 * IOWOW library 7 * 8 * MIT License 9 * 10 * Copyright (c) 2012-2022 Softmotions Ltd <info@softmotions.com> 11 * 12 * Permission is hereby granted, free of charge, to any person obtaining a copy 13 * of this software and associated documentation files (the "Software"), to deal 14 * in the Software without restriction, including without limitation the rights 15 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 16 * copies of the Software, and to permit persons to whom the Software is 17 * furnished to do so, subject to the following conditions: 18 * 19 * The above copyright notice and this permission notice shall be included in all 20 * copies or substantial portions of the Software. 21 * 22 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 23 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 24 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 25 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 26 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 27 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 28 * SOFTWARE. 29 *************************************************************************************************/ 30 31 /** @file 32 * @brief Auto-expandable file with support of reader/writer address space 33 locking and free space block management using bitmaps. 34 * @author Anton Adamansky (adamansky@softmotions.com) 35 * 36 * @note Before using API of this module you should call 37 * `iw_init(void)` iowow module initialization routine. 38 * 39 * <strong>Features:</strong> 40 * 41 * - Address blocks allocation and deallocation using bitmaps. 42 * - Read/write file address space locking. 43 * - Tunable file expansion policies. 44 * - Read/write methods locking option in multithreaded environment. 45 * - File shrinking/truncation support. 46 * - A number mmaped regions can be registered in the file's address space. 47 * These regions used in read/write operation and automatically maintained 48 * during file resize operations. 49 * 50 * File operations implemented as function pointers contained 51 * in `IWFS_FSM` `C` structure. 52 * The `iwfs_fsmfile_open(IWFS_FSM *f, const IWFS_FSM_OPTS *opts)` opens file 53 * and initializes a given `IWFS_FSM` structure. 54 * 55 * <strong>File format:</strong> 56 * @verbatim 57 [FSM_CTL_MAGICK u32][block pow u8] 58 [bmoffset u64][bmlength u64] 59 [crzsum u64][crznum u32][crszvar u64][reserved u256] 60 [custom header size u32][custom header data...] 61 [fsm data...] @endverbatim 62 * 63 * <strong>where:</strong> 64 * 65 * - <b>FSM_CTL_MAGICK:</b> Free-space file magic number (32 bit) 66 * - <b>block pow:</b> Block size as power of `2` Eg: `6` means `64` bit block 67 size. (8 bit) 68 * - <b>bmoffset:</b> Free space bitmap area offset in bytes (64 bit) 69 * - <b>bmlength:</b> Free space bitmap area length. (64 bit) 70 * - <b>crzsum:</b> Number of allocated blocks. (64 bit) 71 * - <b>crznum:</b> Number of all allocated continuous areas. (32 bit) 72 * - <b>crszvar</b> Allocated areas length standard variance (deviation^2 * N) (64 bit) 73 * - <b>reserved:</b> Reserved space. 74 * - <b>custom header size:</b> Length of custom header area. See 75 `IWFS_FSM::writehdr` and `IWFS_FSM::readhdr` 76 */ 77 78 #include "iwexfile.h" 79 #include <stdbool.h> 80 #include <math.h> 81 82 IW_EXTERN_C_START 83 84 /** Free space allocation flags 85 * @see IWFS_FSM::allocate 86 */ 87 typedef uint8_t iwfs_fsm_aflags; 88 89 /** Use default allocation settings */ 90 #define IWFSM_ALLOC_DEFAULTS ((iwfs_fsm_aflags) 0x00U) 91 92 /** Do not @em overallocate a requested free space in order to reduce fragmentation */ 93 #define IWFSM_ALLOC_NO_OVERALLOCATE ((iwfs_fsm_aflags) 0x01U) 94 95 /** Do not extend the file and its bitmap free space mapping in the case if 96 * file size expansion is required. 97 * In this case the `IWFS_ERROR_NO_FREE_SPACE` error will be raised.*/ 98 #define IWFSM_ALLOC_NO_EXTEND ((iwfs_fsm_aflags) 0x02U) 99 100 /** Force offset of an allocated space to be page aligned. */ 101 #define IWFSM_ALLOC_PAGE_ALIGNED ((iwfs_fsm_aflags) 0x04U) 102 103 /** Do not collect internal allocation stats for this allocation. */ 104 #define IWFSM_ALLOC_NO_STATS ((iwfs_fsm_aflags) 0x08U) 105 106 /** Force all of the allocated address space backed by real file address space. */ 107 #define IWFSM_SOLID_ALLOCATED_SPACE ((iwfs_fsm_aflags) 0x10U) 108 109 /** Do msync of bitmap allocation index. */ 110 #define IWFSM_SYNC_BMAP ((iwfs_fsm_aflags) 0x20U) 111 112 #define IWFSM_MAGICK 0x19cc7cc 113 #define IWFSM_CUSTOM_HDR_DATA_OFFSET \ 114 (4 /*magic*/ + 1 /*block pow*/ + 8 /*fsm bitmap block offset */ + 8 /*fsm bitmap block length*/ \ 115 + 8 /*all allocated block length sum */ + 4 /*number of all allocated areas */ \ 116 + 8 /* allocated areas length standard variance (deviation^2 * N) */ + 32 /*reserved*/ \ 117 + 4 /*custom hdr size*/) 118 119 /** File cleanup flags used in `IWFS_FSM::clear` */ 120 typedef uint8_t iwfs_fsm_clrfalgs; 121 122 /** Perform file size trimming after cleanup */ 123 #define IWFSM_CLEAR_TRIM ((iwfs_fsm_clrfalgs) 0x01U) 124 125 /** `IWFS_FSM` file open modes used in `IWFS_FSM_OPTS` */ 126 typedef uint8_t iwfs_fsm_openflags; 127 128 /** Do not use threading locks */ 129 #define IWFSM_NOLOCKS ((iwfs_fsm_openflags) 0x01U) 130 131 /** Strict block checking for alloc/dealloc operations. 10-15% performance overhead. */ 132 #define IWFSM_STRICT ((iwfs_fsm_openflags) 0x02U) 133 134 /** Do not trim fsm file on close */ 135 #define IWFSM_NO_TRIM_ON_CLOSE ((iwfs_fsm_openflags) 0x04U) 136 137 /** 138 * @brief Error codes specific to `IWFS_FSM`. 139 */ 140 typedef enum { 141 _IWFS_FSM_ERROR_START = (IW_ERROR_START + 4000UL), 142 IWFS_ERROR_NO_FREE_SPACE, /**< No free space. */ 143 IWFS_ERROR_INVALID_BLOCK_SIZE, /**< Invalid block size specified */ 144 IWFS_ERROR_RANGE_NOT_ALIGNED, 145 /**< Specified range/offset is not aligned with 146 page/block */ 147 IWFS_ERROR_FSM_SEGMENTATION, /**< Free-space map segmentation error */ 148 IWFS_ERROR_INVALID_FILEMETA, /**< Invalid file-metadata */ 149 IWFS_ERROR_PLATFORM_PAGE, 150 /**< Platform page size incopatibility, data 151 migration required. */ 152 IWFS_ERROR_RESIZE_FAIL, /**< Failed to resize file */ 153 _IWFS_FSM_ERROR_END, 154 } iwfs_fsm_ecode; 155 156 /** 157 * @brief `IWFS_FSM` file options. 158 * @see iwfs_fsmfile_open(IWFS_FSM *f, const IWFS_FSM_OPTS *opts) 159 */ 160 typedef struct IWFS_FSM_OPTS { 161 IWFS_EXT_OPTS exfile; 162 size_t bmlen; /**< Initial size of free-space bitmap */ 163 uint32_t hdrlen; /**< Length of custom file header.*/ 164 iwfs_fsm_openflags oflags; /**< Operation mode flags */ 165 iwfs_ext_mmap_opts_t mmap_opts; /**< Defaul mmap options used in `add_mmap` */ 166 uint8_t bpow; /**< Block size power for 2 */ 167 bool mmap_all; /**< Mmap all file data */ 168 } IWFS_FSM_OPTS; 169 170 /** 171 * @brief `IWFS_FSM` file state container. 172 * @see IWFS_FSM::state 173 */ 174 typedef struct IWFS_FSM_STATE { 175 IWFS_EXT_STATE exfile; /**< File pool state */ 176 size_t block_size; /**< Size of data block in bytes. */ 177 iwfs_fsm_openflags oflags; /**< Operation mode flags. */ 178 uint64_t blocks_num; /**< Number of available data blocks. */ 179 uint32_t free_segments_num; /**< Number of free (deallocated) continuous data segments */ 180 uint32_t hdrlen; /**< Length of custom file header length in bytes */ 181 double_t avg_alloc_size; /**< Average allocation number of blocks */ 182 double_t alloc_dispersion; /**< Average allocation blocks dispersion */ 183 } IWFS_FSM_STATE; 184 185 typedef struct IWFS_FSMDBG_STATE { 186 IWFS_FSM_STATE state; 187 uint64_t bmoff; 188 uint64_t bmlen; 189 uint64_t lfbklen; 190 uint64_t lfbkoff; 191 } IWFS_FSMDBG_STATE; 192 193 /** 194 * @brief Auto-expandable file with support of reader/writer address space 195 * locking 196 * and free space blocks management using bitmaps. 197 */ 198 typedef struct IWFS_FSM { 199 struct fsm *impl; 200 201 /** 202 * @brief Allocate a continuous address space within a file 203 * with length greater or equal to the desired @a len bytes. 204 * 205 * `Offset` and `length` allocated area will be block size aligned. 206 * 207 * @param f `IWFS_FSM` file. 208 * @param len Desired length of an allocated area in bytes. 209 * @param [in,out] oaddr Placeholder for the address of an allocated area. 210 * Value of @a oaddr passed to this function used as 211 * `hint` in order 212 * to allocate area located closely to the specified @a 213 * oaddr value. 214 * @param [out] olen Actual length of an allocated area in bytes. 215 * @param opts Allocation options bitmask flag @ref iwfs_fsm_aflags 216 * @return `0` on success or error code. 217 */ 218 iwrc (*allocate)( 219 struct IWFS_FSM *f, off_t len, off_t *oaddr, off_t *olen, 220 iwfs_fsm_aflags opts); 221 222 /** 223 * @brief Reallocate and adjust a size of an allocated block. 224 * 225 * If the given @a nlen value lesser than actual length of segment @a olen in 226 * that case 227 * segment will be truncated. 228 * 229 * @param f `IWFS_FSM` file. 230 * @param nlen Desired length of segment in bytes. 231 * @param oaddr [in,out] Address of an allocated segment. Placeholder for new 232 * address of reallocated segment. 233 * @param olen [in,out] Length of an allocated segment. Placeholder for length 234 * of reallocated segment. 235 * @param opts Allocation options bitmask flag @ref iwfs_fsm_aflags 236 * @return `0` on success or error code. 237 */ 238 iwrc (*reallocate)( 239 struct IWFS_FSM *f, off_t nlen, off_t *oaddr, off_t *olen, 240 iwfs_fsm_aflags opts); 241 242 /** 243 * @brief Free a previously allocated area. 244 * @param addr Address space offset in bytes <em>it must be block size 245 * aligned</em>. 246 * @param len Length of area to release. 247 * @return `0` on success or error code. 248 */ 249 iwrc (*deallocate)(struct IWFS_FSM *f, off_t addr, off_t len); 250 251 252 /** 253 * @brief Check allocation status of region specified by @a addr and @a len 254 * @return `0` on success or error code. 255 */ 256 iwrc (*check_allocation_status)(struct IWFS_FSM *f, off_t addr, off_t len, bool allocated); 257 258 /** 259 * @brief Write a data to the custom file header. 260 * 261 * A custom file header size specified in IWFS_FSM_OPTS::hdrlen options on 262 * file creation. 263 * 264 * @param off Offset position relative to custom header start offset. 265 * @param buf Data buffer to write 266 * @param siz Number of bytes of @a buf to write into header. 267 * @return `0` on success or error code. 268 */ 269 iwrc (*writehdr)(struct IWFS_FSM *f, off_t off, const void *buf, off_t siz); 270 271 /** 272 * @brief Read a data from the custom file header. 273 * 274 * A custom file header size specified in IWFS_FSM_OPTS::hdrlen options on 275 * file creation. 276 * 277 * @param off Offset position relative to custom header start offset. 278 * @param [out] buf Data buffer to read into 279 * @param Number of bytes to read 280 */ 281 iwrc (*readhdr)(struct IWFS_FSM *f, off_t off, void *buf, off_t siz); 282 283 /** 284 * @brief Cleanup all allocated data blocks and reset the file to the initial 285 * empty state. 286 * 287 * @param clrflags 288 * @return `0` on success or error code. 289 */ 290 iwrc (*clear)(struct IWFS_FSM *f, iwfs_fsm_clrfalgs clrflags); 291 292 /* See iwexfile.h */ 293 294 /** @see IWFS_EXT::ensure_size */ 295 iwrc (*ensure_size)(struct IWFS_FSM *f, off_t size); 296 297 298 /** @see IWFS_EXT::add_mmap */ 299 iwrc (*add_mmap)(struct IWFS_FSM *f, off_t off, size_t maxlen, iwfs_ext_mmap_opts_t opts); 300 301 302 /** @see IWFS_EXT::remap_all */ 303 iwrc (*remap_all)(struct IWFS_FSM *f); 304 305 /** 306 * @brief Get a pointer to the registered mmap area starting at `off`. 307 * 308 * WARNING: Internal read lock will be acquired and 309 * must be released by subsequent `release_mmap()` call 310 * after all activity with mmaped region has finished. 311 * 312 * @see IWFS_FSM::add_mmap 313 * @see IWFS_EXT::acquire_mmap 314 */ 315 iwrc (*acquire_mmap)(struct IWFS_FSM *f, off_t off, uint8_t **mm, size_t *sp); 316 317 /** 318 * @brief Retrieve mmaped region by its offset @a off 319 */ 320 iwrc (*probe_mmap)(struct IWFS_FSM *f, off_t off, uint8_t **mm, size_t *sp); 321 322 /** 323 * @brief Release the lock acquired by successfull call of `acquire_mmap()` 324 */ 325 iwrc (*release_mmap)(struct IWFS_FSM *f); 326 327 /** @see IWFS_EXT::remove_mmap */ 328 iwrc (*remove_mmap)(struct IWFS_FSM *f, off_t off); 329 330 /** @see IWFS_EXT::sync_mmap */ 331 iwrc (*sync_mmap)(struct IWFS_FSM *f, off_t off, iwfs_sync_flags flags); 332 333 /* See iwfile.h */ 334 335 /** @see IWFS_FILE::write */ 336 iwrc (*write)( 337 struct IWFS_FSM *f, off_t off, const void *buf, size_t siz, 338 size_t *sp); 339 340 /** @see IWFS_FILE::read */ 341 iwrc (*read)( 342 struct IWFS_FSM *f, off_t off, void *buf, size_t siz, 343 size_t *sp); 344 345 /** @see IWFS_FILE::close */ 346 iwrc (*close)(struct IWFS_FSM *f); 347 348 /** @see IWFS_FILE::sync */ 349 iwrc (*sync)(struct IWFS_FSM *f, iwfs_sync_flags flags); 350 351 /** @see IWFS_FILE::state */ 352 iwrc (*state)(struct IWFS_FSM *f, IWFS_FSM_STATE *state); 353 354 /** get access to the underlying iwextfile instance */ 355 iwrc (*extfile)(struct IWFS_FSM *f, IWFS_EXT **ext); 356 } IWFS_FSM; 357 358 /** 359 * @brief Open `IWFS_FSM` file. 360 * 361 * <strong>Example:</strong> 362 * 363 * Open a buffer pool file for multithreaded env with fibonacci file resize 364 * policy with block size of 64 bytes and custom file header of 255 bytes 365 * length. 366 * 367 * @code {.c} 368 * IWFS_FSM_OPTS opts = { 369 * .exfile = { 370 * .file = { 371 * .path = "myfile.dat", 372 * .omode = IWFS_OWRITE | IWFS_OCREATE, 373 * .lock_mode = IWP_WLOCK 374 * }, 375 * .rspolicy = iw_exfile_szpolicy_fibo 376 * }, 377 * .bpow = 6, // 2^6 bytes block size 378 * .hdrlen = 255, // Size of custom file header 379 * .oflags = IWFSM_STRICT // Use verbose free-space bitmap checking for 380 * // allocations (10-15% overhead) 381 * }; 382 * 383 * IWFS_FSM f; 384 * size_t sp; 385 * off_t space_len, space_addr = 0; 386 * 387 * iwrc rc = iwfs_fsmfile_open(&f, &opts); 388 * 389 * //Allocate 2 blocks of file space 390 * rc = f.allocate(&f, 128, &space_addr, &space_len, 0); 391 * if (!rc) { 392 * int data = 33; 393 * // Write some data to the allocated block with writer lock acquired on 394 * // `[space_addr, sizeof(data))` 395 * rc = f.lwrite(&f, space_addr, &data, sizeof(data), &sp); 396 * ... 397 * } 398 * ... 399 * @endcode 400 * 401 * @param f File handle 402 * @param opts File open options 403 * @relatesalso IWFS_FSM 404 */ 405 IW_EXPORT WUR iwrc iwfs_fsmfile_open(IWFS_FSM *f, const IWFS_FSM_OPTS *opts); 406 407 /** 408 * @brief Init `iwfsmfile` submodule. 409 */ 410 IW_EXPORT WUR iwrc iwfs_fsmfile_init(void); 411 412 IW_EXTERN_C_END 413 414 #endif 415