1 #pragma once 2 #ifndef IWFSMFILE_H 3 #define IWFSMFILE_H 4 5 /************************************************************************************************** 6 * IOWOW library 7 * 8 * MIT License 9 * 10 * Copyright (c) 2012-2020 Softmotions Ltd <info@softmotions.com> 11 * 12 * Permission is hereby granted, free of charge, to any person obtaining a copy 13 * of this software and associated documentation files (the "Software"), to deal 14 * in the Software without restriction, including without limitation the rights 15 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 16 * copies of the Software, and to permit persons to whom the Software is 17 * furnished to do so, subject to the following conditions: 18 * 19 * The above copyright notice and this permission notice shall be included in all 20 * copies or substantial portions of the Software. 21 * 22 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 23 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 24 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 25 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 26 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 27 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 28 * SOFTWARE. 29 *************************************************************************************************/ 30 31 /** @file 32 * @brief Auto-expandable file with support of reader/writer address space 33 locking and free space block management using bitmaps. 34 * @author Anton Adamansky (adamansky@softmotions.com) 35 * 36 * @note Before using API of this module you should call 37 * `iw_init(void)` iowow module initialization routine. 38 * 39 * <strong>Features:</strong> 40 * 41 * - Address blocks allocation and deallocation using bitmaps. 42 * - Read/write file address space locking. 43 * - Tunable file expansion policies. 44 * - Read/write methods locking option in multithreaded environment. 45 * - File shrinking/truncation support. 46 * - A number mmaped regions can be registered in the file's address space. 47 * These regions used in read/write operation and automatically maintained 48 * during file resize operations. 49 * 50 * File operations implemented as function pointers contained 51 * in `IWFS_FSM` `C` structure. 52 * The `iwfs_fsmfile_open(IWFS_FSM *f, const IWFS_FSM_OPTS *opts)` opens file 53 * and initializes a given `IWFS_FSM` structure. 54 * 55 * <strong>File format:</strong> 56 * @verbatim 57 [FSM_CTL_MAGICK u32][block pow u8] 58 [bmoffset u64][bmlength u64] 59 [crzsum u64][crznum u32][crszvar u64][reserved u256] 60 [custom header size u32][custom header data...] 61 [fsm data...] @endverbatim 62 * 63 * <strong>where:</strong> 64 * 65 * - <b>FSM_CTL_MAGICK:</b> Free-space file magic number (32 bit) 66 * - <b>block pow:</b> Block size as power of `2` Eg: `6` means `64` bit block 67 size. (8 bit) 68 * - <b>bmoffset:</b> Free space bitmap area offset in bytes (64 bit) 69 * - <b>bmlength:</b> Free space bitmap area length. (64 bit) 70 * - <b>crzsum:</b> Number of allocated blocks. (64 bit) 71 * - <b>crznum:</b> Number of all allocated continuous areas. (32 bit) 72 * - <b>crszvar</b> Allocated areas length standard variance (deviation^2 * N) (64 bit) 73 * - <b>reserved:</b> Reserved space. 74 * - <b>custom header size:</b> Length of custom header area. See 75 `IWFS_FSM::writehdr` and `IWFS_FSM::readhdr` 76 */ 77 78 #include "iwexfile.h" 79 #include <stdbool.h> 80 #include <math.h> 81 82 IW_EXTERN_C_START 83 84 /** Free space allocation flags 85 * @see IWFS_FSM::allocate 86 */ 87 typedef uint8_t iwfs_fsm_aflags; 88 89 /** Use default allocation settings */ 90 #define IWFSM_ALLOC_DEFAULTS ((iwfs_fsm_aflags) 0x00U) 91 92 /** Do not @em overallocate a requested free space in order to reduce fragmentation */ 93 #define IWFSM_ALLOC_NO_OVERALLOCATE ((iwfs_fsm_aflags) 0x01U) 94 95 /** Do not extend the file and its bitmap free space mapping in the case if 96 * file size expansion is required. 97 * In this case the `IWFS_ERROR_NO_FREE_SPACE` error will be raised.*/ 98 #define IWFSM_ALLOC_NO_EXTEND ((iwfs_fsm_aflags) 0x02U) 99 100 /** Force offset of an allocated space to be page aligned. */ 101 #define IWFSM_ALLOC_PAGE_ALIGNED ((iwfs_fsm_aflags) 0x04U) 102 103 /** Do not collect internal allocation stats for this allocation. */ 104 #define IWFSM_ALLOC_NO_STATS ((iwfs_fsm_aflags) 0x08U) 105 106 /** Force all of the allocated address space backed by real file address space. */ 107 #define IWFSM_SOLID_ALLOCATED_SPACE ((iwfs_fsm_aflags) 0x10U) 108 109 /** Do msync of bitmap allocation index. */ 110 #define IWFSM_SYNC_BMAP ((iwfs_fsm_aflags) 0x20U) 111 112 #define IWFSM_MAGICK 0x19cc7cc 113 #define IWFSM_CUSTOM_HDR_DATA_OFFSET \ 114 (4 /*magic*/ + 1 /*block pow*/ + 8 /*fsm bitmap block offset */ + 8 /*fsm bitmap block length*/ + \ 115 8 /*all allocated block length sum */ + 4 /*number of all allocated areas */ + \ 116 8 /* allocated areas length standard variance (deviation^2 * N) */ + 32 /*reserved*/ + \ 117 4 /*custom hdr size*/) 118 119 /** File cleanup flags used in `IWFS_FSM::clear` */ 120 typedef uint8_t iwfs_fsm_clrfalgs; 121 122 /** Perform file size trimming after cleanup */ 123 #define IWFSM_CLEAR_TRIM ((iwfs_fsm_clrfalgs) 0x01U) 124 125 /** `IWFS_FSM` file open modes used in `IWFS_FSM_OPTS` */ 126 typedef uint8_t iwfs_fsm_openflags; 127 128 /** Do not use threading locks */ 129 #define IWFSM_NOLOCKS ((iwfs_fsm_openflags) 0x01U) 130 131 /** Strict block checking for alloc/dealloc operations. 10-15% performance overhead. */ 132 #define IWFSM_STRICT ((iwfs_fsm_openflags) 0x02U) 133 134 /** Do not trim fsm file on close */ 135 #define IWFSM_NO_TRIM_ON_CLOSE ((iwfs_fsm_openflags) 0x04U) 136 137 /** 138 * @brief Error codes specific to `IWFS_FSM`. 139 */ 140 typedef enum { 141 _IWFS_FSM_ERROR_START = (IW_ERROR_START + 4000UL), 142 IWFS_ERROR_NO_FREE_SPACE, /**< No free space. */ 143 IWFS_ERROR_INVALID_BLOCK_SIZE, /**< Invalid block size specified */ 144 IWFS_ERROR_RANGE_NOT_ALIGNED, /**< Specified range/offset is not aligned with 145 page/block */ 146 IWFS_ERROR_FSM_SEGMENTATION, /**< Free-space map segmentation error */ 147 IWFS_ERROR_INVALID_FILEMETA, /**< Invalid file-metadata */ 148 IWFS_ERROR_PLATFORM_PAGE, /**< Platform page size incopatibility, data 149 migration required. */ 150 IWFS_ERROR_RESIZE_FAIL, /**< Failed to resize file */ 151 _IWFS_FSM_ERROR_END 152 } iwfs_fsm_ecode; 153 154 /** 155 * @brief `IWFS_FSM` file options. 156 * @see iwfs_fsmfile_open(IWFS_FSM *f, const IWFS_FSM_OPTS *opts) 157 */ 158 typedef struct IWFS_FSM_OPTS { 159 IWFS_EXT_OPTS exfile; 160 size_t bmlen; /**< Initial size of free-space bitmap */ 161 uint32_t hdrlen; /**< Length of custom file header.*/ 162 iwfs_fsm_openflags oflags; /**< Operation mode flags */ 163 iwfs_ext_mmap_opts_t mmap_opts; /**< Defaul mmap options used in `add_mmap` */ 164 uint8_t bpow; /**< Block size power for 2 */ 165 bool mmap_all; /**< Mmap all file data */ 166 } IWFS_FSM_OPTS; 167 168 /** 169 * @brief `IWFS_FSM` file state container. 170 * @see IWFS_FSM::state 171 */ 172 typedef struct IWFS_FSM_STATE { 173 IWFS_EXT_STATE exfile; /**< File pool state */ 174 size_t block_size; /**< Size of data block in bytes. */ 175 iwfs_fsm_openflags oflags; /**< Operation mode flags. */ 176 uint32_t hdrlen; /**< Length of custom file header length in bytes */ 177 uint64_t blocks_num; /**< Number of available data blocks. */ 178 uint64_t free_segments_num; /**< Number of free (deallocated) continuous data 179 segments. */ 180 double_t avg_alloc_size; /**< Average allocation number of blocks */ 181 double_t alloc_dispersion; /**< Average allocation blocks dispersion */ 182 } IWFS_FSM_STATE; 183 184 typedef struct IWFS_FSMDBG_STATE { 185 IWFS_FSM_STATE state; 186 uint64_t bmoff; 187 uint64_t bmlen; 188 uint64_t lfbklen; 189 uint64_t lfbkoff; 190 } IWFS_FSMDBG_STATE; 191 192 /** 193 * @brief Auto-expandable file with support of reader/writer address space 194 * locking 195 * and free space blocks management using bitmaps. 196 */ 197 typedef struct IWFS_FSM { 198 struct IWFS_FSM_IMPL *impl; 199 200 /** 201 * @brief Allocate a continuous address space within a file 202 * with length greater or equal to the desired @a len bytes. 203 * 204 * `Offset` and `length` allocated area will be block size aligned. 205 * 206 * @param f `IWFS_FSM` file. 207 * @param len Desired length of an allocated area in bytes. 208 * @param [in,out] oaddr Placeholder for the address of an allocated area. 209 * Value of @a oaddr passed to this function used as 210 * `hint` in order 211 * to allocate area located closely to the specified @a 212 * oaddr value. 213 * @param [out] olen Actual length of an allocated area in bytes. 214 * @param opts Allocation options bitmask flag @ref iwfs_fsm_aflags 215 * @return `0` on success or error code. 216 */ 217 iwrc(*allocate)(struct IWFS_FSM *f, off_t len, off_t *oaddr, off_t *olen, 218 iwfs_fsm_aflags opts); 219 220 /** 221 * @brief Reallocate and adjust a size of an allocated block. 222 * 223 * If the given @a nlen value lesser than actual length of segment @a olen in 224 * that case 225 * segment will be truncated. 226 * 227 * @param f `IWFS_FSM` file. 228 * @param nlen Desired length of segment in bytes. 229 * @param oaddr [in,out] Address of an allocated segment. Placeholder for new 230 * address of reallocated segment. 231 * @param olen [in,out] Length of an allocated segment. Placeholder for length 232 * of reallocated segment. 233 * @param opts Allocation options bitmask flag @ref iwfs_fsm_aflags 234 * @return `0` on success or error code. 235 */ 236 iwrc(*reallocate)(struct IWFS_FSM *f, off_t nlen, off_t *oaddr, off_t *olen, 237 iwfs_fsm_aflags opts); 238 239 /** 240 * @brief Free a previously allocated area. 241 * @param addr Address space offset in bytes <em>it must be block size 242 * aligned</em>. 243 * @param len Length of area to release. 244 * @return `0` on success or error code. 245 */ 246 iwrc(*deallocate)(struct IWFS_FSM *f, off_t addr, off_t len); 247 248 249 /** 250 * @brief Check allocation status of region specified by @a addr and @a len 251 * @return `0` on success or error code. 252 */ 253 iwrc(*check_allocation_status)(struct IWFS_FSM *f, off_t addr, off_t len, bool allocated); 254 255 /** 256 * @brief Write a data to the custom file header. 257 * 258 * A custom file header size specified in IWFS_FSM_OPTS::hdrlen options on 259 * file creation. 260 * 261 * @param off Offset position relative to custom header start offset. 262 * @param buf Data buffer to write 263 * @param siz Number of bytes of @a buf to write into header. 264 * @return `0` on success or error code. 265 */ 266 iwrc(*writehdr)(struct IWFS_FSM *f, off_t off, const void *buf, off_t siz); 267 268 /** 269 * @brief Read a data from the custom file header. 270 * 271 * A custom file header size specified in IWFS_FSM_OPTS::hdrlen options on 272 * file creation. 273 * 274 * @param off Offset position relative to custom header start offset. 275 * @param [out] buf Data buffer to read into 276 * @param Number of bytes to read 277 */ 278 iwrc(*readhdr)(struct IWFS_FSM *f, off_t off, void *buf, off_t siz); 279 280 /** 281 * @brief Cleanup all allocated data blocks and reset the file to the initial 282 * empty state. 283 * 284 * @param clrflags 285 * @return `0` on success or error code. 286 */ 287 iwrc(*clear)(struct IWFS_FSM *f, iwfs_fsm_clrfalgs clrflags); 288 289 /* See iwexfile.h */ 290 291 /** @see IWFS_EXT::ensure_size */ 292 iwrc(*ensure_size)(struct IWFS_FSM *f, off_t size); 293 294 295 /** @see IWFS_EXT::add_mmap */ 296 iwrc(*add_mmap)(struct IWFS_FSM *f, off_t off, size_t maxlen, iwfs_ext_mmap_opts_t opts); 297 298 299 /** @see IWFS_EXT::remap_all */ 300 iwrc(*remap_all)(struct IWFS_FSM *f); 301 302 /** 303 * @brief Get a pointer to the registered mmap area starting at `off`. 304 * 305 * WARNING: Internal read lock will be acquired and 306 * must be released by subsequent `release_mmap()` call 307 * after all activity with mmaped region has finished. 308 * 309 * @see IWFS_FSM::add_mmap 310 * @see IWFS_EXT::acquire_mmap 311 */ 312 iwrc(*acquire_mmap)(struct IWFS_FSM *f, off_t off, uint8_t **mm, size_t *sp); 313 314 /** 315 * @brief Retrieve mmaped region by its offset @a off 316 */ 317 iwrc(*probe_mmap)(struct IWFS_FSM *f, off_t off, uint8_t **mm, size_t *sp); 318 319 /** 320 * @brief Release the lock acquired by successfull call of `acquire_mmap()` 321 */ 322 iwrc(*release_mmap)(struct IWFS_FSM *f); 323 324 /** @see IWFS_EXT::remove_mmap */ 325 iwrc(*remove_mmap)(struct IWFS_FSM *f, off_t off); 326 327 /** @see IWFS_EXT::sync_mmap */ 328 iwrc(*sync_mmap)(struct IWFS_FSM *f, off_t off, iwfs_sync_flags flags); 329 330 /* See iwfile.h */ 331 332 /** @see IWFS_FILE::write */ 333 iwrc(*write)(struct IWFS_FSM *f, off_t off, const void *buf, size_t siz, 334 size_t *sp); 335 336 /** @see IWFS_FILE::read */ 337 iwrc(*read)(struct IWFS_FSM *f, off_t off, void *buf, size_t siz, 338 size_t *sp); 339 340 /** @see IWFS_FILE::close */ 341 iwrc(*close)(struct IWFS_FSM *f); 342 343 /** @see IWFS_FILE::sync */ 344 iwrc(*sync)(struct IWFS_FSM *f, iwfs_sync_flags flags); 345 346 /** @see IWFS_FILE::state */ 347 iwrc(*state)(struct IWFS_FSM *f, IWFS_FSM_STATE *state); 348 349 /** get access to the underlying iwextfile instance */ 350 iwrc(*extfile)(struct IWFS_FSM *f, IWFS_EXT **ext); 351 352 } IWFS_FSM; 353 354 /** 355 * @brief Open `IWFS_FSM` file. 356 * 357 * <strong>Example:</strong> 358 * 359 * Open a buffer pool file for multithreaded env with fibonacci file resize 360 * policy with block size of 64 bytes and custom file header of 255 bytes 361 * length. 362 * 363 * @code {.c} 364 * IWFS_FSM_OPTS opts = { 365 * .exfile = { 366 * .file = { 367 * .path = "myfile.dat", 368 * .omode = IWFS_OWRITE | IWFS_OCREATE, 369 * .lock_mode = IWP_WLOCK 370 * }, 371 * .rspolicy = iw_exfile_szpolicy_fibo 372 * }, 373 * .bpow = 6, // 2^6 bytes block size 374 * .hdrlen = 255, // Size of custom file header 375 * .oflags = IWFSM_STRICT // Use verbose free-space bitmap checking for 376 * // allocations (10-15% overhead) 377 * }; 378 * 379 * IWFS_FSM f; 380 * size_t sp; 381 * off_t space_len, space_addr = 0; 382 * 383 * iwrc rc = iwfs_fsmfile_open(&f, &opts); 384 * 385 * //Allocate 2 blocks of file space 386 * rc = f.allocate(&f, 128, &space_addr, &space_len, 0); 387 * if (!rc) { 388 * int data = 33; 389 * // Write some data to the allocated block with writer lock acquired on 390 * // `[space_addr, sizeof(data))` 391 * rc = f.lwrite(&f, space_addr, &data, sizeof(data), &sp); 392 * ... 393 * } 394 * ... 395 * @endcode 396 * 397 * @param f File handle 398 * @param opts File open options 399 * @relatesalso IWFS_FSM 400 */ 401 IW_EXPORT WUR iwrc iwfs_fsmfile_open(IWFS_FSM *f, const IWFS_FSM_OPTS *opts); 402 /** 403 * @brief Init `iwfsmfile` submodule. 404 */ 405 IW_EXPORT WUR iwrc iwfs_fsmfile_init(void); 406 407 IW_EXTERN_C_END 408 409 #endif 410