1 #pragma once 2 #ifndef IWKV_H 3 #define IWKV_H 4 5 /************************************************************************************************** 6 * IOWOW library 7 * 8 * MIT License 9 * 10 * Copyright (c) 2012-2020 Softmotions Ltd <info@softmotions.com> 11 * 12 * Permission is hereby granted, free of charge, to any person obtaining a copy 13 * of this software and associated documentation files (the "Software"), to deal 14 * in the Software without restriction, including without limitation the rights 15 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 16 * copies of the Software, and to permit persons to whom the Software is 17 * furnished to do so, subject to the following conditions: 18 * 19 * The above copyright notice and this permission notice shall be included in all 20 * copies or substantial portions of the Software. 21 * 22 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 23 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 24 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 25 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 26 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 27 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 28 * SOFTWARE. 29 *************************************************************************************************/ 30 31 /** @file 32 * @brief Persistent key-value storage based on skiplist 33 * datastructure (https://en.wikipedia.org/wiki/Skip_list). 34 * @author Anton Adamansky (adamansky@softmotions.com) 35 * 36 * <strong>Features:<strong> 37 * - Simple design of key value storage 38 * - Lightweight shared/static library: 200Kb 39 * - Support of multiple key-value databases within a single file 40 * - Compound keys supported 41 * - Ultra-fast traversal of database records 42 * - Native support of integer keys 43 * 44 * <strong>Limitations:<strong> 45 * - Maximum iwkv storage file size: 512 GB (0x7fffffff80) 46 * - Total size of a single key+value record must be not greater than 255Mb (0xfffffff) 47 * - In-memory cache for every opened database takes ~130Kb, cache can be disposed by `iwkv_db_cache_release()` 48 */ 49 50 #include "iowow.h" 51 #include "iwfsmfile.h" 52 #include <stddef.h> 53 #include <stdbool.h> 54 55 IW_EXTERN_C_START 56 57 // Max key + value size: 255Mb 58 #define IWKV_MAX_KVSZ 0xfffffff 59 60 /** 61 * @brief IWKV error codes. 62 */ 63 typedef enum { 64 _IWKV_ERROR_START = (IW_ERROR_START + 5000UL), 65 IWKV_ERROR_NOTFOUND, /**< Key not found (IWKV_ERROR_NOTFOUND) */ 66 IWKV_ERROR_KEY_EXISTS, /**< Key already exists (IWKV_ERROR_KEY_EXISTS) */ 67 IWKV_ERROR_MAXKVSZ, /**< Size of Key+value must be not greater than 0xfffffff bytes (IWKV_ERROR_MAXKVSZ) */ 68 IWKV_ERROR_CORRUPTED, /**< Database file invalid or corrupted (IWKV_ERROR_CORRUPTED) */ 69 IWKV_ERROR_DUP_VALUE_SIZE, /**< Value size is not compatible for insertion into sorted values array (IWKV_ERROR_DUP_VALUE_SIZE) */ 70 IWKV_ERROR_KEY_NUM_VALUE_SIZE, /**< Given key is not compatible to storage as number (IWKV_ERROR_KEY_NUM_VALUE_SIZE) */ 71 IWKV_ERROR_INCOMPATIBLE_DB_MODE, /**< Incorpatible database open mode (IWKV_ERROR_INCOMPATIBLE_DB_MODE) */ 72 IWKV_ERROR_INCOMPATIBLE_DB_FORMAT, /**< Incompatible database format version, please migrate database data (IWKV_ERROR_INCOMPATIBLE_DB_FORMAT) */ 73 IWKV_ERROR_CORRUPTED_WAL_FILE, /**< Corrupted WAL file (IWKV_ERROR_CORRUPTED_WAL_FILE) */ 74 IWKV_ERROR_VALUE_CANNOT_BE_INCREMENTED, /**< Stored value cannot be incremented/descremented (IWKV_ERROR_VALUE_CANNOT_BE_INCREMENTED) */ 75 IWKV_ERROR_WAL_MODE_REQUIRED, /**< Operation requires WAL enabled database. (IWKV_ERROR_WAL_MODE_REQUIRED) */ 76 IWKV_ERROR_BACKUP_IN_PROGRESS, /**< Backup operation in progress. (IWKV_ERROR_BACKUP_IN_PROGRESS) */ 77 _IWKV_ERROR_END, 78 79 // Internal only 80 _IWKV_RC_KVBLOCK_FULL, 81 _IWKV_RC_REQUIRE_NLEVEL, 82 _IWKV_RC_END, 83 } iwkv_ecode; 84 85 /** Database file open modes. */ 86 typedef uint8_t iwkv_openflags; 87 /** Open storage file in read-only mode */ 88 #define IWKV_RDONLY ((iwkv_openflags) 0x02U) 89 /** Truncate storage file on open */ 90 #define IWKV_TRUNC ((iwkv_openflags) 0x04U) 91 #define IWKV_NO_TRIM_ON_CLOSE ((iwkv_openflags) 0x08U) 92 93 /** Database initialization modes */ 94 typedef uint8_t iwdb_flags_t; 95 96 /** Floating point number keys represented as string (char*) value. */ 97 #define IWDB_REALNUM_KEYS ((iwdb_flags_t) 0x10U) 98 99 /** Variable-length number keys */ 100 #define IWDB_VNUM64_KEYS ((iwdb_flags_t) 0x20U) 101 102 /** 103 * Enable compound database keys. Keys stored in the following format: `<key value prefix><numeric key suffix>` 104 * Allows associate one `key value` with many references represented as VNUM64 (eg.: Non unique table indexes). 105 * @see IWKV_val.compound 106 */ 107 #define IWDB_COMPOUND_KEYS ((iwdb_flags_t) 0x40U) 108 109 /** Record store modes used in `iwkv_put()` and `iwkv_cursor_set()` functions. */ 110 typedef uint8_t iwkv_opflags; 111 112 /** Do not overwrite value for an existing key. 113 `IWKV_ERROR_KEY_EXISTS` will be returned in such cases. */ 114 #define IWKV_NO_OVERWRITE ((iwkv_opflags) 0x01U) 115 116 /** Flush changes on disk after operation */ 117 #define IWKV_SYNC ((iwkv_opflags) 0x04U) 118 119 /** Increment/decrement stored UINT32|UINT64 value by given INT32|INT64 number 120 `IWKV_ERROR_KEY_EXISTS` does not makes sense if this flag set. */ 121 #define IWKV_VAL_INCREMENT ((iwkv_opflags) 0x10U) 122 123 struct _IWKV; 124 typedef struct _IWKV *IWKV; 125 126 struct _IWDB; 127 typedef struct _IWDB *IWDB; 128 129 /** 130 * @brief Write ahead log (WAL) options. 131 */ 132 typedef struct IWKV_WAL_OPTS { 133 bool enabled; /**< WAL enabled */ 134 bool check_crc_on_checkpoint; /**< Check CRC32 sum of data blocks during checkpoint. Default: false */ 135 uint32_t savepoint_timeout_sec; /**< Savepoint timeout seconds. Default: 10 sec */ 136 uint32_t checkpoint_timeout_sec; /**< Checkpoint timeout seconds. Default: 300 sec (5 min); */ 137 size_t wal_buffer_sz; /**< WAL file intermediate buffer size. Default: 8Mb */ 138 uint64_t checkpoint_buffer_sz; /**< Checkpoint buffer size in bytes. Default: 1Gb */ 139 iwrc(*wal_lock_interceptor)(bool, void *); 140 /**< Optional function called 141 - before acquiring 142 - after releasing 143 exclusive database lock by WAL checkpoint thread. 144 In the case of `before lock` first argument will be set to true */ 145 void *wal_lock_interceptor_opaque;/**< Opaque data for `wal_lock_interceptor` */ 146 } IWKV_WAL_OPTS; 147 148 /** 149 * @brief IWKV storage open options. 150 */ 151 typedef struct IWKV_OPTS { 152 const char *path; /**< Path to database file */ 153 uint32_t random_seed; /**< Random seed used for iwu random generator */ 154 /** 155 * Database storage format version. 156 * Leave it as zero for the latest supported format. 157 * Used only for newly created databases, 158 */ 159 int32_t fmt_version; 160 iwkv_openflags oflags; /**< Bitmask of database file open modes */ 161 bool file_lock_fail_fast; /**< Do not wait and raise error if database is locked by another process */ 162 IWKV_WAL_OPTS wal; /**< WAL options */ 163 } IWKV_OPTS; 164 165 /** 166 * @brief Data container for key/value. 167 */ 168 typedef struct IWKV_val { 169 void *data; /**< Data buffer */ 170 size_t size; /**< Data buffer size */ 171 /** Extra key part used for key comparison. 172 * If set to non zero and database is created with `IWDB_COMPOUND_KEYS` mode 173 * keys will behave as compound: `<key value><compound>` consisting of two parts. 174 * `compound` field ignored if db not in `IWDB_COMPOUND_KEYS` mode. 175 */ 176 int64_t compound; 177 } IWKV_val; 178 179 /** 180 * @brief Cursor opaque handler. 181 */ 182 struct _IWKV_cursor; 183 typedef struct _IWKV_cursor *IWKV_cursor; 184 185 /** 186 * @brief Database cursor operations and position flags. 187 */ 188 typedef enum IWKV_cursor_op { 189 IWKV_CURSOR_BEFORE_FIRST = 1, /**< Set cursor to position before first record */ 190 IWKV_CURSOR_AFTER_LAST, /**< Set cursor to position after last record */ 191 IWKV_CURSOR_NEXT, /**< Move cursor to the next record */ 192 IWKV_CURSOR_PREV, /**< Move cursor to the previous record */ 193 IWKV_CURSOR_EQ, /**< Set cursor to the specified key value */ 194 IWKV_CURSOR_GE /**< Set cursor to the key which greater or equal key specified */ 195 } IWKV_cursor_op; 196 197 /** 198 * @brief Initialize iwkv storage. 199 * @details This method must be called before using of any iwkv public API function. 200 * @note iwkv implicitly initialized by iw_init() 201 */ 202 IW_EXPORT WUR iwrc iwkv_init(void); 203 204 /** 205 * @brief Open iwkv storage. 206 * @code {.c} 207 * IWKV iwkv; 208 * IWKV_OPTS opts = { 209 * .path = "mystore.db" 210 * }; 211 * iwrc rc = iwkv_open(&opts, &iwkv); 212 * @endcode 213 * @note Any opened iwkv storage must be closed by `iwkv_close()` after usage. 214 * @param opts Database open options. 215 * @param [out] iwkvp Pointer to @ref IWKV structure. 216 */ 217 IW_EXPORT WUR iwrc iwkv_open(const IWKV_OPTS *opts, IWKV *iwkvp); 218 219 /** 220 * @brief Get iwkv database handler identified by specified `dbid` number. 221 * @details In the case if no database matched `dbid` 222 * a new database will be created using specified function arguments. 223 * 224 * @note Database handler doesn't require to be explicitly closed or freed. 225 * Although it may be usefull to release database cache memory of unused databases 226 * dependening on memory requirements of your application by `iwkv_db_cache_release()`. 227 * @note Database `flags` argument must be same for all subsequent 228 * calls after first call for particular database, 229 * otherwise `IWKV_ERROR_INCOMPATIBLE_DB_MODE` will be reported. 230 * 231 * @param iwkv Pointer to @ref IWKV handler 232 * @param dbid Database identifier 233 * @param flags Database initialization flags 234 * @param [out] dbp Pointer to database opaque structure 235 */ 236 IW_EXPORT WUR iwrc iwkv_db(IWKV iwkv, uint32_t dbid, iwdb_flags_t flags, IWDB *dbp); 237 238 /** 239 * @brief Create new database with next available database id. 240 * @see iwrc iwkv_db() 241 * 242 * @param flags Database initialization flags 243 * @param [out] dbidp Database identifier placeholder will be filled with next available id. 244 * @param [out] dbp Pointer to database opaque structure 245 */ 246 IW_EXPORT WUR iwrc iwkv_new_db(IWKV iwkv, iwdb_flags_t dbflg, uint32_t *dbidp, IWDB *dbp); 247 248 /** 249 * @brief Frees memory resources used by database cache 250 * until to next database access operation (get/put/cursor). 251 * Typicaly it will free ~130Kb of memory per database in use. 252 * 253 * @param db Database handler 254 */ 255 IW_EXPORT iwrc iwkv_db_cache_release(IWDB db); 256 257 /** 258 * @brief Destroy(drop) existing database and cleanup all of its data. 259 * 260 * @param dbp Pointer to database opened. 261 */ 262 IW_EXPORT iwrc iwkv_db_destroy(IWDB *dbp); 263 264 /** 265 * @brief Sync iwkv storage state with disk. 266 * 267 * @note It will cause deadlock if current thread holds opened cursors and WAL is enabled, 268 * use method with caution. 269 * 270 * @param iwkv IWKV handler. 271 * @param flags Sync flags. 272 */ 273 IW_EXPORT iwrc iwkv_sync(IWKV iwkv, iwfs_sync_flags flags); 274 275 /** 276 * @brief Close iwkv storage. 277 * @details Upon successfull call of iwkv_close() 278 * no farther operations on storage or any of its databases are allowed. 279 * 280 * @param iwkvp 281 */ 282 IW_EXPORT iwrc iwkv_close(IWKV *iwkvp); 283 284 /** 285 * @brief Store record in database. 286 * 287 * iwkv_opflags opflags: 288 * - `IWKV_NO_OVERWRITE` If a key is already exists the `IWKV_ERROR_KEY_EXISTS` error will returned. 289 * - `IWKV_SYNC` Flush changes on disk after operation 290 * 291 * @note `iwkv_put()` adds a new value to sorted values array for existing keys if 292 * database created with `IWDB_DUP_UINT32_VALS`|`IWDB_DUP_UINT64_VALS` flags 293 * 294 * @param db Database handler 295 * @param key Key data container 296 * @param val Value data container 297 * @param opflags Put options used 298 */ 299 IW_EXPORT iwrc iwkv_put(IWDB db, const IWKV_val *key, const IWKV_val *val, iwkv_opflags opflags); 300 301 /** 302 * @brief Intercepts old(replaced) value in put operation. 303 * @note If `oldval` is not zero IWKV_PUT_HANDLER responsive for releasing it using iwkv_val_dispose() 304 * @warning Use `IWKV_PUT_HANDLER` with caution: mind deadlocks. 305 * 306 * @param key Key used in put operation 307 * @param val Value used in put operation 308 * @param oldval Old value which will be replaced by `val` may be `NULL` 309 * @param op Arbitrary opaqued data passed to this handler 310 */ 311 typedef iwrc(*IWKV_PUT_HANDLER)(const IWKV_val *key, const IWKV_val *val, IWKV_val *oldval, void *op); 312 313 /** 314 * @brief Store record in database. 315 * @see iwkv_put() 316 */ 317 IW_EXPORT iwrc iwkv_puth(IWDB db, const IWKV_val *key, const IWKV_val *val, 318 iwkv_opflags opflags, IWKV_PUT_HANDLER ph, void *phop); 319 320 /** 321 * @brief Get value for given `key`. 322 * 323 * @note If not matching record found `IWKV_ERROR_NOTFOUND` will be returned. 324 * @note On success a returned value must be freed with `iwkv_val_dispose()` 325 * 326 * @param db Database handler 327 * @param key Key data 328 * @param [out] oval Value associated with `key` or `NULL` 329 */ 330 IW_EXPORT iwrc iwkv_get(IWDB db, const IWKV_val *key, IWKV_val *oval); 331 332 /** 333 * @brief Get value for given `key` and copy it into provided `vbuf` using up to `vbufsz` bytes. 334 * 335 * @param db Database handler 336 * @param key Key data 337 * @param vbuf Pointer to value buffer 338 * @param vbufsz Value buffer size 339 * @param [out] vsz Actual value size 340 */ 341 IW_EXPORT iwrc iwkv_get_copy(IWDB db, const IWKV_val *key, void *vbuf, size_t vbufsz, size_t *vsz); 342 343 /** 344 * @brief Set arbitrary data associated with database. 345 * Database write lock will acquired for this operation. 346 * 347 * @param db Database handler 348 * @param buf Data buffer 349 * @param sz Size of data buffer 350 */ 351 IW_EXPORT iwrc iwkv_db_set_meta(IWDB db, void *buf, size_t sz); 352 353 /** 354 * @brief Get arbitrary data associated with database. 355 * @param db Database handler 356 * @param buf Output buffer 357 * @param sz Size of target buffer 358 * @param [out] rsz Number of bytes read actually 359 */ 360 IW_EXPORT iwrc iwkv_db_get_meta(IWDB db, void *buf, size_t sz, size_t *rsz); 361 362 /** 363 * @brief Remove record identified by `key`. 364 * 365 * Returns `IWKV_ERROR_NOTFOUND` is no matching key found 366 * @param db Database handler 367 * @param key Key data container 368 */ 369 IW_EXPORT iwrc iwkv_del(IWDB db, const IWKV_val *key, iwkv_opflags opflags); 370 371 /** 372 * @brief Destroy key/value data container. 373 * 374 */ 375 IW_EXPORT void iwkv_val_dispose(IWKV_val *kval); 376 377 /** 378 * @brief Dispose data containers for key and value respectively. 379 * 380 * @note This method is shortland of: 381 * @code {.c} 382 * iwkv_kv_dispose(key); 383 * iwkv_kv_dispose(val); 384 * @endcode 385 * 386 * @param key Key data containers 387 * @param val Value data containers 388 */ 389 IW_EXPORT void iwkv_kv_dispose(IWKV_val *key, IWKV_val *val); 390 391 /** 392 * @brief Open database cursor. 393 * 394 * @param db Database handler 395 * @param cur Pointer to an allocated cursor structure to be initialized 396 * @param op Cursor open mode/initial positions flags 397 * @param key Optional key argument, required to point cursor to the given key. 398 */ 399 IW_EXPORT WUR iwrc iwkv_cursor_open(IWDB db, 400 IWKV_cursor *cur, 401 IWKV_cursor_op op, 402 const IWKV_val *key); 403 /** 404 * @brief Move cursor to the next position. 405 * 406 * @param cur Opened cursor object 407 * @param op Cursor position operation 408 */ 409 IW_EXPORT WUR iwrc iwkv_cursor_to(IWKV_cursor cur, IWKV_cursor_op op); 410 411 /** 412 * @brief Move cursor to the next position. 413 * 414 * @param cur Opened cursor object 415 * @param op Cursor position operation 416 * @param key Optional key argument used to move cursor to the given key. 417 */ 418 IW_EXPORT WUR iwrc iwkv_cursor_to_key(IWKV_cursor cur, IWKV_cursor_op op, const IWKV_val *key); 419 420 /** 421 * @brief Get key and value at current cursor position. 422 * @note Data stored in okey/oval containers must be freed with `iwkv_val_dispose()`. 423 * 424 * @param cur Opened cursor object 425 * @param okey Key container to be initialized by key at current position. Can be null. 426 * @param oval Value container to be initialized by value at current position. Can be null. 427 */ 428 IW_EXPORT iwrc iwkv_cursor_get(IWKV_cursor cur, IWKV_val *okey, IWKV_val *oval); 429 430 /** 431 * @brief Get value at current cursor position. 432 * @note Data stored in `oval` container must be freed with `iwkv_val_dispose()`. 433 * @param cur Opened cursor object 434 * @param oval Value holder to be initialized by value at current position 435 */ 436 IW_EXPORT iwrc iwkv_cursor_val(IWKV_cursor cur, IWKV_val *oval); 437 438 /** 439 * @brief Copy value data to the specified buffer at the current cursor position. 440 * @note At most of `bufsz` bytes will be copied into `vbuf`. 441 * 442 * @param cur Opened cursor object 443 * @param vbuf Pointer to value buffer 444 * @param vbufsz Value buffer size 445 * @param [out] vsz Actual value size 446 */ 447 IW_EXPORT iwrc iwkv_cursor_copy_val(IWKV_cursor cur, void *vbuf, size_t vbufsz, size_t *vsz); 448 449 /** 450 * @brief Get key at current cursor position. 451 * @note Data stored in okey container must be freed with `iwkv_val_dispose()`. 452 * 453 * @param cur Opened cursor object 454 * @param oval Key holder to be initialized by key at current position 455 */ 456 IW_EXPORT iwrc iwkv_cursor_key(IWKV_cursor cur, IWKV_val *okey); 457 458 /** 459 * @brief Copy key data to the specified buffer at the current cursor position. 460 * @note At most of `bufsz` bytes will be copied into `kbuf`. 461 * 462 * @param cur Opened cursor object 463 * @param kbuf Pointer to value buffer, can be zero if kbufsz is zero too. 464 * @param kbufsz Key buffer size, cab be zero. 465 * @param [out] ksz Actual key size 466 * @param [out] compound Compound key part value, can be zero. 467 */ 468 IW_EXPORT iwrc iwkv_cursor_copy_key(IWKV_cursor cur, void *kbuf, size_t kbufsz, size_t *ksz, int64_t *compound); 469 470 IW_EXPORT iwrc iwkv_cursor_is_matched_key(IWKV_cursor cur, const IWKV_val *key, bool *ores, int64_t *ocompound); 471 472 /** 473 * @brief Set record value at current cursor position. 474 * @note This is equivalent to `iwkv_put()` operation. 475 * 476 * iwkv_opflags opflags: 477 * - `IWKV_NO_OVERWRITE` If a key is already exists the `IWKV_ERROR_KEY_EXISTS` error will returned. 478 * - `IWKV_SYNC` Flush changes on disk after operation 479 * 480 * @note `iwkv_cursor_set()` adds a new value to sorted values array for existing keys if 481 * database created with `IWDB_DUP_UINT32_VALS`|`IWDB_DUP_UINT64_VALS` flags 482 * 483 * @param cur Opened cursor object 484 * @param val Value holder 485 * @param opflags Update value mode 486 */ 487 IW_EXPORT iwrc iwkv_cursor_set(IWKV_cursor cur, IWKV_val *val, iwkv_opflags opflags); 488 489 IW_EXPORT iwrc iwkv_cursor_seth(IWKV_cursor cur, IWKV_val *val, iwkv_opflags opflags, 490 IWKV_PUT_HANDLER ph, void *phop); 491 492 /** 493 * @brief Remove record value at current cursor position. 494 * @param cur Opened cursor object 495 */ 496 IW_EXPORT iwrc iwkv_cursor_del(IWKV_cursor cur, iwkv_opflags opflags); 497 498 /** 499 * @brief Close cursor object. 500 * @param cur Opened cursor 501 */ 502 IW_EXPORT iwrc iwkv_cursor_close(IWKV_cursor *cur); 503 504 /** 505 * Creates an online database backup image and copies it into the specified `target_file`. 506 * During online backup phase read/write database operations are not 507 * blocked for significant amount of time. Backup finish time is 508 * placed into `ts` as number of milliseconds since epoch. 509 * Online backup guaranties what all records before `ts` timestamp will 510 * be stored in backup image. Later, online backup image can be 511 * opened as ordinary database file. 512 * 513 * @note In order to avoid deadlocks: close all opened database cursors 514 * before calling this method. 515 * 516 * @param iwkv 517 * @param [out] ts Backup completion timestamp 518 * @param target_file backup file path 519 */ 520 IW_EXPORT iwrc iwkv_online_backup(IWKV iwkv, uint64_t *ts, const char *target_file); 521 522 /** 523 * @brief Get database file status info. 524 * @note Database should be in opened state. 525 * 526 * @see IWFS_FILE::state 527 * @param db Database handler 528 * @param [out] out IWFS_FSM_STATE placeholder iwkv file state 529 */ 530 IW_EXPORT iwrc iwkv_state(IWKV iwkv, IWFS_FSM_STATE *out); 531 532 // Do not print random levels of skiplist blocks 533 #define IWKVD_PRINT_NO_LEVEVELS 0x1 534 535 // Print record values 536 #define IWKVD_PRINT_VALS 0x2 537 538 void iwkvd_db(FILE *f, IWDB db, int flags, int plvl); 539 540 IW_EXTERN_C_END 541 542 #endif 543