1 #pragma once 2 #ifndef IWKV_H 3 #define IWKV_H 4 5 /************************************************************************************************** 6 * IOWOW library 7 * 8 * MIT License 9 * 10 * Copyright (c) 2012-2022 Softmotions Ltd <info@softmotions.com> 11 * 12 * Permission is hereby granted, free of charge, to any person obtaining a copy 13 * of this software and associated documentation files (the "Software"), to deal 14 * in the Software without restriction, including without limitation the rights 15 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 16 * copies of the Software, and to permit persons to whom the Software is 17 * furnished to do so, subject to the following conditions: 18 * 19 * The above copyright notice and this permission notice shall be included in all 20 * copies or substantial portions of the Software. 21 * 22 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 23 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 24 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 25 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 26 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 27 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 28 * SOFTWARE. 29 *************************************************************************************************/ 30 31 /** @file 32 * @brief Persistent key-value storage based on skiplist 33 * datastructure (https://en.wikipedia.org/wiki/Skip_list). 34 * @author Anton Adamansky (adamansky@softmotions.com) 35 * 36 * <strong>Features:<strong> 37 * - Simple design of key value storage 38 * - Lightweight shared/static library: 200Kb 39 * - Support of multiple key-value databases within a single file 40 * - Compound keys supported 41 * - Ultra-fast traversal of database records 42 * - Native support of integer keys 43 * 44 * <strong>Limitations:<strong> 45 * - Maximum iwkv storage file size: 512 GB (0x7fffffff80) 46 * - Total size of a single key+value record must be not greater than 255Mb (0xfffffff) 47 */ 48 49 #include "iowow.h" 50 #include "iwfsmfile.h" 51 #include <stddef.h> 52 #include <stdbool.h> 53 54 IW_EXTERN_C_START 55 56 // Max key + value size: 255Mb 57 #define IWKV_MAX_KVSZ 0xfffffff 58 59 /** 60 * @brief IWKV error codes. 61 */ 62 typedef enum { 63 _IWKV_ERROR_START = (IW_ERROR_START + 5000UL), 64 IWKV_ERROR_NOTFOUND, /**< Key not found (IWKV_ERROR_NOTFOUND) */ 65 IWKV_ERROR_KEY_EXISTS, /**< Key already exists (IWKV_ERROR_KEY_EXISTS) */ 66 IWKV_ERROR_MAXKVSZ, 67 /**< Size of Key+value must be not greater than 0xfffffff bytes 68 (IWKV_ERROR_MAXKVSZ) */ 69 IWKV_ERROR_CORRUPTED, /**< Database file invalid or corrupted (IWKV_ERROR_CORRUPTED) */ 70 IWKV_ERROR_DUP_VALUE_SIZE, 71 /**< Value size is not compatible for insertion into sorted values array 72 (IWKV_ERROR_DUP_VALUE_SIZE) */ 73 IWKV_ERROR_KEY_NUM_VALUE_SIZE, 74 /**< Given key is not compatible to storage as number 75 (IWKV_ERROR_KEY_NUM_VALUE_SIZE) */ 76 IWKV_ERROR_INCOMPATIBLE_DB_MODE, /**< Incorpatible database open mode (IWKV_ERROR_INCOMPATIBLE_DB_MODE) */ 77 IWKV_ERROR_INCOMPATIBLE_DB_FORMAT, 78 /**< Incompatible database format version, please migrate database data 79 (IWKV_ERROR_INCOMPATIBLE_DB_FORMAT) */ 80 IWKV_ERROR_CORRUPTED_WAL_FILE, /**< Corrupted WAL file (IWKV_ERROR_CORRUPTED_WAL_FILE) */ 81 IWKV_ERROR_VALUE_CANNOT_BE_INCREMENTED, 82 /**< Stored value cannot be incremented/descremented 83 (IWKV_ERROR_VALUE_CANNOT_BE_INCREMENTED) */ 84 IWKV_ERROR_WAL_MODE_REQUIRED, 85 /**< Operation requires WAL enabled database. (IWKV_ERROR_WAL_MODE_REQUIRED) 86 */ 87 IWKV_ERROR_BACKUP_IN_PROGRESS, /**< Backup operation in progress. (IWKV_ERROR_BACKUP_IN_PROGRESS) */ 88 _IWKV_ERROR_END, 89 // Internal only 90 _IWKV_RC_KVBLOCK_FULL, 91 _IWKV_RC_REQUIRE_NLEVEL, 92 _IWKV_RC_END, 93 } iwkv_ecode; 94 95 /** Database file open modes. */ 96 typedef uint8_t iwkv_openflags; 97 /** Open storage file in read-only mode */ 98 #define IWKV_RDONLY ((iwkv_openflags) 0x02U) 99 /** Truncate storage file on open */ 100 #define IWKV_TRUNC ((iwkv_openflags) 0x04U) 101 #define IWKV_NO_TRIM_ON_CLOSE ((iwkv_openflags) 0x08U) 102 103 /** Database initialization modes */ 104 typedef uint8_t iwdb_flags_t; 105 106 /** Floating point number keys represented as string (char*) value. */ 107 #define IWDB_REALNUM_KEYS ((iwdb_flags_t) 0x10U) 108 109 /** Variable-length number keys */ 110 #define IWDB_VNUM64_KEYS ((iwdb_flags_t) 0x20U) 111 112 /** 113 * Enable compound database keys. Keys stored in the following format: `<key value prefix><numeric key suffix>` 114 * Allows associate one `key value` with many references represented as VNUM64 (eg.: Non unique table indexes). 115 * @see IWKV_val.compound 116 */ 117 #define IWDB_COMPOUND_KEYS ((iwdb_flags_t) 0x40U) 118 119 /** Record store modes used in `iwkv_put()` and `iwkv_cursor_set()` functions. */ 120 typedef uint8_t iwkv_opflags; 121 122 /** Do not overwrite value for an existing key. 123 `IWKV_ERROR_KEY_EXISTS` will be returned in such cases. */ 124 #define IWKV_NO_OVERWRITE ((iwkv_opflags) 0x01U) 125 126 /** Flush changes on disk after operation */ 127 #define IWKV_SYNC ((iwkv_opflags) 0x04U) 128 129 /** Increment/decrement stored UINT32|UINT64 value by given INT32|INT64 number 130 `IWKV_ERROR_KEY_EXISTS` does not makes sense if this flag set. */ 131 #define IWKV_VAL_INCREMENT ((iwkv_opflags) 0x10U) 132 133 struct _IWKV; 134 typedef struct _IWKV*IWKV; 135 136 struct _IWDB; 137 typedef struct _IWDB*IWDB; 138 139 /** 140 * @brief Write ahead log (WAL) options. 141 */ 142 typedef struct IWKV_WAL_OPTS { 143 bool enabled; /**< WAL enabled */ 144 bool check_crc_on_checkpoint; /**< Check CRC32 sum of data blocks during checkpoint. Default: false */ 145 uint32_t savepoint_timeout_sec; /**< Savepoint timeout seconds. Default: 10 sec */ 146 uint32_t checkpoint_timeout_sec; /**< Checkpoint timeout seconds. Default: 300 sec (5 min); */ 147 size_t wal_buffer_sz; /**< WAL file intermediate buffer size. Default: 8Mb */ 148 uint64_t checkpoint_buffer_sz; /**< Checkpoint buffer size in bytes. Default: 1Gb */ 149 iwrc (*wal_lock_interceptor)(bool, void*); 150 /**< Optional function called 151 - before acquiring 152 - after releasing 153 exclusive database lock by WAL checkpoint thread. 154 In the case of `before lock` first argument will be set to true */ 155 void *wal_lock_interceptor_opaque;/**< Opaque data for `wal_lock_interceptor` */ 156 } IWKV_WAL_OPTS; 157 158 /** 159 * @brief IWKV storage open options. 160 */ 161 typedef struct IWKV_OPTS { 162 const char *path; /**< Path to database file */ 163 uint32_t random_seed; /**< Random seed used for iwu random generator */ 164 /** 165 * Database storage format version. 166 * Leave it as zero for the latest supported format. 167 * Used only for newly created databases, 168 */ 169 int32_t fmt_version; 170 iwkv_openflags oflags; /**< Bitmask of database file open modes */ 171 bool file_lock_fail_fast; /**< Do not wait and raise error if database is locked by another process */ 172 IWKV_WAL_OPTS wal; /**< WAL options */ 173 } IWKV_OPTS; 174 175 /** 176 * @brief Data container for key/value. 177 */ 178 typedef struct IWKV_val { 179 void *data; /**< Data buffer */ 180 size_t size; /**< Data buffer size */ 181 /** Extra key part used for key comparison. 182 * If set to non zero and database is created with `IWDB_COMPOUND_KEYS` mode 183 * keys will behave as compound: `<key value><compound>` consisting of two parts. 184 * `compound` field ignored if db not in `IWDB_COMPOUND_KEYS` mode. 185 */ 186 int64_t compound; 187 } IWKV_val; 188 189 /** 190 * @brief Cursor opaque handler. 191 */ 192 struct _IWKV_cursor; 193 typedef struct _IWKV_cursor*IWKV_cursor; 194 195 /** 196 * @brief Database cursor operations and position flags. 197 */ 198 typedef enum IWKV_cursor_op { 199 IWKV_CURSOR_BEFORE_FIRST = 1, /**< Set cursor to position before first record */ 200 IWKV_CURSOR_AFTER_LAST, /**< Set cursor to position after last record */ 201 IWKV_CURSOR_NEXT, /**< Move cursor to the next record */ 202 IWKV_CURSOR_PREV, /**< Move cursor to the previous record */ 203 IWKV_CURSOR_EQ, /**< Set cursor to the specified key value */ 204 IWKV_CURSOR_GE, /**< Set cursor to the key which greater or equal key specified */ 205 } IWKV_cursor_op; 206 207 /** 208 * @brief Initialize iwkv storage. 209 * @details This method must be called before using of any iwkv public API function. 210 * @note iwkv implicitly initialized by iw_init() 211 */ 212 IW_EXPORT WUR iwrc iwkv_init(void); 213 214 /** 215 * @brief Open iwkv storage. 216 * @code {.c} 217 * IWKV iwkv; 218 * IWKV_OPTS opts = { 219 * .path = "mystore.db" 220 * }; 221 * iwrc rc = iwkv_open(&opts, &iwkv); 222 * @endcode 223 * @note Any opened iwkv storage must be closed by `iwkv_close()` after usage. 224 * @param opts Database open options. 225 * @param [out] iwkvp Pointer to @ref IWKV structure. 226 */ 227 IW_EXPORT WUR iwrc iwkv_open(const IWKV_OPTS *opts, IWKV *iwkvp); 228 229 /** 230 * @brief Get iwkv database handler identified by specified `dbid` number. 231 * @details In the case if no database matched `dbid` 232 * a new database will be created using specified function arguments. 233 * 234 * @note Database handler doesn't require to be explicitly closed or freed. 235 * @note Database `flags` argument must be same for all subsequent 236 * calls after first call for particular database, 237 * otherwise `IWKV_ERROR_INCOMPATIBLE_DB_MODE` will be reported. 238 * 239 * @param iwkv Pointer to @ref IWKV handler 240 * @param dbid Database identifier 241 * @param flags Database initialization flags 242 * @param [out] dbp Pointer to database opaque structure 243 */ 244 IW_EXPORT WUR iwrc iwkv_db(IWKV iwkv, uint32_t dbid, iwdb_flags_t flags, IWDB *dbp); 245 246 /** 247 * @brief Create new database with next available database id. 248 * @see iwrc iwkv_db() 249 * 250 * @param flags Database initialization flags 251 * @param [out] dbidp Database identifier placeholder will be filled with next available id. 252 * @param [out] dbp Pointer to database opaque structure 253 */ 254 IW_EXPORT WUR iwrc iwkv_new_db(IWKV iwkv, iwdb_flags_t dbflg, uint32_t *dbidp, IWDB *dbp); 255 256 /** 257 * @brief Destroy(drop) existing database and cleanup all of its data. 258 * 259 * @param dbp Pointer to database opened. 260 */ 261 IW_EXPORT iwrc iwkv_db_destroy(IWDB *dbp); 262 263 /** 264 * @brief Sync iwkv storage state with disk. 265 * 266 * @note It will cause deadlock if current thread holds opened cursors and WAL is enabled, 267 * use method with caution. 268 * 269 * @param iwkv IWKV handler. 270 * @param flags Sync flags. 271 */ 272 IW_EXPORT iwrc iwkv_sync(IWKV iwkv, iwfs_sync_flags flags); 273 274 /** 275 * @brief Close iwkv storage. 276 * @details Upon successfull call of iwkv_close() 277 * no farther operations on storage or any of its databases are allowed. 278 * 279 * @param iwkvp 280 */ 281 IW_EXPORT iwrc iwkv_close(IWKV *iwkvp); 282 283 /** 284 * @brief Store record in database. 285 * 286 * iwkv_opflags opflags: 287 * - `IWKV_NO_OVERWRITE` If a key is already exists the `IWKV_ERROR_KEY_EXISTS` error will returned. 288 * - `IWKV_SYNC` Flush changes on disk after operation 289 * 290 * @note `iwkv_put()` adds a new value to sorted values array for existing keys if 291 * database created with `IWDB_DUP_UINT32_VALS`|`IWDB_DUP_UINT64_VALS` flags 292 * 293 * @param db Database handler 294 * @param key Key data container 295 * @param val Value data container 296 * @param opflags Put options used 297 */ 298 IW_EXPORT iwrc iwkv_put(IWDB db, const IWKV_val *key, const IWKV_val *val, iwkv_opflags opflags); 299 300 /** 301 * @brief Intercepts old(replaced) value in put operation. 302 * @note If `oldval` is not zero IWKV_PUT_HANDLER responsive for releasing it using iwkv_val_dispose() 303 * @warning Use `IWKV_PUT_HANDLER` with caution: mind deadlocks. 304 * 305 * @param key Key used in put operation 306 * @param val Value used in put operation 307 * @param oldval Old value which will be replaced by `val` may be `NULL` 308 * @param op Arbitrary opaqued data passed to this handler 309 */ 310 typedef iwrc (*IWKV_PUT_HANDLER)(const IWKV_val *key, const IWKV_val *val, IWKV_val *oldval, void *op); 311 312 /** 313 * @brief Store record in database. 314 * @see iwkv_put() 315 */ 316 IW_EXPORT iwrc iwkv_puth( 317 IWDB db, const IWKV_val *key, const IWKV_val *val, 318 iwkv_opflags opflags, IWKV_PUT_HANDLER ph, void *phop); 319 320 /** 321 * @brief Get value for given `key`. 322 * 323 * @note If not matching record found `IWKV_ERROR_NOTFOUND` will be returned. 324 * @note On success a returned value must be freed with `iwkv_val_dispose()` 325 * 326 * @param db Database handler 327 * @param key Key data 328 * @param [out] oval Value associated with `key` or `NULL` 329 */ 330 IW_EXPORT iwrc iwkv_get(IWDB db, const IWKV_val *key, IWKV_val *oval); 331 332 /** 333 * @brief Get value for given `key` and copy it into provided `vbuf` using up to `vbufsz` bytes. 334 * 335 * @param db Database handler 336 * @param key Key data 337 * @param vbuf Pointer to value buffer 338 * @param vbufsz Value buffer size 339 * @param [out] vsz Actual value size 340 */ 341 IW_EXPORT iwrc iwkv_get_copy(IWDB db, const IWKV_val *key, void *vbuf, size_t vbufsz, size_t *vsz); 342 343 /** 344 * @brief Set arbitrary data associated with database. 345 * Database write lock will acquired for this operation. 346 * 347 * @param db Database handler 348 * @param buf Data buffer 349 * @param sz Size of data buffer 350 */ 351 IW_EXPORT iwrc iwkv_db_set_meta(IWDB db, void *buf, size_t sz); 352 353 /** 354 * @brief Get arbitrary data associated with database. 355 * @param db Database handler 356 * @param buf Output buffer 357 * @param sz Size of target buffer 358 * @param [out] rsz Number of bytes read actually 359 */ 360 IW_EXPORT iwrc iwkv_db_get_meta(IWDB db, void *buf, size_t sz, size_t *rsz); 361 362 /** 363 * @brief Remove record identified by `key`. 364 * 365 * Returns `IWKV_ERROR_NOTFOUND` is no matching key found 366 * @param db Database handler 367 * @param key Key data container 368 */ 369 IW_EXPORT iwrc iwkv_del(IWDB db, const IWKV_val *key, iwkv_opflags opflags); 370 371 /** 372 * @brief Destroy key/value data container. 373 * 374 */ 375 IW_EXPORT void iwkv_val_dispose(IWKV_val *kval); 376 377 /** 378 * @brief Dispose data containers for key and value respectively. 379 * 380 * @note This method is shortland of: 381 * @code {.c} 382 * iwkv_kv_dispose(key); 383 * iwkv_kv_dispose(val); 384 * @endcode 385 * 386 * @param key Key data containers 387 * @param val Value data containers 388 */ 389 IW_EXPORT void iwkv_kv_dispose(IWKV_val *key, IWKV_val *val); 390 391 /** 392 * @brief Open database cursor. 393 * 394 * @param db Database handler 395 * @param cur Pointer to an allocated cursor structure to be initialized 396 * @param op Cursor open mode/initial positions flags 397 * @param key Optional key argument, required to point cursor to the given key. 398 */ 399 IW_EXPORT WUR iwrc iwkv_cursor_open( 400 IWDB db, 401 IWKV_cursor *cur, 402 IWKV_cursor_op op, 403 const IWKV_val *key); 404 405 /** 406 * @brief Move cursor to the next position. 407 * 408 * @param cur Opened cursor object 409 * @param op Cursor position operation 410 */ 411 IW_EXPORT WUR iwrc iwkv_cursor_to(IWKV_cursor cur, IWKV_cursor_op op); 412 413 /** 414 * @brief Move cursor to the next position. 415 * 416 * @param cur Opened cursor object 417 * @param op Cursor position operation 418 * @param key Optional key argument used to move cursor to the given key. 419 */ 420 IW_EXPORT WUR iwrc iwkv_cursor_to_key(IWKV_cursor cur, IWKV_cursor_op op, const IWKV_val *key); 421 422 /** 423 * @brief Get key and value at current cursor position. 424 * @note Data stored in okey/oval containers must be freed with `iwkv_val_dispose()`. 425 * 426 * @param cur Opened cursor object 427 * @param okey Key container to be initialized by key at current position. Can be null. 428 * @param oval Value container to be initialized by value at current position. Can be null. 429 */ 430 IW_EXPORT iwrc iwkv_cursor_get(IWKV_cursor cur, IWKV_val *okey, IWKV_val *oval); 431 432 /** 433 * @brief Get value at current cursor position. 434 * @note Data stored in `oval` container must be freed with `iwkv_val_dispose()`. 435 * @param cur Opened cursor object 436 * @param oval Value holder to be initialized by value at current position 437 */ 438 IW_EXPORT iwrc iwkv_cursor_val(IWKV_cursor cur, IWKV_val *oval); 439 440 /** 441 * @brief Copy value data to the specified buffer at the current cursor position. 442 * @note At most of `bufsz` bytes will be copied into `vbuf`. 443 * 444 * @param cur Opened cursor object 445 * @param vbuf Pointer to value buffer 446 * @param vbufsz Value buffer size 447 * @param [out] vsz Actual value size 448 */ 449 IW_EXPORT iwrc iwkv_cursor_copy_val(IWKV_cursor cur, void *vbuf, size_t vbufsz, size_t *vsz); 450 451 /** 452 * @brief Get key at current cursor position. 453 * @note Data stored in okey container must be freed with `iwkv_val_dispose()`. 454 * 455 * @param cur Opened cursor object 456 * @param oval Key holder to be initialized by key at current position 457 */ 458 IW_EXPORT iwrc iwkv_cursor_key(IWKV_cursor cur, IWKV_val *okey); 459 460 /** 461 * @brief Copy key data to the specified buffer at the current cursor position. 462 * @note At most of `bufsz` bytes will be copied into `kbuf`. 463 * 464 * @param cur Opened cursor object 465 * @param kbuf Pointer to value buffer, can be zero if kbufsz is zero too. 466 * @param kbufsz Key buffer size, cab be zero. 467 * @param [out] ksz Actual key size 468 * @param [out] compound Compound key part value, can be zero. 469 */ 470 IW_EXPORT iwrc iwkv_cursor_copy_key(IWKV_cursor cur, void *kbuf, size_t kbufsz, size_t *ksz, int64_t *compound); 471 472 IW_EXPORT iwrc iwkv_cursor_is_matched_key(IWKV_cursor cur, const IWKV_val *key, bool *ores, int64_t *ocompound); 473 474 /** 475 * @brief Set record value at current cursor position. 476 * @note This is equivalent to `iwkv_put()` operation. 477 * 478 * iwkv_opflags opflags: 479 * - `IWKV_NO_OVERWRITE` If a key is already exists the `IWKV_ERROR_KEY_EXISTS` error will returned. 480 * - `IWKV_SYNC` Flush changes on disk after operation 481 * 482 * @note `iwkv_cursor_set()` adds a new value to sorted values array for existing keys if 483 * database created with `IWDB_DUP_UINT32_VALS`|`IWDB_DUP_UINT64_VALS` flags 484 * 485 * @param cur Opened cursor object 486 * @param val Value holder 487 * @param opflags Update value mode 488 */ 489 IW_EXPORT iwrc iwkv_cursor_set(IWKV_cursor cur, IWKV_val *val, iwkv_opflags opflags); 490 491 IW_EXPORT iwrc iwkv_cursor_seth( 492 IWKV_cursor cur, IWKV_val *val, iwkv_opflags opflags, 493 IWKV_PUT_HANDLER ph, void *phop); 494 495 /** 496 * @brief Remove record value at current cursor position. 497 * @param cur Opened cursor object 498 */ 499 IW_EXPORT iwrc iwkv_cursor_del(IWKV_cursor cur, iwkv_opflags opflags); 500 501 /** 502 * @brief Close cursor object. 503 * @param cur Opened cursor 504 */ 505 IW_EXPORT iwrc iwkv_cursor_close(IWKV_cursor *cur); 506 507 /** 508 * Creates an online database backup image and copies it into the specified `target_file`. 509 * During online backup phase read/write database operations are not 510 * blocked for significant amount of time. Backup finish time is 511 * placed into `ts` as number of milliseconds since epoch. 512 * Online backup guaranties what all records before `ts` timestamp will 513 * be stored in backup image. Later, online backup image can be 514 * opened as ordinary database file. 515 * 516 * @note In order to avoid deadlocks: close all opened database cursors 517 * before calling this method. 518 * 519 * @param iwkv 520 * @param [out] ts Backup completion timestamp 521 * @param target_file backup file path 522 */ 523 IW_EXPORT iwrc iwkv_online_backup(IWKV iwkv, uint64_t *ts, const char *target_file); 524 525 /** 526 * @brief Get database file status info. 527 * @note Database should be in opened state. 528 * 529 * @see IWFS_FILE::state 530 * @param db Database handler 531 * @param [out] out IWFS_FSM_STATE placeholder iwkv file state 532 */ 533 IW_EXPORT iwrc iwkv_state(IWKV iwkv, IWFS_FSM_STATE *out); 534 535 // Do not print random levels of skiplist blocks 536 #define IWKVD_PRINT_NO_LEVEVELS 0x1 537 538 // Print record values 539 #define IWKVD_PRINT_VALS 0x2 540 541 void iwkvd_db(FILE *f, IWDB db, int flags, int plvl); 542 543 IW_EXTERN_C_END 544 545 #endif 546