1 #pragma once 2 #ifndef EJDB2_INTERNAL_H 3 #define EJDB2_INTERNAL_H 4 5 /************************************************************************************************** 6 * EJDB2 7 * 8 * MIT License 9 * 10 * Copyright (c) 2012-2021 Softmotions Ltd <info@softmotions.com> 11 * 12 * Permission is hereby granted, free of charge, to any person obtaining a copy 13 * of this software and associated documentation files (the "Software"), to deal 14 * in the Software without restriction, including without limitation the rights 15 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 16 * copies of the Software, and to permit persons to whom the Software is 17 * furnished to do so, subject to the following conditions: 18 * 19 * The above copyright notice and this permission notice shall be included in all 20 * copies or substantial portions of the Software. 21 * 22 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 23 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 24 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 25 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 26 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 27 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 28 * SOFTWARE. 29 *************************************************************************************************/ 30 31 #include "ejdb2.h" 32 #include "jql.h" 33 #ifdef JB_HTTP 34 #include "jbr.h" 35 #endif 36 #include "jql_internal.h" 37 #include "jbl_internal.h" 38 #include <ejdb2/iowow/iwkv.h> 39 #include <ejdb2/iowow/iwxstr.h> 40 #include <ejdb2/iowow/iwexfile.h> 41 #include <ejdb2/iowow/iwutils.h> 42 #include <ejdb2/iowow/iwstree.h> 43 #include <pthread.h> 44 #include <unistd.h> 45 #include <assert.h> 46 #include <setjmp.h> 47 #include "khash.h" 48 #include "ejdb2cfg.h" 49 50 static_assert(JBNUMBUF_SIZE >= IWFTOA_BUFSIZE, "JBNUMBUF_SIZE >= IWFTOA_BUFSIZE"); 51 52 #define METADB_ID 1 53 #define NUMRECSDB_ID 2 // DB for number of records per index/collection 54 #define KEY_PREFIX_COLLMETA "c." // Full key format: c.<coldbid> 55 #define KEY_PREFIX_IDXMETA "i." // Full key format: i.<coldbid>.<idxdbid> 56 57 #define ENSURE_OPEN(db_) \ 58 if (!(db_) || !((db_)->open)) { \ 59 iwlog_error2("Database is not open"); \ 60 return IW_ERROR_INVALID_STATE; \ 61 } 62 63 #define API_RLOCK(db_, rci_) \ 64 ENSURE_OPEN(db_); \ 65 rci_ = pthread_rwlock_rdlock(&(db_)->rwl); \ 66 if (rci_) return iwrc_set_errno(IW_ERROR_THREADING_ERRNO, rci_) 67 68 #define API_WLOCK(db_, rci_) \ 69 ENSURE_OPEN(db_); \ 70 rci_ = pthread_rwlock_wrlock(&(db_)->rwl); \ 71 if (rci_) return iwrc_set_errno(IW_ERROR_THREADING_ERRNO, rci_) 72 73 #define API_WLOCK2(db_, rci_) \ 74 rci_ = pthread_rwlock_wrlock(&(db_)->rwl); \ 75 if (rci_) return iwrc_set_errno(IW_ERROR_THREADING_ERRNO, rci_) 76 77 #define API_UNLOCK(db_, rci_, rc_) \ 78 rci_ = pthread_rwlock_unlock(&(db_)->rwl); \ 79 if (rci_) IWRC(iwrc_set_errno(IW_ERROR_THREADING_ERRNO, rci_), rc_) 80 81 #define API_COLL_UNLOCK(jbc_, rci_, rc_) \ 82 do { \ 83 rci_ = pthread_rwlock_unlock(&(jbc_)->rwl); \ 84 if (rci_) IWRC(iwrc_set_errno(IW_ERROR_THREADING_ERRNO, rci_), rc_); \ 85 API_UNLOCK((jbc_)->db, rci_, rc_); \ 86 } while (0) 87 88 struct _JBIDX; 89 typedef struct _JBIDX*JBIDX; 90 91 /** Database collection */ 92 typedef struct _JBCOLL { 93 uint32_t dbid; /**< IWKV collection database ID */ 94 const char *name; /**< Collection name */ 95 IWDB cdb; /**< IWKV collection database */ 96 EJDB db; /**< Main database reference */ 97 JBL meta; /**< Collection meta object */ 98 JBIDX idx; /**< First index in chain */ 99 int64_t rnum; /**< Number of records stored in collection */ 100 pthread_rwlock_t rwl; 101 int64_t id_seq; 102 } *JBCOLL; 103 104 /** Database collection index */ 105 struct _JBIDX { 106 struct _JBIDX *next; /**< Next index in chain */ 107 int64_t rnum; /**< Number of records stored in index */ 108 JBCOLL jbc; /**< Owner document collection */ 109 JBL_PTR ptr; /**< Indexed JSON path poiner 0*/ 110 IWDB idb; /**< KV database for this index */ 111 uint32_t dbid; /**< IWKV collection database ID */ 112 ejdb_idx_mode_t mode; /**< Index mode/type mask */ 113 iwdb_flags_t idbf; /**< Index database flags */ 114 }; 115 116 /** Pair: collection name, document id */ 117 struct _JBDOCREF { 118 int64_t id; 119 const char *coll; 120 }; 121 122 // -V:KHASH_MAP_INIT_STR:522 123 KHASH_MAP_INIT_STR(JBCOLLM, JBCOLL) 124 125 struct _EJDB { 126 IWKV iwkv; 127 IWDB metadb; 128 IWDB nrecdb; 129 #ifdef JB_HTTP 130 JBR jbr; 131 #endif 132 khash_t(JBCOLLM) * mcolls; 133 iwkv_openflags oflags; 134 pthread_rwlock_t rwl; /**< Main RWL */ 135 struct _EJDB_OPTS opts; 136 volatile bool open; 137 }; 138 139 struct _JBPHCTX { 140 int64_t id; 141 JBCOLL jbc; 142 JBL jbl; 143 IWKV_val oldval; 144 }; 145 146 struct _JBEXEC; 147 148 typedef iwrc (*JB_SCAN_CONSUMER)( 149 struct _JBEXEC *ctx, IWKV_cursor cur, int64_t id, 150 int64_t *step, bool *matched, iwrc err); 151 152 /** 153 * @brief Index can sorter consumer context 154 */ 155 struct _JBSSC { 156 iwrc rc; /**< RC code used for in `_jb_do_sorting` */ 157 uint32_t *refs; /**< Document references array */ 158 uint32_t refs_asz; /**< Document references array allocated size */ 159 uint32_t refs_num; /**< Document references array elements count */ 160 uint32_t docs_asz; /**< Documents array allocated size */ 161 uint8_t *docs; /**< Documents byte array */ 162 uint32_t docs_npos; /**< Next document offset */ 163 jmp_buf fatal_jmp; 164 IWFS_EXT sof; /**< Sort overflow file */ 165 bool sof_active; 166 }; 167 168 struct _JBMIDX { 169 JBIDX idx; /**< Index matched this filter */ 170 JQP_FILTER *filter; /**< Query filter */ 171 JQP_EXPR *nexpr; /**< Filter node expression */ 172 JQP_EXPR *expr1; /**< Start index expression (optional) */ 173 JQP_EXPR *expr2; /**< End index expression (optional) */ 174 IWKV_cursor_op cursor_init; /**< Initial index cursor position (optional) */ 175 IWKV_cursor_op cursor_step; /**< Next index cursor step */ 176 bool orderby_support; /**< Index supported first order-by clause */ 177 }; 178 179 typedef struct _JBEXEC { 180 EJDB_EXEC *ux; /**< User defined context */ 181 JBCOLL jbc; /**< Collection */ 182 183 int64_t istep; 184 iwrc (*scanner)(struct _JBEXEC *ctx, JB_SCAN_CONSUMER consumer); 185 uint8_t *jblbuf; /**< Buffer used to keep currently processed document */ 186 size_t jblbufsz; /**< Size of jblbuf allocated memory */ 187 bool sorting; /**< Resultset sorting needed */ 188 IWKV_cursor_op cursor_init; /**< Initial index cursor position (optional) */ 189 IWKV_cursor_op cursor_step; /**< Next index cursor step */ 190 struct _JBMIDX midx; /**< Index matching context */ 191 struct _JBSSC ssc; /**< Result set sorting context */ 192 193 // JQL joned nodes cache 194 IWSTREE *proj_joined_nodes_cache; 195 IWPOOL *proj_joined_nodes_pool; 196 } JBEXEC; 197 198 199 typedef uint8_t jb_coll_acquire_t; 200 #define JB_COLL_ACQUIRE_WRITE ((jb_coll_acquire_t) 0x01U) 201 #define JB_COLL_ACQUIRE_EXISTING ((jb_coll_acquire_t) 0x02U) 202 203 // Index selector empiric constants 204 #define JB_IDX_EMPIRIC_MAX_INOP_ARRAY_SIZE 500 205 #define JB_IDX_EMPIRIC_MIN_INOP_ARRAY_SIZE 10 206 #define JB_IDX_EMPIRIC_MAX_INOP_ARRAY_RATIO 200 207 208 void jbi_jbl_fill_ikey(JBIDX idx, JBL jbv, IWKV_val *ikey, char numbuf[static JBNUMBUF_SIZE]); 209 void jbi_jqval_fill_ikey(JBIDX idx, const JQVAL *jqval, IWKV_val *ikey, char numbuf[static JBNUMBUF_SIZE]); 210 void jbi_node_fill_ikey(JBIDX idx, JBL_NODE node, IWKV_val *ikey, char numbuf[static JBNUMBUF_SIZE]); 211 212 iwrc jbi_consumer(struct _JBEXEC *ctx, IWKV_cursor cur, int64_t id, int64_t *step, bool *matched, iwrc err); 213 iwrc jbi_sorter_consumer(struct _JBEXEC *ctx, IWKV_cursor cur, int64_t id, int64_t *step, bool *matched, iwrc err); 214 iwrc jbi_full_scanner(struct _JBEXEC *ctx, JB_SCAN_CONSUMER consumer); 215 iwrc jbi_selection(JBEXEC *ctx); 216 iwrc jbi_pk_scanner(struct _JBEXEC *ctx, JB_SCAN_CONSUMER consumer); 217 iwrc jbi_uniq_scanner(struct _JBEXEC *ctx, JB_SCAN_CONSUMER consumer); 218 iwrc jbi_dup_scanner(struct _JBEXEC *ctx, JB_SCAN_CONSUMER consumer); 219 bool jbi_node_expr_matched(JQP_AUX *aux, JBIDX idx, IWKV_cursor cur, JQP_EXPR *expr, iwrc *rcp); 220 221 iwrc jb_get(EJDB db, const char *coll, int64_t id, jb_coll_acquire_t acm, JBL *jblp); 222 iwrc jb_put(JBCOLL jbc, JBL jbl, int64_t id); 223 iwrc jb_del(JBCOLL jbc, JBL jbl, int64_t id); 224 iwrc jb_cursor_set(JBCOLL jbc, IWKV_cursor cur, int64_t id, JBL jbl); 225 iwrc jb_cursor_del(JBCOLL jbc, IWKV_cursor cur, int64_t id, JBL jbl); 226 227 iwrc jb_collection_join_resolver(int64_t id, const char *coll, JBL *out, JBEXEC *ctx); 228 int jb_proj_node_cache_cmp(const void *v1, const void *v2); 229 void jb_proj_node_kvfree(void *key, void *val); 230 231 #endif 232