• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#----------------------------------------------------------------------
2#  Copyright (c) 1999-2001, Digital Creations, Fredericksburg, VA, USA
3#  and Andrew Kuchling. All rights reserved.
4#
5#  Redistribution and use in source and binary forms, with or without
6#  modification, are permitted provided that the following conditions are
7#  met:
8#
9#    o Redistributions of source code must retain the above copyright
10#      notice, this list of conditions, and the disclaimer that follows.
11#
12#    o Redistributions in binary form must reproduce the above copyright
13#      notice, this list of conditions, and the following disclaimer in
14#      the documentation and/or other materials provided with the
15#      distribution.
16#
17#    o Neither the name of Digital Creations nor the names of its
18#      contributors may be used to endorse or promote products derived
19#      from this software without specific prior written permission.
20#
21#  THIS SOFTWARE IS PROVIDED BY DIGITAL CREATIONS AND CONTRIBUTORS *AS
22#  IS* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
23#  TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
24#  PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL DIGITAL
25#  CREATIONS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
26#  INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
27#  BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
28#  OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
29#  ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
30#  TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
31#  USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
32#  DAMAGE.
33#----------------------------------------------------------------------
34
35
36"""Support for Berkeley DB 4.3 through 5.3 with a simple interface.
37
38For the full featured object oriented interface use the bsddb.db module
39instead.  It mirrors the Oracle Berkeley DB C API.
40"""
41
42import sys
43absolute_import = (sys.version_info[0] >= 3)
44
45if (sys.version_info >= (2, 6)) and (sys.version_info < (3, 0)) :
46    import warnings
47    if sys.py3kwarning and (__name__ != 'bsddb3') :
48        warnings.warnpy3k("in 3.x, the bsddb module has been removed; "
49                          "please use the pybsddb project instead",
50                          DeprecationWarning, 2)
51    warnings.filterwarnings("ignore", ".*CObject.*", DeprecationWarning,
52                            "bsddb.__init__")
53
54try:
55    if __name__ == 'bsddb3':
56        # import _pybsddb binary as it should be the more recent version from
57        # a standalone pybsddb addon package than the version included with
58        # python as bsddb._bsddb.
59        if absolute_import :
60            # Because this syntaxis is not valid before Python 2.5
61            exec("from . import _pybsddb")
62        else :
63            import _pybsddb
64        _bsddb = _pybsddb
65        from bsddb3.dbutils import DeadlockWrap as _DeadlockWrap
66    else:
67        import _bsddb
68        from bsddb.dbutils import DeadlockWrap as _DeadlockWrap
69except ImportError:
70    # Remove ourselves from sys.modules
71    import sys
72    del sys.modules[__name__]
73    raise
74
75# bsddb3 calls it db, but provide _db for backwards compatibility
76db = _db = _bsddb
77__version__ = db.__version__
78
79error = db.DBError  # So bsddb.error will mean something...
80
81#----------------------------------------------------------------------
82
83import sys, os
84
85from weakref import ref
86
87if sys.version_info < (2, 6) :
88    import UserDict
89    MutableMapping = UserDict.DictMixin
90else :
91    import collections
92    MutableMapping = collections.MutableMapping
93
94class _iter_mixin(MutableMapping):
95    def _make_iter_cursor(self):
96        cur = _DeadlockWrap(self.db.cursor)
97        key = id(cur)
98        self._cursor_refs[key] = ref(cur, self._gen_cref_cleaner(key))
99        return cur
100
101    def _gen_cref_cleaner(self, key):
102        # use generate the function for the weakref callback here
103        # to ensure that we do not hold a strict reference to cur
104        # in the callback.
105        return lambda ref: self._cursor_refs.pop(key, None)
106
107    def __iter__(self):
108        self._kill_iteration = False
109        self._in_iter += 1
110        try:
111            try:
112                cur = self._make_iter_cursor()
113
114                # FIXME-20031102-greg: race condition.  cursor could
115                # be closed by another thread before this call.
116
117                # since we're only returning keys, we call the cursor
118                # methods with flags=0, dlen=0, dofs=0
119                key = _DeadlockWrap(cur.first, 0,0,0)[0]
120                yield key
121
122                next = getattr(cur, "next")
123                while 1:
124                    try:
125                        key = _DeadlockWrap(next, 0,0,0)[0]
126                        yield key
127                    except _bsddb.DBCursorClosedError:
128                        if self._kill_iteration:
129                            raise RuntimeError('Database changed size '
130                                               'during iteration.')
131                        cur = self._make_iter_cursor()
132                        # FIXME-20031101-greg: race condition.  cursor could
133                        # be closed by another thread before this call.
134                        _DeadlockWrap(cur.set, key,0,0,0)
135                        next = getattr(cur, "next")
136            except _bsddb.DBNotFoundError:
137                pass
138            except _bsddb.DBCursorClosedError:
139                # the database was modified during iteration.  abort.
140                pass
141# When Python 2.4 not supported in bsddb3, we can change this to "finally"
142        except :
143            self._in_iter -= 1
144            raise
145
146        self._in_iter -= 1
147
148    def iteritems(self):
149        if not self.db:
150            return
151        self._kill_iteration = False
152        self._in_iter += 1
153        try:
154            try:
155                cur = self._make_iter_cursor()
156
157                # FIXME-20031102-greg: race condition.  cursor could
158                # be closed by another thread before this call.
159
160                kv = _DeadlockWrap(cur.first)
161                key = kv[0]
162                yield kv
163
164                next = getattr(cur, "next")
165                while 1:
166                    try:
167                        kv = _DeadlockWrap(next)
168                        key = kv[0]
169                        yield kv
170                    except _bsddb.DBCursorClosedError:
171                        if self._kill_iteration:
172                            raise RuntimeError('Database changed size '
173                                               'during iteration.')
174                        cur = self._make_iter_cursor()
175                        # FIXME-20031101-greg: race condition.  cursor could
176                        # be closed by another thread before this call.
177                        _DeadlockWrap(cur.set, key,0,0,0)
178                        next = getattr(cur, "next")
179            except _bsddb.DBNotFoundError:
180                pass
181            except _bsddb.DBCursorClosedError:
182                # the database was modified during iteration.  abort.
183                pass
184# When Python 2.4 not supported in bsddb3, we can change this to "finally"
185        except :
186            self._in_iter -= 1
187            raise
188
189        self._in_iter -= 1
190
191
192class _DBWithCursor(_iter_mixin):
193    """
194    A simple wrapper around DB that makes it look like the bsddbobject in
195    the old module.  It uses a cursor as needed to provide DB traversal.
196    """
197    def __init__(self, db):
198        self.db = db
199        self.db.set_get_returns_none(0)
200
201        # FIXME-20031101-greg: I believe there is still the potential
202        # for deadlocks in a multithreaded environment if someone
203        # attempts to use the any of the cursor interfaces in one
204        # thread while doing a put or delete in another thread.  The
205        # reason is that _checkCursor and _closeCursors are not atomic
206        # operations.  Doing our own locking around self.dbc,
207        # self.saved_dbc_key and self._cursor_refs could prevent this.
208        # TODO: A test case demonstrating the problem needs to be written.
209
210        # self.dbc is a DBCursor object used to implement the
211        # first/next/previous/last/set_location methods.
212        self.dbc = None
213        self.saved_dbc_key = None
214
215        # a collection of all DBCursor objects currently allocated
216        # by the _iter_mixin interface.
217        self._cursor_refs = {}
218        self._in_iter = 0
219        self._kill_iteration = False
220
221    def __del__(self):
222        self.close()
223
224    def _checkCursor(self):
225        if self.dbc is None:
226            self.dbc = _DeadlockWrap(self.db.cursor)
227            if self.saved_dbc_key is not None:
228                _DeadlockWrap(self.dbc.set, self.saved_dbc_key)
229                self.saved_dbc_key = None
230
231    # This method is needed for all non-cursor DB calls to avoid
232    # Berkeley DB deadlocks (due to being opened with DB_INIT_LOCK
233    # and DB_THREAD to be thread safe) when intermixing database
234    # operations that use the cursor internally with those that don't.
235    def _closeCursors(self, save=1):
236        if self.dbc:
237            c = self.dbc
238            self.dbc = None
239            if save:
240                try:
241                    self.saved_dbc_key = _DeadlockWrap(c.current, 0,0,0)[0]
242                except db.DBError:
243                    pass
244            _DeadlockWrap(c.close)
245            del c
246        for cref in self._cursor_refs.values():
247            c = cref()
248            if c is not None:
249                _DeadlockWrap(c.close)
250
251    def _checkOpen(self):
252        if self.db is None:
253            raise error, "BSDDB object has already been closed"
254
255    def isOpen(self):
256        return self.db is not None
257
258    def __len__(self):
259        self._checkOpen()
260        return _DeadlockWrap(lambda: len(self.db))  # len(self.db)
261
262    if sys.version_info >= (2, 6) :
263        def __repr__(self) :
264            if self.isOpen() :
265                return repr(dict(_DeadlockWrap(self.db.items)))
266            return repr(dict())
267
268    def __getitem__(self, key):
269        self._checkOpen()
270        return _DeadlockWrap(lambda: self.db[key])  # self.db[key]
271
272    def __setitem__(self, key, value):
273        self._checkOpen()
274        self._closeCursors()
275        if self._in_iter and key not in self:
276            self._kill_iteration = True
277        def wrapF():
278            self.db[key] = value
279        _DeadlockWrap(wrapF)  # self.db[key] = value
280
281    def __delitem__(self, key):
282        self._checkOpen()
283        self._closeCursors()
284        if self._in_iter and key in self:
285            self._kill_iteration = True
286        def wrapF():
287            del self.db[key]
288        _DeadlockWrap(wrapF)  # del self.db[key]
289
290    def close(self):
291        self._closeCursors(save=0)
292        if self.dbc is not None:
293            _DeadlockWrap(self.dbc.close)
294        v = 0
295        if self.db is not None:
296            v = _DeadlockWrap(self.db.close)
297        self.dbc = None
298        self.db = None
299        return v
300
301    def keys(self):
302        self._checkOpen()
303        return _DeadlockWrap(self.db.keys)
304
305    def has_key(self, key):
306        self._checkOpen()
307        return _DeadlockWrap(self.db.has_key, key)
308
309    def set_location(self, key):
310        self._checkOpen()
311        self._checkCursor()
312        return _DeadlockWrap(self.dbc.set_range, key)
313
314    def next(self):  # Renamed by "2to3"
315        self._checkOpen()
316        self._checkCursor()
317        rv = _DeadlockWrap(getattr(self.dbc, "next"))
318        return rv
319
320    if sys.version_info[0] >= 3 :  # For "2to3" conversion
321        next = __next__
322
323    def previous(self):
324        self._checkOpen()
325        self._checkCursor()
326        rv = _DeadlockWrap(self.dbc.prev)
327        return rv
328
329    def first(self):
330        self._checkOpen()
331        # fix 1725856: don't needlessly try to restore our cursor position
332        self.saved_dbc_key = None
333        self._checkCursor()
334        rv = _DeadlockWrap(self.dbc.first)
335        return rv
336
337    def last(self):
338        self._checkOpen()
339        # fix 1725856: don't needlessly try to restore our cursor position
340        self.saved_dbc_key = None
341        self._checkCursor()
342        rv = _DeadlockWrap(self.dbc.last)
343        return rv
344
345    def sync(self):
346        self._checkOpen()
347        return _DeadlockWrap(self.db.sync)
348
349
350#----------------------------------------------------------------------
351# Compatibility object factory functions
352
353def hashopen(file, flag='c', mode=0666, pgsize=None, ffactor=None, nelem=None,
354            cachesize=None, lorder=None, hflags=0):
355
356    flags = _checkflag(flag, file)
357    e = _openDBEnv(cachesize)
358    d = db.DB(e)
359    d.set_flags(hflags)
360    if pgsize is not None:    d.set_pagesize(pgsize)
361    if lorder is not None:    d.set_lorder(lorder)
362    if ffactor is not None:   d.set_h_ffactor(ffactor)
363    if nelem is not None:     d.set_h_nelem(nelem)
364    d.open(file, db.DB_HASH, flags, mode)
365    return _DBWithCursor(d)
366
367#----------------------------------------------------------------------
368
369def btopen(file, flag='c', mode=0666,
370            btflags=0, cachesize=None, maxkeypage=None, minkeypage=None,
371            pgsize=None, lorder=None):
372
373    flags = _checkflag(flag, file)
374    e = _openDBEnv(cachesize)
375    d = db.DB(e)
376    if pgsize is not None: d.set_pagesize(pgsize)
377    if lorder is not None: d.set_lorder(lorder)
378    d.set_flags(btflags)
379    if minkeypage is not None: d.set_bt_minkey(minkeypage)
380    if maxkeypage is not None: d.set_bt_maxkey(maxkeypage)
381    d.open(file, db.DB_BTREE, flags, mode)
382    return _DBWithCursor(d)
383
384#----------------------------------------------------------------------
385
386
387def rnopen(file, flag='c', mode=0666,
388            rnflags=0, cachesize=None, pgsize=None, lorder=None,
389            rlen=None, delim=None, source=None, pad=None):
390
391    flags = _checkflag(flag, file)
392    e = _openDBEnv(cachesize)
393    d = db.DB(e)
394    if pgsize is not None: d.set_pagesize(pgsize)
395    if lorder is not None: d.set_lorder(lorder)
396    d.set_flags(rnflags)
397    if delim is not None: d.set_re_delim(delim)
398    if rlen is not None: d.set_re_len(rlen)
399    if source is not None: d.set_re_source(source)
400    if pad is not None: d.set_re_pad(pad)
401    d.open(file, db.DB_RECNO, flags, mode)
402    return _DBWithCursor(d)
403
404#----------------------------------------------------------------------
405
406def _openDBEnv(cachesize):
407    e = db.DBEnv()
408    if cachesize is not None:
409        if cachesize >= 20480:
410            e.set_cachesize(0, cachesize)
411        else:
412            raise error, "cachesize must be >= 20480"
413    e.set_lk_detect(db.DB_LOCK_DEFAULT)
414    e.open('.', db.DB_PRIVATE | db.DB_CREATE | db.DB_THREAD | db.DB_INIT_LOCK | db.DB_INIT_MPOOL)
415    return e
416
417def _checkflag(flag, file):
418    if flag == 'r':
419        flags = db.DB_RDONLY
420    elif flag == 'rw':
421        flags = 0
422    elif flag == 'w':
423        flags =  db.DB_CREATE
424    elif flag == 'c':
425        flags =  db.DB_CREATE
426    elif flag == 'n':
427        flags = db.DB_CREATE
428        #flags = db.DB_CREATE | db.DB_TRUNCATE
429        # we used db.DB_TRUNCATE flag for this before but Berkeley DB
430        # 4.2.52 changed to disallowed truncate with txn environments.
431        if file is not None and os.path.isfile(file):
432            os.unlink(file)
433    else:
434        raise error, "flags should be one of 'r', 'w', 'c' or 'n'"
435    return flags | db.DB_THREAD
436
437#----------------------------------------------------------------------
438
439
440# This is a silly little hack that allows apps to continue to use the
441# DB_THREAD flag even on systems without threads without freaking out
442# Berkeley DB.
443#
444# This assumes that if Python was built with thread support then
445# Berkeley DB was too.
446
447try:
448    # 2to3 automatically changes "import thread" to "import _thread"
449    import thread as T
450    del T
451
452except ImportError:
453    db.DB_THREAD = 0
454
455#----------------------------------------------------------------------
456