mirror of
https://gitee.com/kekingcn/file-online-preview.git
synced 2026-06-19 12:27:06 +00:00
优化项目结构、优化 maven 结构
This commit is contained in:
chenkailing
@@ -0,0 +1,455 @@
|
||||
#----------------------------------------------------------------------
|
||||
# Copyright (c) 1999-2001, Digital Creations, Fredericksburg, VA, USA
|
||||
# and Andrew Kuchling. All rights reserved.
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are
|
||||
# met:
|
||||
#
|
||||
# o Redistributions of source code must retain the above copyright
|
||||
# notice, this list of conditions, and the disclaimer that follows.
|
||||
#
|
||||
# o Redistributions in binary form must reproduce the above copyright
|
||||
# notice, this list of conditions, and the following disclaimer in
|
||||
# the documentation and/or other materials provided with the
|
||||
# distribution.
|
||||
#
|
||||
# o Neither the name of Digital Creations nor the names of its
|
||||
# contributors may be used to endorse or promote products derived
|
||||
# from this software without specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY DIGITAL CREATIONS AND CONTRIBUTORS *AS
|
||||
# IS* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
|
||||
# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
|
||||
# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL DIGITAL
|
||||
# CREATIONS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
|
||||
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
|
||||
# OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
|
||||
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
|
||||
# TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
|
||||
# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
|
||||
# DAMAGE.
|
||||
#----------------------------------------------------------------------
|
||||
|
||||
|
||||
"""Support for Berkeley DB 4.3 through 5.3 with a simple interface.
|
||||
|
||||
For the full featured object oriented interface use the bsddb.db module
|
||||
instead. It mirrors the Oracle Berkeley DB C API.
|
||||
"""
|
||||
|
||||
import sys
|
||||
absolute_import = (sys.version_info[0] >= 3)
|
||||
|
||||
if (sys.version_info >= (2, 6)) and (sys.version_info < (3, 0)) :
|
||||
import warnings
|
||||
if sys.py3kwarning and (__name__ != 'bsddb3') :
|
||||
warnings.warnpy3k("in 3.x, the bsddb module has been removed; "
|
||||
"please use the pybsddb project instead",
|
||||
DeprecationWarning, 2)
|
||||
warnings.filterwarnings("ignore", ".*CObject.*", DeprecationWarning,
|
||||
"bsddb.__init__")
|
||||
|
||||
try:
|
||||
if __name__ == 'bsddb3':
|
||||
# import _pybsddb binary as it should be the more recent version from
|
||||
# a standalone pybsddb addon package than the version included with
|
||||
# python as bsddb._bsddb.
|
||||
if absolute_import :
|
||||
# Because this syntaxis is not valid before Python 2.5
|
||||
exec("from . import _pybsddb")
|
||||
else :
|
||||
import _pybsddb
|
||||
_bsddb = _pybsddb
|
||||
from bsddb3.dbutils import DeadlockWrap as _DeadlockWrap
|
||||
else:
|
||||
import _bsddb
|
||||
from bsddb.dbutils import DeadlockWrap as _DeadlockWrap
|
||||
except ImportError:
|
||||
# Remove ourselves from sys.modules
|
||||
import sys
|
||||
del sys.modules[__name__]
|
||||
raise
|
||||
|
||||
# bsddb3 calls it db, but provide _db for backwards compatibility
|
||||
db = _db = _bsddb
|
||||
__version__ = db.__version__
|
||||
|
||||
error = db.DBError # So bsddb.error will mean something...
|
||||
|
||||
#----------------------------------------------------------------------
|
||||
|
||||
import sys, os
|
||||
|
||||
from weakref import ref
|
||||
|
||||
if sys.version_info < (2, 6) :
|
||||
import UserDict
|
||||
MutableMapping = UserDict.DictMixin
|
||||
else :
|
||||
import collections
|
||||
MutableMapping = collections.MutableMapping
|
||||
|
||||
class _iter_mixin(MutableMapping):
|
||||
def _make_iter_cursor(self):
|
||||
cur = _DeadlockWrap(self.db.cursor)
|
||||
key = id(cur)
|
||||
self._cursor_refs[key] = ref(cur, self._gen_cref_cleaner(key))
|
||||
return cur
|
||||
|
||||
def _gen_cref_cleaner(self, key):
|
||||
# use generate the function for the weakref callback here
|
||||
# to ensure that we do not hold a strict reference to cur
|
||||
# in the callback.
|
||||
return lambda ref: self._cursor_refs.pop(key, None)
|
||||
|
||||
def __iter__(self):
|
||||
self._kill_iteration = False
|
||||
self._in_iter += 1
|
||||
try:
|
||||
try:
|
||||
cur = self._make_iter_cursor()
|
||||
|
||||
# FIXME-20031102-greg: race condition. cursor could
|
||||
# be closed by another thread before this call.
|
||||
|
||||
# since we're only returning keys, we call the cursor
|
||||
# methods with flags=0, dlen=0, dofs=0
|
||||
key = _DeadlockWrap(cur.first, 0,0,0)[0]
|
||||
yield key
|
||||
|
||||
next = getattr(cur, "next")
|
||||
while 1:
|
||||
try:
|
||||
key = _DeadlockWrap(next, 0,0,0)[0]
|
||||
yield key
|
||||
except _bsddb.DBCursorClosedError:
|
||||
if self._kill_iteration:
|
||||
raise RuntimeError('Database changed size '
|
||||
'during iteration.')
|
||||
cur = self._make_iter_cursor()
|
||||
# FIXME-20031101-greg: race condition. cursor could
|
||||
# be closed by another thread before this call.
|
||||
_DeadlockWrap(cur.set, key,0,0,0)
|
||||
next = getattr(cur, "next")
|
||||
except _bsddb.DBNotFoundError:
|
||||
pass
|
||||
except _bsddb.DBCursorClosedError:
|
||||
# the database was modified during iteration. abort.
|
||||
pass
|
||||
# When Python 2.4 not supported in bsddb3, we can change this to "finally"
|
||||
except :
|
||||
self._in_iter -= 1
|
||||
raise
|
||||
|
||||
self._in_iter -= 1
|
||||
|
||||
def iteritems(self):
|
||||
if not self.db:
|
||||
return
|
||||
self._kill_iteration = False
|
||||
self._in_iter += 1
|
||||
try:
|
||||
try:
|
||||
cur = self._make_iter_cursor()
|
||||
|
||||
# FIXME-20031102-greg: race condition. cursor could
|
||||
# be closed by another thread before this call.
|
||||
|
||||
kv = _DeadlockWrap(cur.first)
|
||||
key = kv[0]
|
||||
yield kv
|
||||
|
||||
next = getattr(cur, "next")
|
||||
while 1:
|
||||
try:
|
||||
kv = _DeadlockWrap(next)
|
||||
key = kv[0]
|
||||
yield kv
|
||||
except _bsddb.DBCursorClosedError:
|
||||
if self._kill_iteration:
|
||||
raise RuntimeError('Database changed size '
|
||||
'during iteration.')
|
||||
cur = self._make_iter_cursor()
|
||||
# FIXME-20031101-greg: race condition. cursor could
|
||||
# be closed by another thread before this call.
|
||||
_DeadlockWrap(cur.set, key,0,0,0)
|
||||
next = getattr(cur, "next")
|
||||
except _bsddb.DBNotFoundError:
|
||||
pass
|
||||
except _bsddb.DBCursorClosedError:
|
||||
# the database was modified during iteration. abort.
|
||||
pass
|
||||
# When Python 2.4 not supported in bsddb3, we can change this to "finally"
|
||||
except :
|
||||
self._in_iter -= 1
|
||||
raise
|
||||
|
||||
self._in_iter -= 1
|
||||
|
||||
|
||||
class _DBWithCursor(_iter_mixin):
|
||||
"""
|
||||
A simple wrapper around DB that makes it look like the bsddbobject in
|
||||
the old module. It uses a cursor as needed to provide DB traversal.
|
||||
"""
|
||||
def __init__(self, db):
|
||||
self.db = db
|
||||
self.db.set_get_returns_none(0)
|
||||
|
||||
# FIXME-20031101-greg: I believe there is still the potential
|
||||
# for deadlocks in a multithreaded environment if someone
|
||||
# attempts to use the any of the cursor interfaces in one
|
||||
# thread while doing a put or delete in another thread. The
|
||||
# reason is that _checkCursor and _closeCursors are not atomic
|
||||
# operations. Doing our own locking around self.dbc,
|
||||
# self.saved_dbc_key and self._cursor_refs could prevent this.
|
||||
# TODO: A test case demonstrating the problem needs to be written.
|
||||
|
||||
# self.dbc is a DBCursor object used to implement the
|
||||
# first/next/previous/last/set_location methods.
|
||||
self.dbc = None
|
||||
self.saved_dbc_key = None
|
||||
|
||||
# a collection of all DBCursor objects currently allocated
|
||||
# by the _iter_mixin interface.
|
||||
self._cursor_refs = {}
|
||||
self._in_iter = 0
|
||||
self._kill_iteration = False
|
||||
|
||||
def __del__(self):
|
||||
self.close()
|
||||
|
||||
def _checkCursor(self):
|
||||
if self.dbc is None:
|
||||
self.dbc = _DeadlockWrap(self.db.cursor)
|
||||
if self.saved_dbc_key is not None:
|
||||
_DeadlockWrap(self.dbc.set, self.saved_dbc_key)
|
||||
self.saved_dbc_key = None
|
||||
|
||||
# This method is needed for all non-cursor DB calls to avoid
|
||||
# Berkeley DB deadlocks (due to being opened with DB_INIT_LOCK
|
||||
# and DB_THREAD to be thread safe) when intermixing database
|
||||
# operations that use the cursor internally with those that don't.
|
||||
def _closeCursors(self, save=1):
|
||||
if self.dbc:
|
||||
c = self.dbc
|
||||
self.dbc = None
|
||||
if save:
|
||||
try:
|
||||
self.saved_dbc_key = _DeadlockWrap(c.current, 0,0,0)[0]
|
||||
except db.DBError:
|
||||
pass
|
||||
_DeadlockWrap(c.close)
|
||||
del c
|
||||
for cref in self._cursor_refs.values():
|
||||
c = cref()
|
||||
if c is not None:
|
||||
_DeadlockWrap(c.close)
|
||||
|
||||
def _checkOpen(self):
|
||||
if self.db is None:
|
||||
raise error, "BSDDB object has already been closed"
|
||||
|
||||
def isOpen(self):
|
||||
return self.db is not None
|
||||
|
||||
def __len__(self):
|
||||
self._checkOpen()
|
||||
return _DeadlockWrap(lambda: len(self.db)) # len(self.db)
|
||||
|
||||
if sys.version_info >= (2, 6) :
|
||||
def __repr__(self) :
|
||||
if self.isOpen() :
|
||||
return repr(dict(_DeadlockWrap(self.db.items)))
|
||||
return repr(dict())
|
||||
|
||||
def __getitem__(self, key):
|
||||
self._checkOpen()
|
||||
return _DeadlockWrap(lambda: self.db[key]) # self.db[key]
|
||||
|
||||
def __setitem__(self, key, value):
|
||||
self._checkOpen()
|
||||
self._closeCursors()
|
||||
if self._in_iter and key not in self:
|
||||
self._kill_iteration = True
|
||||
def wrapF():
|
||||
self.db[key] = value
|
||||
_DeadlockWrap(wrapF) # self.db[key] = value
|
||||
|
||||
def __delitem__(self, key):
|
||||
self._checkOpen()
|
||||
self._closeCursors()
|
||||
if self._in_iter and key in self:
|
||||
self._kill_iteration = True
|
||||
def wrapF():
|
||||
del self.db[key]
|
||||
_DeadlockWrap(wrapF) # del self.db[key]
|
||||
|
||||
def close(self):
|
||||
self._closeCursors(save=0)
|
||||
if self.dbc is not None:
|
||||
_DeadlockWrap(self.dbc.close)
|
||||
v = 0
|
||||
if self.db is not None:
|
||||
v = _DeadlockWrap(self.db.close)
|
||||
self.dbc = None
|
||||
self.db = None
|
||||
return v
|
||||
|
||||
def keys(self):
|
||||
self._checkOpen()
|
||||
return _DeadlockWrap(self.db.keys)
|
||||
|
||||
def has_key(self, key):
|
||||
self._checkOpen()
|
||||
return _DeadlockWrap(self.db.has_key, key)
|
||||
|
||||
def set_location(self, key):
|
||||
self._checkOpen()
|
||||
self._checkCursor()
|
||||
return _DeadlockWrap(self.dbc.set_range, key)
|
||||
|
||||
def next(self): # Renamed by "2to3"
|
||||
self._checkOpen()
|
||||
self._checkCursor()
|
||||
rv = _DeadlockWrap(getattr(self.dbc, "next"))
|
||||
return rv
|
||||
|
||||
if sys.version_info[0] >= 3 : # For "2to3" conversion
|
||||
next = __next__
|
||||
|
||||
def previous(self):
|
||||
self._checkOpen()
|
||||
self._checkCursor()
|
||||
rv = _DeadlockWrap(self.dbc.prev)
|
||||
return rv
|
||||
|
||||
def first(self):
|
||||
self._checkOpen()
|
||||
# fix 1725856: don't needlessly try to restore our cursor position
|
||||
self.saved_dbc_key = None
|
||||
self._checkCursor()
|
||||
rv = _DeadlockWrap(self.dbc.first)
|
||||
return rv
|
||||
|
||||
def last(self):
|
||||
self._checkOpen()
|
||||
# fix 1725856: don't needlessly try to restore our cursor position
|
||||
self.saved_dbc_key = None
|
||||
self._checkCursor()
|
||||
rv = _DeadlockWrap(self.dbc.last)
|
||||
return rv
|
||||
|
||||
def sync(self):
|
||||
self._checkOpen()
|
||||
return _DeadlockWrap(self.db.sync)
|
||||
|
||||
|
||||
#----------------------------------------------------------------------
|
||||
# Compatibility object factory functions
|
||||
|
||||
def hashopen(file, flag='c', mode=0666, pgsize=None, ffactor=None, nelem=None,
|
||||
cachesize=None, lorder=None, hflags=0):
|
||||
|
||||
flags = _checkflag(flag, file)
|
||||
e = _openDBEnv(cachesize)
|
||||
d = db.DB(e)
|
||||
d.set_flags(hflags)
|
||||
if pgsize is not None: d.set_pagesize(pgsize)
|
||||
if lorder is not None: d.set_lorder(lorder)
|
||||
if ffactor is not None: d.set_h_ffactor(ffactor)
|
||||
if nelem is not None: d.set_h_nelem(nelem)
|
||||
d.open(file, db.DB_HASH, flags, mode)
|
||||
return _DBWithCursor(d)
|
||||
|
||||
#----------------------------------------------------------------------
|
||||
|
||||
def btopen(file, flag='c', mode=0666,
|
||||
btflags=0, cachesize=None, maxkeypage=None, minkeypage=None,
|
||||
pgsize=None, lorder=None):
|
||||
|
||||
flags = _checkflag(flag, file)
|
||||
e = _openDBEnv(cachesize)
|
||||
d = db.DB(e)
|
||||
if pgsize is not None: d.set_pagesize(pgsize)
|
||||
if lorder is not None: d.set_lorder(lorder)
|
||||
d.set_flags(btflags)
|
||||
if minkeypage is not None: d.set_bt_minkey(minkeypage)
|
||||
if maxkeypage is not None: d.set_bt_maxkey(maxkeypage)
|
||||
d.open(file, db.DB_BTREE, flags, mode)
|
||||
return _DBWithCursor(d)
|
||||
|
||||
#----------------------------------------------------------------------
|
||||
|
||||
|
||||
def rnopen(file, flag='c', mode=0666,
|
||||
rnflags=0, cachesize=None, pgsize=None, lorder=None,
|
||||
rlen=None, delim=None, source=None, pad=None):
|
||||
|
||||
flags = _checkflag(flag, file)
|
||||
e = _openDBEnv(cachesize)
|
||||
d = db.DB(e)
|
||||
if pgsize is not None: d.set_pagesize(pgsize)
|
||||
if lorder is not None: d.set_lorder(lorder)
|
||||
d.set_flags(rnflags)
|
||||
if delim is not None: d.set_re_delim(delim)
|
||||
if rlen is not None: d.set_re_len(rlen)
|
||||
if source is not None: d.set_re_source(source)
|
||||
if pad is not None: d.set_re_pad(pad)
|
||||
d.open(file, db.DB_RECNO, flags, mode)
|
||||
return _DBWithCursor(d)
|
||||
|
||||
#----------------------------------------------------------------------
|
||||
|
||||
def _openDBEnv(cachesize):
|
||||
e = db.DBEnv()
|
||||
if cachesize is not None:
|
||||
if cachesize >= 20480:
|
||||
e.set_cachesize(0, cachesize)
|
||||
else:
|
||||
raise error, "cachesize must be >= 20480"
|
||||
e.set_lk_detect(db.DB_LOCK_DEFAULT)
|
||||
e.open('.', db.DB_PRIVATE | db.DB_CREATE | db.DB_THREAD | db.DB_INIT_LOCK | db.DB_INIT_MPOOL)
|
||||
return e
|
||||
|
||||
def _checkflag(flag, file):
|
||||
if flag == 'r':
|
||||
flags = db.DB_RDONLY
|
||||
elif flag == 'rw':
|
||||
flags = 0
|
||||
elif flag == 'w':
|
||||
flags = db.DB_CREATE
|
||||
elif flag == 'c':
|
||||
flags = db.DB_CREATE
|
||||
elif flag == 'n':
|
||||
flags = db.DB_CREATE
|
||||
#flags = db.DB_CREATE | db.DB_TRUNCATE
|
||||
# we used db.DB_TRUNCATE flag for this before but Berkeley DB
|
||||
# 4.2.52 changed to disallowed truncate with txn environments.
|
||||
if file is not None and os.path.isfile(file):
|
||||
os.unlink(file)
|
||||
else:
|
||||
raise error, "flags should be one of 'r', 'w', 'c' or 'n'"
|
||||
return flags | db.DB_THREAD
|
||||
|
||||
#----------------------------------------------------------------------
|
||||
|
||||
|
||||
# This is a silly little hack that allows apps to continue to use the
|
||||
# DB_THREAD flag even on systems without threads without freaking out
|
||||
# Berkeley DB.
|
||||
#
|
||||
# This assumes that if Python was built with thread support then
|
||||
# Berkeley DB was too.
|
||||
|
||||
try:
|
||||
# 2to3 automatically changes "import thread" to "import _thread"
|
||||
import thread as T
|
||||
del T
|
||||
|
||||
except ImportError:
|
||||
db.DB_THREAD = 0
|
||||
|
||||
#----------------------------------------------------------------------
|
||||
@@ -0,0 +1,60 @@
|
||||
#----------------------------------------------------------------------
|
||||
# Copyright (c) 1999-2001, Digital Creations, Fredericksburg, VA, USA
|
||||
# and Andrew Kuchling. All rights reserved.
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are
|
||||
# met:
|
||||
#
|
||||
# o Redistributions of source code must retain the above copyright
|
||||
# notice, this list of conditions, and the disclaimer that follows.
|
||||
#
|
||||
# o Redistributions in binary form must reproduce the above copyright
|
||||
# notice, this list of conditions, and the following disclaimer in
|
||||
# the documentation and/or other materials provided with the
|
||||
# distribution.
|
||||
#
|
||||
# o Neither the name of Digital Creations nor the names of its
|
||||
# contributors may be used to endorse or promote products derived
|
||||
# from this software without specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY DIGITAL CREATIONS AND CONTRIBUTORS *AS
|
||||
# IS* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
|
||||
# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
|
||||
# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL DIGITAL
|
||||
# CREATIONS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
|
||||
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
|
||||
# OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
|
||||
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
|
||||
# TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
|
||||
# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
|
||||
# DAMAGE.
|
||||
#----------------------------------------------------------------------
|
||||
|
||||
|
||||
# This module is just a placeholder for possible future expansion, in
|
||||
# case we ever want to augment the stuff in _db in any way. For now
|
||||
# it just simply imports everything from _db.
|
||||
|
||||
import sys
|
||||
absolute_import = (sys.version_info[0] >= 3)
|
||||
|
||||
if not absolute_import :
|
||||
if __name__.startswith('bsddb3.') :
|
||||
# import _pybsddb binary as it should be the more recent version from
|
||||
# a standalone pybsddb addon package than the version included with
|
||||
# python as bsddb._bsddb.
|
||||
from _pybsddb import *
|
||||
from _pybsddb import __version__
|
||||
else:
|
||||
from _bsddb import *
|
||||
from _bsddb import __version__
|
||||
else :
|
||||
# Because this syntaxis is not valid before Python 2.5
|
||||
if __name__.startswith('bsddb3.') :
|
||||
exec("from ._pybsddb import *")
|
||||
exec("from ._pybsddb import __version__")
|
||||
else :
|
||||
exec("from ._bsddb import *")
|
||||
exec("from ._bsddb import __version__")
|
||||
@@ -0,0 +1,266 @@
|
||||
#-------------------------------------------------------------------------
|
||||
# This file contains real Python object wrappers for DB and DBEnv
|
||||
# C "objects" that can be usefully subclassed. The previous SWIG
|
||||
# based interface allowed this thanks to SWIG's shadow classes.
|
||||
# -- Gregory P. Smith
|
||||
#-------------------------------------------------------------------------
|
||||
#
|
||||
# (C) Copyright 2001 Autonomous Zone Industries
|
||||
#
|
||||
# License: This is free software. You may use this software for any
|
||||
# purpose including modification/redistribution, so long as
|
||||
# this header remains intact and that you do not claim any
|
||||
# rights of ownership or authorship of this software. This
|
||||
# software has been tested, but no warranty is expressed or
|
||||
# implied.
|
||||
#
|
||||
|
||||
#
|
||||
# TODO it would be *really nice* to have an automatic shadow class populator
|
||||
# so that new methods don't need to be added here manually after being
|
||||
# added to _bsddb.c.
|
||||
#
|
||||
|
||||
import sys
|
||||
absolute_import = (sys.version_info[0] >= 3)
|
||||
if absolute_import :
|
||||
# Because this syntaxis is not valid before Python 2.5
|
||||
exec("from . import db")
|
||||
else :
|
||||
import db
|
||||
|
||||
if sys.version_info < (2, 6) :
|
||||
from UserDict import DictMixin as MutableMapping
|
||||
else :
|
||||
import collections
|
||||
MutableMapping = collections.MutableMapping
|
||||
|
||||
class DBEnv:
|
||||
def __init__(self, *args, **kwargs):
|
||||
self._cobj = db.DBEnv(*args, **kwargs)
|
||||
|
||||
def close(self, *args, **kwargs):
|
||||
return self._cobj.close(*args, **kwargs)
|
||||
def open(self, *args, **kwargs):
|
||||
return self._cobj.open(*args, **kwargs)
|
||||
def remove(self, *args, **kwargs):
|
||||
return self._cobj.remove(*args, **kwargs)
|
||||
def set_shm_key(self, *args, **kwargs):
|
||||
return self._cobj.set_shm_key(*args, **kwargs)
|
||||
def set_cachesize(self, *args, **kwargs):
|
||||
return self._cobj.set_cachesize(*args, **kwargs)
|
||||
def set_data_dir(self, *args, **kwargs):
|
||||
return self._cobj.set_data_dir(*args, **kwargs)
|
||||
def set_flags(self, *args, **kwargs):
|
||||
return self._cobj.set_flags(*args, **kwargs)
|
||||
def set_lg_bsize(self, *args, **kwargs):
|
||||
return self._cobj.set_lg_bsize(*args, **kwargs)
|
||||
def set_lg_dir(self, *args, **kwargs):
|
||||
return self._cobj.set_lg_dir(*args, **kwargs)
|
||||
def set_lg_max(self, *args, **kwargs):
|
||||
return self._cobj.set_lg_max(*args, **kwargs)
|
||||
def set_lk_detect(self, *args, **kwargs):
|
||||
return self._cobj.set_lk_detect(*args, **kwargs)
|
||||
if db.version() < (4,5):
|
||||
def set_lk_max(self, *args, **kwargs):
|
||||
return self._cobj.set_lk_max(*args, **kwargs)
|
||||
def set_lk_max_locks(self, *args, **kwargs):
|
||||
return self._cobj.set_lk_max_locks(*args, **kwargs)
|
||||
def set_lk_max_lockers(self, *args, **kwargs):
|
||||
return self._cobj.set_lk_max_lockers(*args, **kwargs)
|
||||
def set_lk_max_objects(self, *args, **kwargs):
|
||||
return self._cobj.set_lk_max_objects(*args, **kwargs)
|
||||
def set_mp_mmapsize(self, *args, **kwargs):
|
||||
return self._cobj.set_mp_mmapsize(*args, **kwargs)
|
||||
def set_timeout(self, *args, **kwargs):
|
||||
return self._cobj.set_timeout(*args, **kwargs)
|
||||
def set_tmp_dir(self, *args, **kwargs):
|
||||
return self._cobj.set_tmp_dir(*args, **kwargs)
|
||||
def txn_begin(self, *args, **kwargs):
|
||||
return self._cobj.txn_begin(*args, **kwargs)
|
||||
def txn_checkpoint(self, *args, **kwargs):
|
||||
return self._cobj.txn_checkpoint(*args, **kwargs)
|
||||
def txn_stat(self, *args, **kwargs):
|
||||
return self._cobj.txn_stat(*args, **kwargs)
|
||||
def set_tx_max(self, *args, **kwargs):
|
||||
return self._cobj.set_tx_max(*args, **kwargs)
|
||||
def set_tx_timestamp(self, *args, **kwargs):
|
||||
return self._cobj.set_tx_timestamp(*args, **kwargs)
|
||||
def lock_detect(self, *args, **kwargs):
|
||||
return self._cobj.lock_detect(*args, **kwargs)
|
||||
def lock_get(self, *args, **kwargs):
|
||||
return self._cobj.lock_get(*args, **kwargs)
|
||||
def lock_id(self, *args, **kwargs):
|
||||
return self._cobj.lock_id(*args, **kwargs)
|
||||
def lock_put(self, *args, **kwargs):
|
||||
return self._cobj.lock_put(*args, **kwargs)
|
||||
def lock_stat(self, *args, **kwargs):
|
||||
return self._cobj.lock_stat(*args, **kwargs)
|
||||
def log_archive(self, *args, **kwargs):
|
||||
return self._cobj.log_archive(*args, **kwargs)
|
||||
|
||||
def set_get_returns_none(self, *args, **kwargs):
|
||||
return self._cobj.set_get_returns_none(*args, **kwargs)
|
||||
|
||||
def log_stat(self, *args, **kwargs):
|
||||
return self._cobj.log_stat(*args, **kwargs)
|
||||
|
||||
def dbremove(self, *args, **kwargs):
|
||||
return self._cobj.dbremove(*args, **kwargs)
|
||||
def dbrename(self, *args, **kwargs):
|
||||
return self._cobj.dbrename(*args, **kwargs)
|
||||
def set_encrypt(self, *args, **kwargs):
|
||||
return self._cobj.set_encrypt(*args, **kwargs)
|
||||
|
||||
if db.version() >= (4,4):
|
||||
def fileid_reset(self, *args, **kwargs):
|
||||
return self._cobj.fileid_reset(*args, **kwargs)
|
||||
|
||||
def lsn_reset(self, *args, **kwargs):
|
||||
return self._cobj.lsn_reset(*args, **kwargs)
|
||||
|
||||
|
||||
class DB(MutableMapping):
|
||||
def __init__(self, dbenv, *args, **kwargs):
|
||||
# give it the proper DBEnv C object that its expecting
|
||||
self._cobj = db.DB(*((dbenv._cobj,) + args), **kwargs)
|
||||
|
||||
# TODO are there other dict methods that need to be overridden?
|
||||
def __len__(self):
|
||||
return len(self._cobj)
|
||||
def __getitem__(self, arg):
|
||||
return self._cobj[arg]
|
||||
def __setitem__(self, key, value):
|
||||
self._cobj[key] = value
|
||||
def __delitem__(self, arg):
|
||||
del self._cobj[arg]
|
||||
|
||||
if sys.version_info >= (2, 6) :
|
||||
def __iter__(self) :
|
||||
return self._cobj.__iter__()
|
||||
|
||||
def append(self, *args, **kwargs):
|
||||
return self._cobj.append(*args, **kwargs)
|
||||
def associate(self, *args, **kwargs):
|
||||
return self._cobj.associate(*args, **kwargs)
|
||||
def close(self, *args, **kwargs):
|
||||
return self._cobj.close(*args, **kwargs)
|
||||
def consume(self, *args, **kwargs):
|
||||
return self._cobj.consume(*args, **kwargs)
|
||||
def consume_wait(self, *args, **kwargs):
|
||||
return self._cobj.consume_wait(*args, **kwargs)
|
||||
def cursor(self, *args, **kwargs):
|
||||
return self._cobj.cursor(*args, **kwargs)
|
||||
def delete(self, *args, **kwargs):
|
||||
return self._cobj.delete(*args, **kwargs)
|
||||
def fd(self, *args, **kwargs):
|
||||
return self._cobj.fd(*args, **kwargs)
|
||||
def get(self, *args, **kwargs):
|
||||
return self._cobj.get(*args, **kwargs)
|
||||
def pget(self, *args, **kwargs):
|
||||
return self._cobj.pget(*args, **kwargs)
|
||||
def get_both(self, *args, **kwargs):
|
||||
return self._cobj.get_both(*args, **kwargs)
|
||||
def get_byteswapped(self, *args, **kwargs):
|
||||
return self._cobj.get_byteswapped(*args, **kwargs)
|
||||
def get_size(self, *args, **kwargs):
|
||||
return self._cobj.get_size(*args, **kwargs)
|
||||
def get_type(self, *args, **kwargs):
|
||||
return self._cobj.get_type(*args, **kwargs)
|
||||
def join(self, *args, **kwargs):
|
||||
return self._cobj.join(*args, **kwargs)
|
||||
def key_range(self, *args, **kwargs):
|
||||
return self._cobj.key_range(*args, **kwargs)
|
||||
def has_key(self, *args, **kwargs):
|
||||
return self._cobj.has_key(*args, **kwargs)
|
||||
def items(self, *args, **kwargs):
|
||||
return self._cobj.items(*args, **kwargs)
|
||||
def keys(self, *args, **kwargs):
|
||||
return self._cobj.keys(*args, **kwargs)
|
||||
def open(self, *args, **kwargs):
|
||||
return self._cobj.open(*args, **kwargs)
|
||||
def put(self, *args, **kwargs):
|
||||
return self._cobj.put(*args, **kwargs)
|
||||
def remove(self, *args, **kwargs):
|
||||
return self._cobj.remove(*args, **kwargs)
|
||||
def rename(self, *args, **kwargs):
|
||||
return self._cobj.rename(*args, **kwargs)
|
||||
def set_bt_minkey(self, *args, **kwargs):
|
||||
return self._cobj.set_bt_minkey(*args, **kwargs)
|
||||
def set_bt_compare(self, *args, **kwargs):
|
||||
return self._cobj.set_bt_compare(*args, **kwargs)
|
||||
def set_cachesize(self, *args, **kwargs):
|
||||
return self._cobj.set_cachesize(*args, **kwargs)
|
||||
def set_dup_compare(self, *args, **kwargs) :
|
||||
return self._cobj.set_dup_compare(*args, **kwargs)
|
||||
def set_flags(self, *args, **kwargs):
|
||||
return self._cobj.set_flags(*args, **kwargs)
|
||||
def set_h_ffactor(self, *args, **kwargs):
|
||||
return self._cobj.set_h_ffactor(*args, **kwargs)
|
||||
def set_h_nelem(self, *args, **kwargs):
|
||||
return self._cobj.set_h_nelem(*args, **kwargs)
|
||||
def set_lorder(self, *args, **kwargs):
|
||||
return self._cobj.set_lorder(*args, **kwargs)
|
||||
def set_pagesize(self, *args, **kwargs):
|
||||
return self._cobj.set_pagesize(*args, **kwargs)
|
||||
def set_re_delim(self, *args, **kwargs):
|
||||
return self._cobj.set_re_delim(*args, **kwargs)
|
||||
def set_re_len(self, *args, **kwargs):
|
||||
return self._cobj.set_re_len(*args, **kwargs)
|
||||
def set_re_pad(self, *args, **kwargs):
|
||||
return self._cobj.set_re_pad(*args, **kwargs)
|
||||
def set_re_source(self, *args, **kwargs):
|
||||
return self._cobj.set_re_source(*args, **kwargs)
|
||||
def set_q_extentsize(self, *args, **kwargs):
|
||||
return self._cobj.set_q_extentsize(*args, **kwargs)
|
||||
def stat(self, *args, **kwargs):
|
||||
return self._cobj.stat(*args, **kwargs)
|
||||
def sync(self, *args, **kwargs):
|
||||
return self._cobj.sync(*args, **kwargs)
|
||||
def type(self, *args, **kwargs):
|
||||
return self._cobj.type(*args, **kwargs)
|
||||
def upgrade(self, *args, **kwargs):
|
||||
return self._cobj.upgrade(*args, **kwargs)
|
||||
def values(self, *args, **kwargs):
|
||||
return self._cobj.values(*args, **kwargs)
|
||||
def verify(self, *args, **kwargs):
|
||||
return self._cobj.verify(*args, **kwargs)
|
||||
def set_get_returns_none(self, *args, **kwargs):
|
||||
return self._cobj.set_get_returns_none(*args, **kwargs)
|
||||
|
||||
def set_encrypt(self, *args, **kwargs):
|
||||
return self._cobj.set_encrypt(*args, **kwargs)
|
||||
|
||||
|
||||
class DBSequence:
|
||||
def __init__(self, *args, **kwargs):
|
||||
self._cobj = db.DBSequence(*args, **kwargs)
|
||||
|
||||
def close(self, *args, **kwargs):
|
||||
return self._cobj.close(*args, **kwargs)
|
||||
def get(self, *args, **kwargs):
|
||||
return self._cobj.get(*args, **kwargs)
|
||||
def get_dbp(self, *args, **kwargs):
|
||||
return self._cobj.get_dbp(*args, **kwargs)
|
||||
def get_key(self, *args, **kwargs):
|
||||
return self._cobj.get_key(*args, **kwargs)
|
||||
def init_value(self, *args, **kwargs):
|
||||
return self._cobj.init_value(*args, **kwargs)
|
||||
def open(self, *args, **kwargs):
|
||||
return self._cobj.open(*args, **kwargs)
|
||||
def remove(self, *args, **kwargs):
|
||||
return self._cobj.remove(*args, **kwargs)
|
||||
def stat(self, *args, **kwargs):
|
||||
return self._cobj.stat(*args, **kwargs)
|
||||
def set_cachesize(self, *args, **kwargs):
|
||||
return self._cobj.set_cachesize(*args, **kwargs)
|
||||
def set_flags(self, *args, **kwargs):
|
||||
return self._cobj.set_flags(*args, **kwargs)
|
||||
def set_range(self, *args, **kwargs):
|
||||
return self._cobj.set_range(*args, **kwargs)
|
||||
def get_cachesize(self, *args, **kwargs):
|
||||
return self._cobj.get_cachesize(*args, **kwargs)
|
||||
def get_flags(self, *args, **kwargs):
|
||||
return self._cobj.get_flags(*args, **kwargs)
|
||||
def get_range(self, *args, **kwargs):
|
||||
return self._cobj.get_range(*args, **kwargs)
|
||||
@@ -0,0 +1,190 @@
|
||||
|
||||
"""
|
||||
File-like objects that read from or write to a bsddb record.
|
||||
|
||||
This implements (nearly) all stdio methods.
|
||||
|
||||
f = DBRecIO(db, key, txn=None)
|
||||
f.close() # explicitly release resources held
|
||||
flag = f.isatty() # always false
|
||||
pos = f.tell() # get current position
|
||||
f.seek(pos) # set current position
|
||||
f.seek(pos, mode) # mode 0: absolute; 1: relative; 2: relative to EOF
|
||||
buf = f.read() # read until EOF
|
||||
buf = f.read(n) # read up to n bytes
|
||||
f.truncate([size]) # truncate file at to at most size (default: current pos)
|
||||
f.write(buf) # write at current position
|
||||
f.writelines(list) # for line in list: f.write(line)
|
||||
|
||||
Notes:
|
||||
- fileno() is left unimplemented so that code which uses it triggers
|
||||
an exception early.
|
||||
- There's a simple test set (see end of this file) - not yet updated
|
||||
for DBRecIO.
|
||||
- readline() is not implemented yet.
|
||||
|
||||
|
||||
From:
|
||||
Itamar Shtull-Trauring <itamar@maxnm.com>
|
||||
"""
|
||||
|
||||
import errno
|
||||
import string
|
||||
|
||||
class DBRecIO:
|
||||
def __init__(self, db, key, txn=None):
|
||||
self.db = db
|
||||
self.key = key
|
||||
self.txn = txn
|
||||
self.len = None
|
||||
self.pos = 0
|
||||
self.closed = 0
|
||||
self.softspace = 0
|
||||
|
||||
def close(self):
|
||||
if not self.closed:
|
||||
self.closed = 1
|
||||
del self.db, self.txn
|
||||
|
||||
def isatty(self):
|
||||
if self.closed:
|
||||
raise ValueError, "I/O operation on closed file"
|
||||
return 0
|
||||
|
||||
def seek(self, pos, mode = 0):
|
||||
if self.closed:
|
||||
raise ValueError, "I/O operation on closed file"
|
||||
if mode == 1:
|
||||
pos = pos + self.pos
|
||||
elif mode == 2:
|
||||
pos = pos + self.len
|
||||
self.pos = max(0, pos)
|
||||
|
||||
def tell(self):
|
||||
if self.closed:
|
||||
raise ValueError, "I/O operation on closed file"
|
||||
return self.pos
|
||||
|
||||
def read(self, n = -1):
|
||||
if self.closed:
|
||||
raise ValueError, "I/O operation on closed file"
|
||||
if n < 0:
|
||||
newpos = self.len
|
||||
else:
|
||||
newpos = min(self.pos+n, self.len)
|
||||
|
||||
dlen = newpos - self.pos
|
||||
|
||||
r = self.db.get(self.key, txn=self.txn, dlen=dlen, doff=self.pos)
|
||||
self.pos = newpos
|
||||
return r
|
||||
|
||||
__fixme = """
|
||||
def readline(self, length=None):
|
||||
if self.closed:
|
||||
raise ValueError, "I/O operation on closed file"
|
||||
if self.buflist:
|
||||
self.buf = self.buf + string.joinfields(self.buflist, '')
|
||||
self.buflist = []
|
||||
i = string.find(self.buf, '\n', self.pos)
|
||||
if i < 0:
|
||||
newpos = self.len
|
||||
else:
|
||||
newpos = i+1
|
||||
if length is not None:
|
||||
if self.pos + length < newpos:
|
||||
newpos = self.pos + length
|
||||
r = self.buf[self.pos:newpos]
|
||||
self.pos = newpos
|
||||
return r
|
||||
|
||||
def readlines(self, sizehint = 0):
|
||||
total = 0
|
||||
lines = []
|
||||
line = self.readline()
|
||||
while line:
|
||||
lines.append(line)
|
||||
total += len(line)
|
||||
if 0 < sizehint <= total:
|
||||
break
|
||||
line = self.readline()
|
||||
return lines
|
||||
"""
|
||||
|
||||
def truncate(self, size=None):
|
||||
if self.closed:
|
||||
raise ValueError, "I/O operation on closed file"
|
||||
if size is None:
|
||||
size = self.pos
|
||||
elif size < 0:
|
||||
raise IOError(errno.EINVAL,
|
||||
"Negative size not allowed")
|
||||
elif size < self.pos:
|
||||
self.pos = size
|
||||
self.db.put(self.key, "", txn=self.txn, dlen=self.len-size, doff=size)
|
||||
|
||||
def write(self, s):
|
||||
if self.closed:
|
||||
raise ValueError, "I/O operation on closed file"
|
||||
if not s: return
|
||||
if self.pos > self.len:
|
||||
self.buflist.append('\0'*(self.pos - self.len))
|
||||
self.len = self.pos
|
||||
newpos = self.pos + len(s)
|
||||
self.db.put(self.key, s, txn=self.txn, dlen=len(s), doff=self.pos)
|
||||
self.pos = newpos
|
||||
|
||||
def writelines(self, list):
|
||||
self.write(string.joinfields(list, ''))
|
||||
|
||||
def flush(self):
|
||||
if self.closed:
|
||||
raise ValueError, "I/O operation on closed file"
|
||||
|
||||
|
||||
"""
|
||||
# A little test suite
|
||||
|
||||
def _test():
|
||||
import sys
|
||||
if sys.argv[1:]:
|
||||
file = sys.argv[1]
|
||||
else:
|
||||
file = '/etc/passwd'
|
||||
lines = open(file, 'r').readlines()
|
||||
text = open(file, 'r').read()
|
||||
f = StringIO()
|
||||
for line in lines[:-2]:
|
||||
f.write(line)
|
||||
f.writelines(lines[-2:])
|
||||
if f.getvalue() != text:
|
||||
raise RuntimeError, 'write failed'
|
||||
length = f.tell()
|
||||
print 'File length =', length
|
||||
f.seek(len(lines[0]))
|
||||
f.write(lines[1])
|
||||
f.seek(0)
|
||||
print 'First line =', repr(f.readline())
|
||||
here = f.tell()
|
||||
line = f.readline()
|
||||
print 'Second line =', repr(line)
|
||||
f.seek(-len(line), 1)
|
||||
line2 = f.read(len(line))
|
||||
if line != line2:
|
||||
raise RuntimeError, 'bad result after seek back'
|
||||
f.seek(len(line2), 1)
|
||||
list = f.readlines()
|
||||
line = list[-1]
|
||||
f.seek(f.tell() - len(line))
|
||||
line2 = f.read()
|
||||
if line != line2:
|
||||
raise RuntimeError, 'bad result after seek back from EOF'
|
||||
print 'Read', len(list), 'more lines'
|
||||
print 'File length =', f.tell()
|
||||
if f.tell() != length:
|
||||
raise RuntimeError, 'bad length'
|
||||
f.close()
|
||||
|
||||
if __name__ == '__main__':
|
||||
_test()
|
||||
"""
|
||||
@@ -0,0 +1,382 @@
|
||||
#!/usr/bin/env python
|
||||
#------------------------------------------------------------------------
|
||||
# Copyright (c) 1997-2001 by Total Control Software
|
||||
# All Rights Reserved
|
||||
#------------------------------------------------------------------------
|
||||
#
|
||||
# Module Name: dbShelve.py
|
||||
#
|
||||
# Description: A reimplementation of the standard shelve.py that
|
||||
# forces the use of cPickle, and DB.
|
||||
#
|
||||
# Creation Date: 11/3/97 3:39:04PM
|
||||
#
|
||||
# License: This is free software. You may use this software for any
|
||||
# purpose including modification/redistribution, so long as
|
||||
# this header remains intact and that you do not claim any
|
||||
# rights of ownership or authorship of this software. This
|
||||
# software has been tested, but no warranty is expressed or
|
||||
# implied.
|
||||
#
|
||||
# 13-Dec-2000: Updated to be used with the new bsddb3 package.
|
||||
# Added DBShelfCursor class.
|
||||
#
|
||||
#------------------------------------------------------------------------
|
||||
|
||||
"""Manage shelves of pickled objects using bsddb database files for the
|
||||
storage.
|
||||
"""
|
||||
|
||||
#------------------------------------------------------------------------
|
||||
|
||||
import sys
|
||||
absolute_import = (sys.version_info[0] >= 3)
|
||||
if absolute_import :
|
||||
# Because this syntaxis is not valid before Python 2.5
|
||||
exec("from . import db")
|
||||
else :
|
||||
import db
|
||||
|
||||
if sys.version_info[0] >= 3 :
|
||||
import cPickle # Will be converted to "pickle" by "2to3"
|
||||
else :
|
||||
if sys.version_info < (2, 6) :
|
||||
import cPickle
|
||||
else :
|
||||
# When we drop support for python 2.4
|
||||
# we could use: (in 2.5 we need a __future__ statement)
|
||||
#
|
||||
# with warnings.catch_warnings():
|
||||
# warnings.filterwarnings(...)
|
||||
# ...
|
||||
#
|
||||
# We can not use "with" as is, because it would be invalid syntax
|
||||
# in python 2.4 and (with no __future__) 2.5.
|
||||
# Here we simulate "with" following PEP 343 :
|
||||
import warnings
|
||||
w = warnings.catch_warnings()
|
||||
w.__enter__()
|
||||
try :
|
||||
warnings.filterwarnings('ignore',
|
||||
message='the cPickle module has been removed in Python 3.0',
|
||||
category=DeprecationWarning)
|
||||
import cPickle
|
||||
finally :
|
||||
w.__exit__()
|
||||
del w
|
||||
|
||||
HIGHEST_PROTOCOL = cPickle.HIGHEST_PROTOCOL
|
||||
def _dumps(object, protocol):
|
||||
return cPickle.dumps(object, protocol=protocol)
|
||||
|
||||
if sys.version_info < (2, 6) :
|
||||
from UserDict import DictMixin as MutableMapping
|
||||
else :
|
||||
import collections
|
||||
MutableMapping = collections.MutableMapping
|
||||
|
||||
#------------------------------------------------------------------------
|
||||
|
||||
|
||||
def open(filename, flags=db.DB_CREATE, mode=0660, filetype=db.DB_HASH,
|
||||
dbenv=None, dbname=None):
|
||||
"""
|
||||
A simple factory function for compatibility with the standard
|
||||
shleve.py module. It can be used like this, where key is a string
|
||||
and data is a pickleable object:
|
||||
|
||||
from bsddb import dbshelve
|
||||
db = dbshelve.open(filename)
|
||||
|
||||
db[key] = data
|
||||
|
||||
db.close()
|
||||
"""
|
||||
if type(flags) == type(''):
|
||||
sflag = flags
|
||||
if sflag == 'r':
|
||||
flags = db.DB_RDONLY
|
||||
elif sflag == 'rw':
|
||||
flags = 0
|
||||
elif sflag == 'w':
|
||||
flags = db.DB_CREATE
|
||||
elif sflag == 'c':
|
||||
flags = db.DB_CREATE
|
||||
elif sflag == 'n':
|
||||
flags = db.DB_TRUNCATE | db.DB_CREATE
|
||||
else:
|
||||
raise db.DBError, "flags should be one of 'r', 'w', 'c' or 'n' or use the bsddb.db.DB_* flags"
|
||||
|
||||
d = DBShelf(dbenv)
|
||||
d.open(filename, dbname, filetype, flags, mode)
|
||||
return d
|
||||
|
||||
#---------------------------------------------------------------------------
|
||||
|
||||
class DBShelveError(db.DBError): pass
|
||||
|
||||
|
||||
class DBShelf(MutableMapping):
|
||||
"""A shelf to hold pickled objects, built upon a bsddb DB object. It
|
||||
automatically pickles/unpickles data objects going to/from the DB.
|
||||
"""
|
||||
def __init__(self, dbenv=None):
|
||||
self.db = db.DB(dbenv)
|
||||
self._closed = True
|
||||
if HIGHEST_PROTOCOL:
|
||||
self.protocol = HIGHEST_PROTOCOL
|
||||
else:
|
||||
self.protocol = 1
|
||||
|
||||
|
||||
def __del__(self):
|
||||
self.close()
|
||||
|
||||
|
||||
def __getattr__(self, name):
|
||||
"""Many methods we can just pass through to the DB object.
|
||||
(See below)
|
||||
"""
|
||||
return getattr(self.db, name)
|
||||
|
||||
|
||||
#-----------------------------------
|
||||
# Dictionary access methods
|
||||
|
||||
def __len__(self):
|
||||
return len(self.db)
|
||||
|
||||
|
||||
def __getitem__(self, key):
|
||||
data = self.db[key]
|
||||
return cPickle.loads(data)
|
||||
|
||||
|
||||
def __setitem__(self, key, value):
|
||||
data = _dumps(value, self.protocol)
|
||||
self.db[key] = data
|
||||
|
||||
|
||||
def __delitem__(self, key):
|
||||
del self.db[key]
|
||||
|
||||
|
||||
def keys(self, txn=None):
|
||||
if txn is not None:
|
||||
return self.db.keys(txn)
|
||||
else:
|
||||
return self.db.keys()
|
||||
|
||||
if sys.version_info >= (2, 6) :
|
||||
def __iter__(self) : # XXX: Load all keys in memory :-(
|
||||
for k in self.db.keys() :
|
||||
yield k
|
||||
|
||||
# Do this when "DB" support iteration
|
||||
# Or is it enough to pass thru "getattr"?
|
||||
#
|
||||
# def __iter__(self) :
|
||||
# return self.db.__iter__()
|
||||
|
||||
|
||||
def open(self, *args, **kwargs):
|
||||
self.db.open(*args, **kwargs)
|
||||
self._closed = False
|
||||
|
||||
|
||||
def close(self, *args, **kwargs):
|
||||
self.db.close(*args, **kwargs)
|
||||
self._closed = True
|
||||
|
||||
|
||||
def __repr__(self):
|
||||
if self._closed:
|
||||
return '<DBShelf @ 0x%x - closed>' % (id(self))
|
||||
else:
|
||||
return repr(dict(self.iteritems()))
|
||||
|
||||
|
||||
def items(self, txn=None):
|
||||
if txn is not None:
|
||||
items = self.db.items(txn)
|
||||
else:
|
||||
items = self.db.items()
|
||||
newitems = []
|
||||
|
||||
for k, v in items:
|
||||
newitems.append( (k, cPickle.loads(v)) )
|
||||
return newitems
|
||||
|
||||
def values(self, txn=None):
|
||||
if txn is not None:
|
||||
values = self.db.values(txn)
|
||||
else:
|
||||
values = self.db.values()
|
||||
|
||||
return map(cPickle.loads, values)
|
||||
|
||||
#-----------------------------------
|
||||
# Other methods
|
||||
|
||||
def __append(self, value, txn=None):
|
||||
data = _dumps(value, self.protocol)
|
||||
return self.db.append(data, txn)
|
||||
|
||||
def append(self, value, txn=None):
|
||||
if self.get_type() == db.DB_RECNO:
|
||||
return self.__append(value, txn=txn)
|
||||
raise DBShelveError, "append() only supported when dbshelve opened with filetype=dbshelve.db.DB_RECNO"
|
||||
|
||||
|
||||
def associate(self, secondaryDB, callback, flags=0):
|
||||
def _shelf_callback(priKey, priData, realCallback=callback):
|
||||
# Safe in Python 2.x because expresion short circuit
|
||||
if sys.version_info[0] < 3 or isinstance(priData, bytes) :
|
||||
data = cPickle.loads(priData)
|
||||
else :
|
||||
data = cPickle.loads(bytes(priData, "iso8859-1")) # 8 bits
|
||||
return realCallback(priKey, data)
|
||||
|
||||
return self.db.associate(secondaryDB, _shelf_callback, flags)
|
||||
|
||||
|
||||
#def get(self, key, default=None, txn=None, flags=0):
|
||||
def get(self, *args, **kw):
|
||||
# We do it with *args and **kw so if the default value wasn't
|
||||
# given nothing is passed to the extension module. That way
|
||||
# an exception can be raised if set_get_returns_none is turned
|
||||
# off.
|
||||
data = self.db.get(*args, **kw)
|
||||
try:
|
||||
return cPickle.loads(data)
|
||||
except (EOFError, TypeError, cPickle.UnpicklingError):
|
||||
return data # we may be getting the default value, or None,
|
||||
# so it doesn't need unpickled.
|
||||
|
||||
def get_both(self, key, value, txn=None, flags=0):
|
||||
data = _dumps(value, self.protocol)
|
||||
data = self.db.get(key, data, txn, flags)
|
||||
return cPickle.loads(data)
|
||||
|
||||
|
||||
def cursor(self, txn=None, flags=0):
|
||||
c = DBShelfCursor(self.db.cursor(txn, flags))
|
||||
c.protocol = self.protocol
|
||||
return c
|
||||
|
||||
|
||||
def put(self, key, value, txn=None, flags=0):
|
||||
data = _dumps(value, self.protocol)
|
||||
return self.db.put(key, data, txn, flags)
|
||||
|
||||
|
||||
def join(self, cursorList, flags=0):
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
#----------------------------------------------
|
||||
# Methods allowed to pass-through to self.db
|
||||
#
|
||||
# close, delete, fd, get_byteswapped, get_type, has_key,
|
||||
# key_range, open, remove, rename, stat, sync,
|
||||
# upgrade, verify, and all set_* methods.
|
||||
|
||||
|
||||
#---------------------------------------------------------------------------
|
||||
|
||||
class DBShelfCursor:
|
||||
"""
|
||||
"""
|
||||
def __init__(self, cursor):
|
||||
self.dbc = cursor
|
||||
|
||||
def __del__(self):
|
||||
self.close()
|
||||
|
||||
|
||||
def __getattr__(self, name):
|
||||
"""Some methods we can just pass through to the cursor object. (See below)"""
|
||||
return getattr(self.dbc, name)
|
||||
|
||||
|
||||
#----------------------------------------------
|
||||
|
||||
def dup(self, flags=0):
|
||||
c = DBShelfCursor(self.dbc.dup(flags))
|
||||
c.protocol = self.protocol
|
||||
return c
|
||||
|
||||
|
||||
def put(self, key, value, flags=0):
|
||||
data = _dumps(value, self.protocol)
|
||||
return self.dbc.put(key, data, flags)
|
||||
|
||||
|
||||
def get(self, *args):
|
||||
count = len(args) # a method overloading hack
|
||||
method = getattr(self, 'get_%d' % count)
|
||||
method(*args)
|
||||
|
||||
def get_1(self, flags):
|
||||
rec = self.dbc.get(flags)
|
||||
return self._extract(rec)
|
||||
|
||||
def get_2(self, key, flags):
|
||||
rec = self.dbc.get(key, flags)
|
||||
return self._extract(rec)
|
||||
|
||||
def get_3(self, key, value, flags):
|
||||
data = _dumps(value, self.protocol)
|
||||
rec = self.dbc.get(key, flags)
|
||||
return self._extract(rec)
|
||||
|
||||
|
||||
def current(self, flags=0): return self.get_1(flags|db.DB_CURRENT)
|
||||
def first(self, flags=0): return self.get_1(flags|db.DB_FIRST)
|
||||
def last(self, flags=0): return self.get_1(flags|db.DB_LAST)
|
||||
def next(self, flags=0): return self.get_1(flags|db.DB_NEXT)
|
||||
def prev(self, flags=0): return self.get_1(flags|db.DB_PREV)
|
||||
def consume(self, flags=0): return self.get_1(flags|db.DB_CONSUME)
|
||||
def next_dup(self, flags=0): return self.get_1(flags|db.DB_NEXT_DUP)
|
||||
def next_nodup(self, flags=0): return self.get_1(flags|db.DB_NEXT_NODUP)
|
||||
def prev_nodup(self, flags=0): return self.get_1(flags|db.DB_PREV_NODUP)
|
||||
|
||||
|
||||
def get_both(self, key, value, flags=0):
|
||||
data = _dumps(value, self.protocol)
|
||||
rec = self.dbc.get_both(key, flags)
|
||||
return self._extract(rec)
|
||||
|
||||
|
||||
def set(self, key, flags=0):
|
||||
rec = self.dbc.set(key, flags)
|
||||
return self._extract(rec)
|
||||
|
||||
def set_range(self, key, flags=0):
|
||||
rec = self.dbc.set_range(key, flags)
|
||||
return self._extract(rec)
|
||||
|
||||
def set_recno(self, recno, flags=0):
|
||||
rec = self.dbc.set_recno(recno, flags)
|
||||
return self._extract(rec)
|
||||
|
||||
set_both = get_both
|
||||
|
||||
def _extract(self, rec):
|
||||
if rec is None:
|
||||
return None
|
||||
else:
|
||||
key, data = rec
|
||||
# Safe in Python 2.x because expresion short circuit
|
||||
if sys.version_info[0] < 3 or isinstance(data, bytes) :
|
||||
return key, cPickle.loads(data)
|
||||
else :
|
||||
return key, cPickle.loads(bytes(data, "iso8859-1")) # 8 bits
|
||||
|
||||
#----------------------------------------------
|
||||
# Methods allowed to pass-through to self.dbc
|
||||
#
|
||||
# close, count, delete, get_recno, join_item
|
||||
|
||||
|
||||
#---------------------------------------------------------------------------
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,83 @@
|
||||
#------------------------------------------------------------------------
|
||||
#
|
||||
# Copyright (C) 2000 Autonomous Zone Industries
|
||||
#
|
||||
# License: This is free software. You may use this software for any
|
||||
# purpose including modification/redistribution, so long as
|
||||
# this header remains intact and that you do not claim any
|
||||
# rights of ownership or authorship of this software. This
|
||||
# software has been tested, but no warranty is expressed or
|
||||
# implied.
|
||||
#
|
||||
# Author: Gregory P. Smith <greg@krypto.org>
|
||||
#
|
||||
# Note: I don't know how useful this is in reality since when a
|
||||
# DBLockDeadlockError happens the current transaction is supposed to be
|
||||
# aborted. If it doesn't then when the operation is attempted again
|
||||
# the deadlock is still happening...
|
||||
# --Robin
|
||||
#
|
||||
#------------------------------------------------------------------------
|
||||
|
||||
|
||||
#
|
||||
# import the time.sleep function in a namespace safe way to allow
|
||||
# "from bsddb.dbutils import *"
|
||||
#
|
||||
from time import sleep as _sleep
|
||||
|
||||
import sys
|
||||
absolute_import = (sys.version_info[0] >= 3)
|
||||
if absolute_import :
|
||||
# Because this syntaxis is not valid before Python 2.5
|
||||
exec("from . import db")
|
||||
else :
|
||||
import db
|
||||
|
||||
# always sleep at least N seconds between retrys
|
||||
_deadlock_MinSleepTime = 1.0/128
|
||||
# never sleep more than N seconds between retrys
|
||||
_deadlock_MaxSleepTime = 3.14159
|
||||
|
||||
# Assign a file object to this for a "sleeping" message to be written to it
|
||||
# each retry
|
||||
_deadlock_VerboseFile = None
|
||||
|
||||
|
||||
def DeadlockWrap(function, *_args, **_kwargs):
|
||||
"""DeadlockWrap(function, *_args, **_kwargs) - automatically retries
|
||||
function in case of a database deadlock.
|
||||
|
||||
This is a function intended to be used to wrap database calls such
|
||||
that they perform retrys with exponentially backing off sleeps in
|
||||
between when a DBLockDeadlockError exception is raised.
|
||||
|
||||
A 'max_retries' parameter may optionally be passed to prevent it
|
||||
from retrying forever (in which case the exception will be reraised).
|
||||
|
||||
d = DB(...)
|
||||
d.open(...)
|
||||
DeadlockWrap(d.put, "foo", data="bar") # set key "foo" to "bar"
|
||||
"""
|
||||
sleeptime = _deadlock_MinSleepTime
|
||||
max_retries = _kwargs.get('max_retries', -1)
|
||||
if 'max_retries' in _kwargs:
|
||||
del _kwargs['max_retries']
|
||||
while True:
|
||||
try:
|
||||
return function(*_args, **_kwargs)
|
||||
except db.DBLockDeadlockError:
|
||||
if _deadlock_VerboseFile:
|
||||
_deadlock_VerboseFile.write(
|
||||
'dbutils.DeadlockWrap: sleeping %1.3f\n' % sleeptime)
|
||||
_sleep(sleeptime)
|
||||
# exponential backoff in the sleep time
|
||||
sleeptime *= 2
|
||||
if sleeptime > _deadlock_MaxSleepTime:
|
||||
sleeptime = _deadlock_MaxSleepTime
|
||||
max_retries -= 1
|
||||
if max_retries == -1:
|
||||
raise
|
||||
|
||||
|
||||
#------------------------------------------------------------------------
|
||||
Reference in New Issue
Block a user