Cyclone-Team/gramps
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Only BSDDB plugin needs bsddb3; back/restore moved to db

Browse Source
This commit is contained in:
Doug Blank 2015-05-12 22:03:10 -04:00
parent 432a05c64b
commit af0b308b1e
22 changed files with 239 additions and 257 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
gramps/cli/clidbman.py
View File

@ -137,57 +137,8 @@ class CLIDbManager(object):
current DB.
Returns ("Unknown", "Unknown", "Unknown") if invalid DB or other error.
"""
from bsddb3 import dbshelve, db
from gramps.gen.db import META, PERSON_TBL
from gramps.gen.db.dbconst import BDBVERSFN
bdbversion_file = os.path.join(dirpath, BDBVERSFN)
if os.path.isfile(bdbversion_file):
vers_file = open(bdbversion_file)
bsddb_version = vers_file.readline().strip()
else:
return "Unknown", "Unknown", "Unknown"
current_bsddb_version = str(db.version())
if bsddb_version != current_bsddb_version:
return "Unknown", bsddb_version, "Unknown"
env = db.DBEnv()
flags = db.DB_CREATE | db.DB_PRIVATE |\
db.DB_INIT_MPOOL |\
db.DB_INIT_LOG | db.DB_INIT_TXN
try:
env.open(dirpath, flags)
except Exception as msg:
LOG.warning("Error opening db environment for '%s': %s" %
(name, str(msg)))
try:
env.close()
except Exception as msg:
LOG.warning("Error closing db environment for '%s': %s" %
(name, str(msg)))
return "Unknown", bsddb_version, "Unknown"
dbmap1 = dbshelve.DBShelf(env)
fname = os.path.join(dirpath, META + ".db")
try:
dbmap1.open(fname, META, db.DB_HASH, db.DB_RDONLY)
except:
env.close()
return "Unknown", bsddb_version, "Unknown"
schema_version = dbmap1.get(b'version', default=None)
dbmap1.close()
dbmap2 = dbshelve.DBShelf(env)
fname = os.path.join(dirpath, PERSON_TBL + ".db")
try:
dbmap2.open(fname, PERSON_TBL, db.DB_HASH, db.DB_RDONLY)
except:
env.close()
return "Unknown", bsddb_version, schema_version
count = len(dbmap2)
dbmap2.close()
env.close()
return (count, bsddb_version, schema_version)
## Maybe return the txt file contents, for now
return ("Unknown", "Unknown", "Unknown")
def family_tree_summary(self):
"""

6
gramps/gen/db/__init__.py
View File

@ -86,11 +86,5 @@ More details can be found in the manual's
from .base import *
from .dbconst import *
#from .cursor import *
#from .read import *
#from .bsddbtxn import *
from .txn import *
#from .undoredo import *
from .exceptions import *
#from .write import *
#from .backup import backup, restore

8
gramps/gen/db/backup.py
View File

@ -1,8 +0,0 @@
import logging
LOG = logging.getLogger(".Backup")
def backup(database):
print("FIXME")
def restore(database):
print("FIXME")

15
gramps/gen/db/dbconst.py
View File

@ -31,8 +31,7 @@ Declare constants used by database modules
__all__ = (
('DBPAGE', 'DBMODE', 'DBCACHE', 'DBLOCKS', 'DBOBJECTS', 'DBUNDO',
'DBEXT', 'DBMODE_R', 'DBMODE_W', 'DBUNDOFN', 'DBLOCKFN',
'DBRECOVFN','BDBVERSFN', 'DBLOGNAME', 'DBFLAGS_O', 'DBFLAGS_R',
'DBFLAGS_D', 'SCHVERSFN', 'PCKVERSFN'
'DBRECOVFN','BDBVERSFN', 'DBLOGNAME', 'SCHVERSFN', 'PCKVERSFN'
) +
('PERSON_KEY', 'FAMILY_KEY', 'SOURCE_KEY', 'CITATION_KEY',
@ -60,18 +59,6 @@ DBLOCKS = 100000 # Maximum number of locks supported
DBOBJECTS = 100000 # Maximum number of simultaneously locked objects
DBUNDO = 1000 # Maximum size of undo buffer
try:
from bsddb3.db import DB_CREATE, DB_AUTO_COMMIT, DB_DUP, DB_DUPSORT, DB_RDONLY
DBFLAGS_O = DB_CREATE | DB_AUTO_COMMIT # Default flags for database open
DBFLAGS_R = DB_RDONLY # Flags to open a database read-only
DBFLAGS_D = DB_DUP | DB_DUPSORT # Default flags for duplicate keys
except:
print("WARNING: no bsddb support")
# FIXME: make this more abstract to deal with other backends, or do not import
DBFLAGS_O = DB_CREATE = DB_AUTO_COMMIT = 0
DBFLAGS_R = DB_RDONLY = 0
DBFLAGS_D = DB_DUP = DB_DUPSORT = 0
PERSON_KEY = 0
FAMILY_KEY = 1
SOURCE_KEY = 2

6
gramps/gen/plug/_gramplet.py
View File

@ -320,8 +320,6 @@ class Gramplet(object):
self._idle_id = 0
LOG.debug("gramplet updater: %s : One time, done!" % self.gui.title)
return False
# FIXME: find out why Data Entry has this error, or just ignore it
import bsddb3 as bsddb
try:
retval = next(self._generator)
if not retval:
@ -332,10 +330,6 @@ class Gramplet(object):
LOG.debug("gramplet updater: %s: return %s" %
(self.gui.title, retval))
return retval
except bsddb.db.DBCursorClosedError:
# not sure why---caused by Data Entry Gramplet
LOG.warn("bsddb.db.DBCursorClosedError in: %s" % self.gui.title)
return False
except StopIteration:
self._idle_id = 0
self._generator.close()

8
gramps/grampsapp.py
View File

@ -136,14 +136,6 @@ if not sys.version_info >= MIN_PYTHON_VERSION :
'v3': MIN_PYTHON_VERSION[2]})
sys.exit(1)
try:
import bsddb3
except ImportError:
logging.warning(_("\nYou don't have the python3 bsddb3 package installed."
" This package is needed to start Gramps.\n\n"
"Gramps will terminate now."))
sys.exit(1)
#-------------------------------------------------------------------------
#
# Gramps libraries

9
gramps/gui/aboutdialog.py
View File

@ -28,7 +28,12 @@
import os
import sys
import io
import bsddb3 as bsddb
try:
import bsddb3 as bsddb ## ok, in try/except
BSDDB_STR = ellipses(str(bsddb.__version__) + " " + str(bsddb.db.version()))
except:
BSDDB_STR = 'not found'
##import logging
##_LOG = logging.getLogger(".GrampsAboutDialog")
@ -125,7 +130,7 @@ class GrampsAboutDialog(Gtk.AboutDialog):
"Distribution: %s")
% (ellipses(str(VERSION)),
ellipses(str(sys.version).replace('\n','')),
ellipses(str(bsddb.__version__) + " " + str(bsddb.db.version())),
BSDDB_STR,
ellipses(get_env_var('LANG','')),
ellipses(operatingsystem),
ellipses(distribution)))

3
gramps/gui/dbman.py
View File

@ -78,7 +78,6 @@ from gramps.cli.clidbman import CLIDbManager, NAME_FILE, time_val
from .ddtargets import DdTargets
from gramps.gen.recentfiles import rename_filename, remove_filename
from .glade import Glade
from gramps.gen.db.backup import restore
from gramps.gen.db.exceptions import DbException
@ -732,7 +731,7 @@ class DbManager(CLIDbManager):
self.__start_cursor(_("Rebuilding database from backup files"))
try:
restore(dbase)
dbase.restore()
except DbException as msg:
DbManager.ERROR(_("Error restoring backup data"), msg)

10
gramps/gui/editors/editfamily.py
View File

@ -26,7 +26,6 @@
# python modules
#
#-------------------------------------------------------------------------
from bsddb3 import db as bsddb_db
import pickle
#-------------------------------------------------------------------------
@ -1027,10 +1026,11 @@ class EditFamily(EditPrimary):
)
def save(self, *obj):
try:
self.__do_save()
except bsddb_db.DBRunRecoveryError as msg:
RunDatabaseRepair(msg[1])
## FIXME: how to catch a specific error?
#try:
self.__do_save()
#except bsddb_db.DBRunRecoveryError as msg:
# RunDatabaseRepair(msg[1])
def __do_save(self):
self.ok_button.set_sensitive(False)

9
gramps/gui/logger/_errorreportassistant.py
View File

@ -30,7 +30,12 @@ from gi.repository import GdkPixbuf
from gi.repository import GObject
import cairo
import sys, os
import bsddb3 as bsddb
try:
import bsddb3 as bsddb # ok, in try/except
BSDDB_STR = str(bsddb.__version__) + " " + str(bsddb.db.version())
except:
BSDDB_STR = 'not found'
#-------------------------------------------------------------------------
#
@ -166,7 +171,7 @@ class ErrorReportAssistant(Gtk.Assistant):
"gobject version: %s\n"\
"cairo version : %s"\
% (str(sys.version).replace('\n',''),
str(bsddb.__version__) + " " + str(bsddb.db.version()),
BSDDB_STR,
str(VERSION),
get_env_var('LANG',''),
operatingsystem,

3
gramps/gui/viewmanager.py
View File

@ -87,7 +87,6 @@ from gramps.gen.utils.file import media_path_full
from .dbloader import DbLoader
from .display import display_help, display_url
from .configure import GrampsPreferences
from gramps.gen.db.backup import backup
from gramps.gen.db.exceptions import DbException
from .aboutdialog import GrampsAboutDialog
from .navigator import Navigator
@ -760,7 +759,7 @@ class ViewManager(CLIManager):
self.uistate.progress.show()
self.uistate.push_message(self.dbstate, _("Autobackup..."))
try:
backup(self.dbstate.db)
self.dbstate.db.backup()
except DbException as msg:
ErrorDialog(_("Error saving backup data"), msg)
self.uistate.set_busy_cursor(False)

1
gramps/plugins/database/bsddb_support/__init__.py
View File

@ -93,4 +93,3 @@ from gramps.gen.db.txn import *
from .undoredo import *
from gramps.gen.db.exceptions import *
from .write import *
from .backup import backup, restore

139
gramps/plugins/database/bsddb_support/backup.py
View File

@ -72,142 +72,3 @@ from .write import FAMILY_TBL, PLACES_TBL, SOURCES_TBL, MEDIA_TBL, \
import logging
LOG = logging.getLogger(".Backup")
def backup(database):
"""
Exports the database to a set of backup files. These files consist
of the pickled database tables, one file for each table.
The heavy lifting is done by the private :py:func:`__do__export` function.
The purpose of this function is to catch any exceptions that occur.
:param database: database instance to backup
:type database: DbDir
"""
try:
__do_export(database)
except (OSError, IOError) as msg:
raise DbException(str(msg))
def __mk_backup_name(database, base):
"""
Return the backup name of the database table
:param database: database instance
:type database: DbDir
:param base: base name of the table
:type base: str
"""
return os.path.join(database.get_save_path(), base + ".gbkp")
def __mk_tmp_name(database, base):
"""
Return the temporary backup name of the database table
:param database: database instance
:type database: DbDir
:param base: base name of the table
:type base: str
"""
return os.path.join(database.get_save_path(), base + ".gbkp.new")
def __do_export(database):
"""
Loop through each table of the database, saving the pickled data
a file.
:param database: database instance to backup
:type database: DbDir
"""
try:
for (base, tbl) in __build_tbl_map(database):
backup_name = __mk_tmp_name(database, base)
backup_table = open(backup_name, 'wb')
cursor = tbl.cursor()
data = cursor.first()
while data:
pickle.dump(data, backup_table, 2)
data = cursor.next()
cursor.close()
backup_table.close()
except (IOError,OSError):
return
for (base, tbl) in __build_tbl_map(database):
new_name = __mk_backup_name(database, base)
old_name = __mk_tmp_name(database, base)
if os.path.isfile(new_name):
os.unlink(new_name)
os.rename(old_name, new_name)
def restore(database):
"""
Restores the database to a set of backup files. These files consist
of the pickled database tables, one file for each table.
The heavy lifting is done by the private :py:func:`__do__restore` function.
The purpose of this function is to catch any exceptions that occur.
:param database: database instance to restore
:type database: DbDir
"""
try:
__do_restore(database)
except (OSError, IOError) as msg:
raise DbException(str(msg))
def __do_restore(database):
"""
Loop through each table of the database, restoring the pickled data
to the appropriate database file.
:param database: database instance to backup
:type database: DbDir
"""
for (base, tbl) in __build_tbl_map(database):
backup_name = __mk_backup_name(database, base)
backup_table = open(backup_name, 'rb')
__load_tbl_txn(database, backup_table, tbl)
database.rebuild_secondary()
def __load_tbl_txn(database, backup_table, tbl):
"""
Return the temporary backup name of the database table
:param database: database instance
:type database: DbDir
:param backup_table: file containing the backup data
:type backup_table: file
:param tbl: Berkeley db database table
:type tbl: Berkeley db database table
"""
try:
while True:
data = pickle.load(backup_table)
txn = database.env.txn_begin()
tbl.put(data[0], data[1], txn=txn)
txn.commit()
except EOFError:
backup_table.close()
def __build_tbl_map(database):
"""
Builds a table map of names to database tables.
:param database: database instance to backup
:type database: DbDir
"""
return [
( PERSON_TBL, database.person_map.db),
( FAMILY_TBL, database.family_map.db),
( PLACES_TBL, database.place_map.db),
( SOURCES_TBL, database.source_map.db),
( CITATIONS_TBL, database.citation_map.db),
( REPO_TBL, database.repository_map.db),
( NOTE_TBL, database.note_map.db),
( MEDIA_TBL, database.media_map.db),
( EVENTS_TBL, database.event_map.db),
( TAG_TBL, database.tag_map.db),
( META, database.metadata.db),
]

62
gramps/plugins/database/bsddb_support/summary.py Normal file
View File

@ -0,0 +1,62 @@
## Removed from clidbman.py
## specific to bsddb
from bsddb3 import dbshelve, db
import os
from gramps.gen.db import META, PERSON_TBL
from gramps.gen.db.dbconst import BDBVERSFN
def get_dbdir_summary(dirpath, name):
"""
Returns (people_count, bsddb_version, schema_version) of
current DB.
Returns ("Unknown", "Unknown", "Unknown") if invalid DB or other error.
"""
bdbversion_file = os.path.join(dirpath, BDBVERSFN)
if os.path.isfile(bdbversion_file):
vers_file = open(bdbversion_file)
bsddb_version = vers_file.readline().strip()
else:
return "Unknown", "Unknown", "Unknown"
current_bsddb_version = str(db.version())
if bsddb_version != current_bsddb_version:
return "Unknown", bsddb_version, "Unknown"
env = db.DBEnv()
flags = db.DB_CREATE | db.DB_PRIVATE |\
db.DB_INIT_MPOOL |\
db.DB_INIT_LOG | db.DB_INIT_TXN
try:
env.open(dirpath, flags)
except Exception as msg:
LOG.warning("Error opening db environment for '%s': %s" %
(name, str(msg)))
try:
env.close()
except Exception as msg:
LOG.warning("Error closing db environment for '%s': %s" %
(name, str(msg)))
return "Unknown", bsddb_version, "Unknown"
dbmap1 = dbshelve.DBShelf(env)
fname = os.path.join(dirpath, META + ".db")
try:
dbmap1.open(fname, META, db.DB_HASH, db.DB_RDONLY)
except:
env.close()
return "Unknown", bsddb_version, "Unknown"
schema_version = dbmap1.get(b'version', default=None)
dbmap1.close()
dbmap2 = dbshelve.DBShelf(env)
fname = os.path.join(dirpath, PERSON_TBL + ".db")
try:
dbmap2.open(fname, PERSON_TBL, db.DB_HASH, db.DB_RDONLY)
except:
env.close()
return "Unknown", bsddb_version, schema_version
count = len(dbmap2)
dbmap2.close()
env.close()
return (count, bsddb_version, schema_version)

0
gramps/gen/db/test/__init__.py → gramps/plugins/database/bsddb_support/test/__init__.py
View File

0
gramps/gen/db/test/cursor_test.py → gramps/plugins/database/bsddb_support/test/cursor_test.py
View File

0
gramps/gen/db/test/db_test.py → gramps/plugins/database/bsddb_support/test/db_test.py
View File

0
gramps/gen/db/test/grampsdbtestbase.py → gramps/plugins/database/bsddb_support/test/grampsdbtestbase.py
View File

0
gramps/gen/db/test/reference_map_test.py → gramps/plugins/database/bsddb_support/test/reference_map_test.py
View File

156
gramps/plugins/database/bsddb_support/write.py
View File

@ -40,16 +40,12 @@ from functools import wraps
import logging
from sys import maxsize, getfilesystemencoding, version_info
try:
from bsddb3 import dbshelve, db
except:
# FIXME: make this more abstract to deal with other backends
class db:
DB_HASH = 0
DBRunRecoveryError = 0
DBAccessError = 0
DBPageNotFoundError = 0
DBInvalidArgError = 0
from bsddb3 import dbshelve, db
from bsddb3.db import DB_CREATE, DB_AUTO_COMMIT, DB_DUP, DB_DUPSORT, DB_RDONLY
DBFLAGS_O = DB_CREATE | DB_AUTO_COMMIT # Default flags for database open
DBFLAGS_R = DB_RDONLY # Flags to open a database read-only
DBFLAGS_D = DB_DUP | DB_DUPSORT # Default flags for duplicate keys
#-------------------------------------------------------------------------
#
@ -2448,6 +2444,146 @@ class DbBsddb(DbBsddbRead, DbWriteBase, UpdateCallback):
"""
return DbTxn
def backup(self):
"""
Exports the database to a set of backup files. These files consist
of the pickled database tables, one file for each table.
The heavy lifting is done by the private :py:func:`__do__export` function.
The purpose of this function is to catch any exceptions that occur.
:param database: database instance to backup
:type database: DbDir
"""
try:
do_export(self)
except (OSError, IOError) as msg:
raise DbException(str(msg))
def restore(self):
"""
Restores the database to a set of backup files. These files consist
of the pickled database tables, one file for each table.
The heavy lifting is done by the private :py:func:`__do__restore` function.
The purpose of this function is to catch any exceptions that occur.
:param database: database instance to restore
:type database: DbDir
"""
try:
do_restore(self)
except (OSError, IOError) as msg:
raise DbException(str(msg))
def mk_backup_name(database, base):
"""
Return the backup name of the database table
:param database: database instance
:type database: DbDir
:param base: base name of the table
:type base: str
"""
return os.path.join(database.get_save_path(), base + ".gbkp")
def mk_tmp_name(database, base):
"""
Return the temporary backup name of the database table
:param database: database instance
:type database: DbDir
:param base: base name of the table
:type base: str
"""
return os.path.join(database.get_save_path(), base + ".gbkp.new")
def do_export(database):
"""
Loop through each table of the database, saving the pickled data
a file.
:param database: database instance to backup
:type database: DbDir
"""
try:
for (base, tbl) in build_tbl_map(database):
backup_name = mk_tmp_name(database, base)
backup_table = open(backup_name, 'wb')
cursor = tbl.cursor()
data = cursor.first()
while data:
pickle.dump(data, backup_table, 2)
data = cursor.next()
cursor.close()
backup_table.close()
except (IOError,OSError):
return
for (base, tbl) in build_tbl_map(database):
new_name = mk_backup_name(database, base)
old_name = mk_tmp_name(database, base)
if os.path.isfile(new_name):
os.unlink(new_name)
os.rename(old_name, new_name)
def do_restore(database):
"""
Loop through each table of the database, restoring the pickled data
to the appropriate database file.
:param database: database instance to backup
:type database: DbDir
"""
for (base, tbl) in build_tbl_map(database):
backup_name = mk_backup_name(database, base)
backup_table = open(backup_name, 'rb')
load_tbl_txn(database, backup_table, tbl)
database.rebuild_secondary()
def load_tbl_txn(database, backup_table, tbl):
"""
Return the temporary backup name of the database table
:param database: database instance
:type database: DbDir
:param backup_table: file containing the backup data
:type backup_table: file
:param tbl: Berkeley db database table
:type tbl: Berkeley db database table
"""
try:
while True:
data = pickle.load(backup_table)
txn = database.env.txn_begin()
tbl.put(data[0], data[1], txn=txn)
txn.commit()
except EOFError:
backup_table.close()
def build_tbl_map(database):
"""
Builds a table map of names to database tables.
:param database: database instance to backup
:type database: DbDir
"""
return [
( PERSON_TBL, database.person_map.db),
( FAMILY_TBL, database.family_map.db),
( PLACES_TBL, database.place_map.db),
( SOURCES_TBL, database.source_map.db),
( CITATIONS_TBL, database.citation_map.db),
( REPO_TBL, database.repository_map.db),
( NOTE_TBL, database.note_map.db),
( MEDIA_TBL, database.media_map.db),
( EVENTS_TBL, database.event_map.db),
( TAG_TBL, database.tag_map.db),
( META, database.metadata.db),
]
def _mkname(path, name):
return os.path.join(path, name + DBEXT)

0
gramps/plugins/database/bsddb_support/dictionary.py → gramps/plugins/database/dictionary.py
View File

8
gramps/plugins/gramplet/leak.py
View File

@ -31,7 +31,6 @@ Show uncollected objects in a window.
#------------------------------------------------------------------------
from gramps.gen.const import GRAMPS_LOCALE as glocale
_ = glocale.translation.gettext
from bsddb3.db import DBError
#------------------------------------------------------------------------
#
@ -156,6 +155,13 @@ class Leak(Gramplet):
parent=self.uistate.window)
def display(self):
try:
from bsddb3.db import DBError
except:
class DBError(Exception):
"""
Dummy.
"""
gc.collect(2)
self.model.clear()
count = 0