From b059bdec66361d9e89f28b5dc215a31ce1bcf8fb Mon Sep 17 00:00:00 2001 From: Doug Blank Date: Tue, 12 May 2015 16:30:46 -0400 Subject: [PATCH] Database backend as a plugin: this set of changes moves most or all of Bsddb from gramps.gen.db to gramps.plugins.database. The id of the plugin is 'bsddb' which can be loaded using the make_database(id, dbstate) API (for now). Next step is to add an identifying text in the directory to indicate which database backend to use. --- gramps/cli/arghandler.py | 5 +- gramps/cli/clidbman.py | 9 +- gramps/cli/grampscli.py | 6 +- gramps/gen/db/__init__.py | 27 ++- gramps/gen/db/backup.py | 209 +---------------- gramps/gen/dbstate.py | 6 +- gramps/gen/plug/_gramplet.py | 1 - gramps/gen/plug/_manager.py | 5 + gramps/gen/plug/_pluginreg.py | 31 ++- gramps/gen/utils/callman.py | 3 +- gramps/gui/dbloader.py | 4 +- gramps/gui/dbman.py | 11 +- gramps/gui/pluginmanager.py | 7 +- gramps/plugins/database/bsddb.gpr.py | 31 +++ gramps/plugins/database/bsddb.py | 3 + .../database/bsddb_support/__init__.py | 96 ++++++++ .../plugins/database/bsddb_support/backup.py | 213 ++++++++++++++++++ .../database/bsddb_support}/bsddbtxn.py | 0 .../database/bsddb_support}/cursor.py | 0 .../database/bsddb_support}/dictionary.py | 27 +-- .../database/bsddb_support}/read.py | 45 ++-- .../database/bsddb_support}/undoredo.py | 8 +- .../database/bsddb_support}/upgrade.py | 31 +-- .../database/bsddb_support}/write.py | 45 ++-- 24 files changed, 507 insertions(+), 316 deletions(-) create mode 100644 gramps/plugins/database/bsddb.gpr.py create mode 100644 gramps/plugins/database/bsddb.py create mode 100644 gramps/plugins/database/bsddb_support/__init__.py create mode 100644 gramps/plugins/database/bsddb_support/backup.py rename gramps/{gen/db => plugins/database/bsddb_support}/bsddbtxn.py (100%) rename gramps/{gen/db => plugins/database/bsddb_support}/cursor.py (100%) rename gramps/{gen/db => plugins/database/bsddb_support}/dictionary.py (98%) rename gramps/{gen/db => plugins/database/bsddb_support}/read.py (98%) rename gramps/{gen/db => plugins/database/bsddb_support}/undoredo.py (98%) rename gramps/{gen/db => plugins/database/bsddb_support}/upgrade.py (98%) rename gramps/{gen/db => plugins/database/bsddb_support}/write.py (99%) diff --git a/gramps/cli/arghandler.py b/gramps/cli/arghandler.py index 0db169d30..c7a5f5a4a 100644 --- a/gramps/cli/arghandler.py +++ b/gramps/cli/arghandler.py @@ -44,7 +44,7 @@ import sys #------------------------------------------------------------------------- from gramps.gen.recentfiles import recent_files from gramps.gen.utils.file import rm_tempdir, get_empty_tempdir -from gramps.gen.db import DbBsddb +from gramps.gen.db import make_database from .clidbman import CLIDbManager, NAME_FILE, find_locker_name from gramps.gen.plug import BasePluginManager @@ -491,7 +491,8 @@ class ArgHandler(object): self.imp_db_path, title = self.dbman.create_new_db_cli() else: self.imp_db_path = get_empty_tempdir("import_dbdir") - newdb = DbBsddb() + + newdb = make_database("bsddb", self.dbstate) newdb.write_version(self.imp_db_path) try: diff --git a/gramps/cli/clidbman.py b/gramps/cli/clidbman.py index 404104afe..52b183013 100644 --- a/gramps/cli/clidbman.py +++ b/gramps/cli/clidbman.py @@ -54,7 +54,7 @@ _LOG = logging.getLogger(DBLOGNAME) #------------------------------------------------------------------------- from gramps.gen.const import GRAMPS_LOCALE as glocale _ = glocale.translation.gettext -from gramps.gen.db import DbBsddb +from gramps.gen.db import make_database from gramps.gen.plug import BasePluginManager from gramps.gen.config import config from gramps.gen.constfunc import win, conv_to_unicode @@ -294,7 +294,8 @@ class CLIDbManager(object): if create_db: # write the version number into metadata - newdb = DbBsddb() + + newdb = make_database("bsddb", self.dbstate) newdb.write_version(new_path) (tval, last) = time_val(new_path) @@ -360,8 +361,8 @@ class CLIDbManager(object): # Create a new database self.__start_cursor(_("Importing data...")) - dbclass = DbBsddb - dbase = dbclass() + + dbase = make_database("bsddb", self.dbstate) dbase.load(new_path, user.callback) import_function = plugin.get_import_function() diff --git a/gramps/cli/grampscli.py b/gramps/cli/grampscli.py index a45564a19..00b66d352 100644 --- a/gramps/cli/grampscli.py +++ b/gramps/cli/grampscli.py @@ -49,7 +49,7 @@ from gramps.gen.config import config from gramps.gen.const import PLUGINS_DIR, USER_PLUGINS from gramps.gen.errors import DbError from gramps.gen.dbstate import DbState -from gramps.gen.db import DbBsddb +from gramps.gen.db import make_database from gramps.gen.db.exceptions import (DbUpgradeRequiredError, BsddbDowngradeError, DbVersionError, @@ -152,9 +152,9 @@ class CLIDbLoader(object): else: mode = 'w' - dbclass = DbBsddb + db = make_database("bsddb", self.dbstate) - self.dbstate.change_database(dbclass()) + self.dbstate.change_database(db) self.dbstate.db.disable_signals() self._begin_progress() diff --git a/gramps/gen/db/__init__.py b/gramps/gen/db/__init__.py index 5b79ce334..b977887b7 100644 --- a/gramps/gen/db/__init__.py +++ b/gramps/gen/db/__init__.py @@ -86,11 +86,26 @@ More details can be found in the manual's from .base import * from .dbconst import * -from .cursor import * -from .read import * -from .bsddbtxn import * +#from .cursor import * +#from .read import * +#from .bsddbtxn import * from .txn import * -from .undoredo import * +#from .undoredo import * from .exceptions import * -from .write import * -from .backup import backup, restore +#from .write import * +#from .backup import backup, restore + +def make_database(id, dbstate): + from gramps.cli.grampscli import CLIManager + from gramps.gen.plug import BasePluginManager + from gramps.cli.user import User + + climanager = CLIManager(dbstate, setloader=False, user=User()) # do not load db_loader + climanager.do_reg_plugins(dbstate, None) + pmgr = BasePluginManager.get_instance() + pdata = pmgr.get_plugin(id) + + mod = pmgr.load_plugin(pdata) + database = getattr(mod, pdata.databaseclass) + return database() + diff --git a/gramps/gen/db/backup.py b/gramps/gen/db/backup.py index 9329caaff..52d4d5f05 100644 --- a/gramps/gen/db/backup.py +++ b/gramps/gen/db/backup.py @@ -1,213 +1,8 @@ -# -# Gramps - a GTK+/GNOME based genealogy program -# -# Copyright (C) 2007 Donald N. Allingham -# Copyright (C) 2011 Tim G L Lyons -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# -# gen/db/backup.py - -""" -Description -=========== - -This module Provides backup and restore functions for a database. The -backup function saves the data into backup files, while the restore -function loads the data back into a database. - -You should only restore the data into an empty database. - -Implementation -============== - -Not all of the database tables need to be backed up, since many are -automatically generated from the others. The tables that are backed up -are the primary tables and the metadata table. - -The database consists of a table of "pickled" tuples. Each of the -primary tables is "walked", and the pickled tuple is extracted, and -written to the backup file. - -Restoring the data is just as simple. The backup file is parsed an -entry at a time, and inserted into the associated database table. The -derived tables are built automatically as the items are entered into -db. -""" - -#------------------------------------------------------------------------- -# -# load standard python libraries -# -#------------------------------------------------------------------------- -import os -import pickle - -#------------------------------------------------------------------------ -# -# Gramps libs -# -#------------------------------------------------------------------------ -from .exceptions import DbException -from .write import FAMILY_TBL, PLACES_TBL, SOURCES_TBL, MEDIA_TBL, \ - EVENTS_TBL, PERSON_TBL, REPO_TBL, NOTE_TBL, TAG_TBL, META, CITATIONS_TBL - -#------------------------------------------------------------------------ -# -# Set up logging -# -#------------------------------------------------------------------------ import logging LOG = logging.getLogger(".Backup") def backup(database): - """ - Exports the database to a set of backup files. These files consist - of the pickled database tables, one file for each table. - - The heavy lifting is done by the private :py:func:`__do__export` function. - The purpose of this function is to catch any exceptions that occur. - - :param database: database instance to backup - :type database: DbDir - """ - try: - __do_export(database) - except (OSError, IOError) as msg: - raise DbException(str(msg)) - -def __mk_backup_name(database, base): - """ - Return the backup name of the database table - - :param database: database instance - :type database: DbDir - :param base: base name of the table - :type base: str - """ - return os.path.join(database.get_save_path(), base + ".gbkp") - -def __mk_tmp_name(database, base): - """ - Return the temporary backup name of the database table - - :param database: database instance - :type database: DbDir - :param base: base name of the table - :type base: str - """ - return os.path.join(database.get_save_path(), base + ".gbkp.new") - -def __do_export(database): - """ - Loop through each table of the database, saving the pickled data - a file. - - :param database: database instance to backup - :type database: DbDir - """ - try: - for (base, tbl) in __build_tbl_map(database): - backup_name = __mk_tmp_name(database, base) - backup_table = open(backup_name, 'wb') - - cursor = tbl.cursor() - data = cursor.first() - while data: - pickle.dump(data, backup_table, 2) - data = cursor.next() - cursor.close() - backup_table.close() - except (IOError,OSError): - return - - for (base, tbl) in __build_tbl_map(database): - new_name = __mk_backup_name(database, base) - old_name = __mk_tmp_name(database, base) - if os.path.isfile(new_name): - os.unlink(new_name) - os.rename(old_name, new_name) + print("FIXME") def restore(database): - """ - Restores the database to a set of backup files. These files consist - of the pickled database tables, one file for each table. - - The heavy lifting is done by the private :py:func:`__do__restore` function. - The purpose of this function is to catch any exceptions that occur. - - :param database: database instance to restore - :type database: DbDir - """ - try: - __do_restore(database) - except (OSError, IOError) as msg: - raise DbException(str(msg)) - -def __do_restore(database): - """ - Loop through each table of the database, restoring the pickled data - to the appropriate database file. - - :param database: database instance to backup - :type database: DbDir - """ - for (base, tbl) in __build_tbl_map(database): - backup_name = __mk_backup_name(database, base) - backup_table = open(backup_name, 'rb') - __load_tbl_txn(database, backup_table, tbl) - - database.rebuild_secondary() - -def __load_tbl_txn(database, backup_table, tbl): - """ - Return the temporary backup name of the database table - - :param database: database instance - :type database: DbDir - :param backup_table: file containing the backup data - :type backup_table: file - :param tbl: Berkeley db database table - :type tbl: Berkeley db database table - """ - try: - while True: - data = pickle.load(backup_table) - txn = database.env.txn_begin() - tbl.put(data[0], data[1], txn=txn) - txn.commit() - except EOFError: - backup_table.close() - -def __build_tbl_map(database): - """ - Builds a table map of names to database tables. - - :param database: database instance to backup - :type database: DbDir - """ - return [ - ( PERSON_TBL, database.person_map.db), - ( FAMILY_TBL, database.family_map.db), - ( PLACES_TBL, database.place_map.db), - ( SOURCES_TBL, database.source_map.db), - ( CITATIONS_TBL, database.citation_map.db), - ( REPO_TBL, database.repository_map.db), - ( NOTE_TBL, database.note_map.db), - ( MEDIA_TBL, database.media_map.db), - ( EVENTS_TBL, database.event_map.db), - ( TAG_TBL, database.tag_map.db), - ( META, database.metadata.db), - ] + print("FIXME") diff --git a/gramps/gen/dbstate.py b/gramps/gen/dbstate.py index 839d05cd2..12ee68ea9 100644 --- a/gramps/gen/dbstate.py +++ b/gramps/gen/dbstate.py @@ -23,8 +23,8 @@ Provide the database state class """ -from .db import DbBsddbRead from .db import DbReadBase +from .db import make_database from .proxy.proxybase import ProxyDbBase from .utils.callback import Callback from .config import config @@ -45,7 +45,7 @@ class DbState(Callback): just a place holder until a real DB is assigned. """ Callback.__init__(self) - self.db = DbBsddbRead() + self.db = make_database("bsddb", self) self.open = False self.stack = [] @@ -88,7 +88,7 @@ class DbState(Callback): """ self.emit('no-database', ()) self.db.close() - self.db = DbBsddbRead() + self.db = make_database("bsddb", self) self.db.db_is_open = False self.open = False self.emit('database-changed', (self.db, )) diff --git a/gramps/gen/plug/_gramplet.py b/gramps/gen/plug/_gramplet.py index c040bae7d..b812a7fb7 100644 --- a/gramps/gen/plug/_gramplet.py +++ b/gramps/gen/plug/_gramplet.py @@ -70,7 +70,6 @@ class Gramplet(object): self.connect(self.gui.textview, "motion-notify-event", self.gui.on_motion) self.connect_signal('Person', self._active_changed) - self._db_changed(self.dbstate.db) active_person = self.get_active('Person') if active_person: # already changed diff --git a/gramps/gen/plug/_manager.py b/gramps/gen/plug/_manager.py index e047f6166..e30a089ae 100644 --- a/gramps/gen/plug/_manager.py +++ b/gramps/gen/plug/_manager.py @@ -412,6 +412,11 @@ class BasePluginManager(object): """ return self.__pgr.sidebar_plugins() + def get_reg_databases(self): + """ Return list of registered database backends + """ + return self.__pgr.database_plugins() + def get_external_opt_dict(self): """ Return the dictionary of external options. """ return self.__external_opt_dict diff --git a/gramps/gen/plug/_pluginreg.py b/gramps/gen/plug/_pluginreg.py index 04167cafc..5619f18ed 100644 --- a/gramps/gen/plug/_pluginreg.py +++ b/gramps/gen/plug/_pluginreg.py @@ -70,8 +70,9 @@ VIEW = 8 RELCALC = 9 GRAMPLET = 10 SIDEBAR = 11 +DATABASE = 12 PTYPE = [REPORT , QUICKREPORT, TOOL, IMPORT, EXPORT, DOCGEN, GENERAL, - MAPSERVICE, VIEW, RELCALC, GRAMPLET, SIDEBAR] + MAPSERVICE, VIEW, RELCALC, GRAMPLET, SIDEBAR, DATABASE] PTYPE_STR = { REPORT: _('Report') , QUICKREPORT: _('Quickreport'), @@ -85,6 +86,7 @@ PTYPE_STR = { RELCALC: _('Relationships'), GRAMPLET: _('Gramplet'), SIDEBAR: _('Sidebar'), + DATABASE: _('Database'), } #possible report categories @@ -206,7 +208,7 @@ class PluginData(object): The python path where the plugin implementation can be found .. attribute:: ptype The plugin type. One of REPORT , QUICKREPORT, TOOL, IMPORT, - EXPORT, DOCGEN, GENERAL, MAPSERVICE, VIEW, GRAMPLET + EXPORT, DOCGEN, GENERAL, MAPSERVICE, VIEW, GRAMPLET, DATABASE .. attribute:: authors List of authors of the plugin, default=[] .. attribute:: authors_email @@ -349,6 +351,11 @@ class PluginData(object): the plugin is appended to the list of plugins. If START, then the plugin is prepended. Only set START if you want a plugin to be the first in the order of plugins + + Attributes for DATABASE plugins + + .. attribute:: databaseclass + The class in the module that is the database class """ def __init__(self): @@ -421,6 +428,8 @@ class PluginData(object): self._menu_label = '' #VIEW and SIDEBAR attr self._order = END + #DATABASE attr + self._databaseclass = None #GENERAL attr self._data = [] self._process = None @@ -931,6 +940,17 @@ class PluginData(object): order = property(_get_order, _set_order) + #DATABASE attributes + def _set_databaseclass(self, databaseclass): + if not self._ptype == DATABASE: + raise ValueError('databaseclass may only be set for DATABASE plugins') + self._databaseclass = databaseclass + + def _get_databaseclass(self): + return self._databaseclass + + databaseclass = property(_get_databaseclass, _set_databaseclass) + #GENERAL attr def _set_data(self, data): if not self._ptype in (GENERAL,): @@ -1032,6 +1052,7 @@ def make_environment(**kwargs): 'REPORT_MODE_CLI': REPORT_MODE_CLI, 'TOOL_MODE_GUI': TOOL_MODE_GUI, 'TOOL_MODE_CLI': TOOL_MODE_CLI, + 'DATABASE': DATABASE, 'GRAMPSVERSION': GRAMPSVERSION, 'START': START, 'END': END, @@ -1297,6 +1318,12 @@ class PluginRegister(object): """ return self.type_plugins(SIDEBAR) + def database_plugins(self): + """ + Return a list of :class:`PluginData` that are of type DATABASE + """ + return self.type_plugins(DATABASE) + def filter_load_on_reg(self): """ Return a list of :class:`PluginData` that have load_on_reg == True diff --git a/gramps/gen/utils/callman.py b/gramps/gen/utils/callman.py index ca6a94d86..939a0f6fa 100644 --- a/gramps/gen/utils/callman.py +++ b/gramps/gen/utils/callman.py @@ -303,7 +303,8 @@ class CallbackManager(object): Do a custom db connect signal outside of the primary object ones managed automatically. """ - self.custom_signal_keys.append(self.database.connect(name, callback)) + if self.database: + self.custom_signal_keys.append(self.database.connect(name, callback)) def __callbackcreator(self, signal, noarg=False): """ diff --git a/gramps/gui/dbloader.py b/gramps/gui/dbloader.py index 6e7f85d5e..18a1c3281 100644 --- a/gramps/gui/dbloader.py +++ b/gramps/gui/dbloader.py @@ -55,7 +55,7 @@ from gramps.gen.const import GRAMPS_LOCALE as glocale _ = glocale.translation.gettext from gramps.cli.grampscli import CLIDbLoader from gramps.gen.config import config -from gramps.gen.db import DbBsddb +from gramps.gen.db import make_database from gramps.gen.db.exceptions import (DbUpgradeRequiredError, BsddbDowngradeError, DbVersionError, @@ -305,7 +305,7 @@ class DbLoader(CLIDbLoader): else: mode = 'w' - db = DbBsddb() + db = make_database("bsddb", self.dbstate) db.disable_signals() self.dbstate.no_database() diff --git a/gramps/gui/dbman.py b/gramps/gui/dbman.py index f3e6214e3..c5ccbe657 100644 --- a/gramps/gui/dbman.py +++ b/gramps/gui/dbman.py @@ -73,7 +73,7 @@ _ = glocale.translation.gettext from gramps.gen.const import URL_WIKISTRING from .user import User from .dialog import ErrorDialog, QuestionDialog, QuestionDialog2 -from gramps.gen.db import DbBsddb +from gramps.gen.db import make_database from .pluginmanager import GuiPluginManager from gramps.cli.clidbman import CLIDbManager, NAME_FILE, time_val from .ddtargets import DdTargets @@ -531,8 +531,8 @@ class DbManager(CLIDbManager): new_path, newname = self._create_new_db("%s : %s" % (parent_name, name)) self.__start_cursor(_("Extracting archive...")) - dbclass = DbBsddb - dbase = dbclass() + + dbase = make_database("bsddb", self.dbstate) dbase.load(new_path, None) self.__start_cursor(_("Importing archive...")) @@ -719,11 +719,10 @@ class DbManager(CLIDbManager): fname = os.path.join(dirname, filename) os.unlink(fname) - newdb = DbBsddb() + newdb = make_database("bsddb", self.dbstate) newdb.write_version(dirname) - dbclass = DbBsddb - dbase = dbclass() + dbase = make_database("bsddb", self.dbstate) dbase.set_save_path(dirname) dbase.load(dirname, None) diff --git a/gramps/gui/pluginmanager.py b/gramps/gui/pluginmanager.py index edde82dac..bb609e984 100644 --- a/gramps/gui/pluginmanager.py +++ b/gramps/gui/pluginmanager.py @@ -205,7 +205,12 @@ class GuiPluginManager(Callback): return [plg for plg in self.basemgr.get_reg_docgens() if plg.id not in self.__hidden_plugins] + def get_reg_databases(self): + """ Return list of non hidden registered database backends + """ + return [plg for plg in self.basemgr.get_reg_databases() + if plg.id not in self.__hidden_plugins] + def get_reg_general(self, category=None): return [plg for plg in self.basemgr.get_reg_general(category) if plg.id not in self.__hidden_plugins] - diff --git a/gramps/plugins/database/bsddb.gpr.py b/gramps/plugins/database/bsddb.gpr.py new file mode 100644 index 000000000..20f2e9fda --- /dev/null +++ b/gramps/plugins/database/bsddb.gpr.py @@ -0,0 +1,31 @@ +# +# Gramps - a GTK+/GNOME based genealogy program +# +# Copyright (C) 2015 Douglas Blank +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# + +plg = newplugin() +plg.id = 'bsddb' +plg.name = _("BSDDB Database Backend") +plg.name_accell = _("_BSDDB Database Backend") +plg.description = _("Berkeley Software Distribution Database Backend") +plg.version = '1.0' +plg.gramps_target_version = "4.2" +plg.status = STABLE +plg.fname = 'bsddb.py' +plg.ptype = DATABASE +plg.databaseclass = 'DbBsddb' diff --git a/gramps/plugins/database/bsddb.py b/gramps/plugins/database/bsddb.py new file mode 100644 index 000000000..9cbca25ce --- /dev/null +++ b/gramps/plugins/database/bsddb.py @@ -0,0 +1,3 @@ + +from bsddb_support import DbBsddb + diff --git a/gramps/plugins/database/bsddb_support/__init__.py b/gramps/plugins/database/bsddb_support/__init__.py new file mode 100644 index 000000000..524ecb660 --- /dev/null +++ b/gramps/plugins/database/bsddb_support/__init__.py @@ -0,0 +1,96 @@ +# +# Gramps - a GTK+/GNOME based genealogy program +# +# Copyright (C) 2000-2007 Donald N. Allingham +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# + +""" +Gramps Database API. + +Database Architecture +===================== + +Access to the database is made through Python classes. Exactly +what functionality you have is dependent on the properties of the +database. For example, if you are accessing a read-only view, then +you will only have access to a subset of the methods available. + +At the root of any database interface is either :py:class:`.DbReadBase` and/or +:py:class:`.DbWriteBase`. These define the methods to read and write to a +database, respectively. + +The full database hierarchy is: + +- :py:class:`.DbBsddb` - read and write implementation to BSDDB databases + + * :py:class:`.DbWriteBase` - virtual and implementation-independent methods + for reading data + + * :py:class:`.DbBsddbRead` - read-only (accessors, getters) implementation + to BSDDB databases + + + :py:class:`.DbReadBase` - virtual and implementation-independent + methods for reading data + + + :py:class:`.Callback` - callback and signal functions + + * :py:class:`.UpdateCallback` - callback functionality + +- :py:class:`.DbDjango` - read and write implementation to Django-based + databases + + * :py:class:`.DbWriteBase` - virtual and implementation-independent methods + for reading data + + * :py:class:`.DbReadBase` - virtual and implementation-independent methods + for reading data + +DbBsddb +======= + +The :py:class:`.DbBsddb` interface defines a hierarchical database +(non-relational) written in +`PyBSDDB `_. There is no +such thing as a database schema, and the meaning of the data is +defined in the Python classes above. The data is stored as pickled +tuples and unserialized into the primary data types (below). + +DbDjango +======== + +The DbDjango interface defines the Gramps data in terms of +*models* and *relations* from the +`Django project `_. The database +backend can be any implementation that supports Django, including +such popular SQL implementations as sqlite, MySQL, Postgresql, and +Oracle. The data is retrieved from the SQL fields, serialized and +then unserialized into the primary data types (below). + +More details can be found in the manual's +`Using database API `_. +""" + +from gramps.gen.db.base import * +from gramps.gen.db.dbconst import * +from .cursor import * +from .read import * +from .bsddbtxn import * +from gramps.gen.db.txn import * +from .undoredo import * +from gramps.gen.db.exceptions import * +from .write import * +from .backup import backup, restore diff --git a/gramps/plugins/database/bsddb_support/backup.py b/gramps/plugins/database/bsddb_support/backup.py new file mode 100644 index 000000000..29dd46c54 --- /dev/null +++ b/gramps/plugins/database/bsddb_support/backup.py @@ -0,0 +1,213 @@ +# +# Gramps - a GTK+/GNOME based genealogy program +# +# Copyright (C) 2007 Donald N. Allingham +# Copyright (C) 2011 Tim G L Lyons +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# gen/db/backup.py + +""" +Description +=========== + +This module Provides backup and restore functions for a database. The +backup function saves the data into backup files, while the restore +function loads the data back into a database. + +You should only restore the data into an empty database. + +Implementation +============== + +Not all of the database tables need to be backed up, since many are +automatically generated from the others. The tables that are backed up +are the primary tables and the metadata table. + +The database consists of a table of "pickled" tuples. Each of the +primary tables is "walked", and the pickled tuple is extracted, and +written to the backup file. + +Restoring the data is just as simple. The backup file is parsed an +entry at a time, and inserted into the associated database table. The +derived tables are built automatically as the items are entered into +db. +""" + +#------------------------------------------------------------------------- +# +# load standard python libraries +# +#------------------------------------------------------------------------- +import os +import pickle + +#------------------------------------------------------------------------ +# +# Gramps libs +# +#------------------------------------------------------------------------ +from gramps.gen.db.exceptions import DbException +from .write import FAMILY_TBL, PLACES_TBL, SOURCES_TBL, MEDIA_TBL, \ + EVENTS_TBL, PERSON_TBL, REPO_TBL, NOTE_TBL, TAG_TBL, META, CITATIONS_TBL + +#------------------------------------------------------------------------ +# +# Set up logging +# +#------------------------------------------------------------------------ +import logging +LOG = logging.getLogger(".Backup") + +def backup(database): + """ + Exports the database to a set of backup files. These files consist + of the pickled database tables, one file for each table. + + The heavy lifting is done by the private :py:func:`__do__export` function. + The purpose of this function is to catch any exceptions that occur. + + :param database: database instance to backup + :type database: DbDir + """ + try: + __do_export(database) + except (OSError, IOError) as msg: + raise DbException(str(msg)) + +def __mk_backup_name(database, base): + """ + Return the backup name of the database table + + :param database: database instance + :type database: DbDir + :param base: base name of the table + :type base: str + """ + return os.path.join(database.get_save_path(), base + ".gbkp") + +def __mk_tmp_name(database, base): + """ + Return the temporary backup name of the database table + + :param database: database instance + :type database: DbDir + :param base: base name of the table + :type base: str + """ + return os.path.join(database.get_save_path(), base + ".gbkp.new") + +def __do_export(database): + """ + Loop through each table of the database, saving the pickled data + a file. + + :param database: database instance to backup + :type database: DbDir + """ + try: + for (base, tbl) in __build_tbl_map(database): + backup_name = __mk_tmp_name(database, base) + backup_table = open(backup_name, 'wb') + + cursor = tbl.cursor() + data = cursor.first() + while data: + pickle.dump(data, backup_table, 2) + data = cursor.next() + cursor.close() + backup_table.close() + except (IOError,OSError): + return + + for (base, tbl) in __build_tbl_map(database): + new_name = __mk_backup_name(database, base) + old_name = __mk_tmp_name(database, base) + if os.path.isfile(new_name): + os.unlink(new_name) + os.rename(old_name, new_name) + +def restore(database): + """ + Restores the database to a set of backup files. These files consist + of the pickled database tables, one file for each table. + + The heavy lifting is done by the private :py:func:`__do__restore` function. + The purpose of this function is to catch any exceptions that occur. + + :param database: database instance to restore + :type database: DbDir + """ + try: + __do_restore(database) + except (OSError, IOError) as msg: + raise DbException(str(msg)) + +def __do_restore(database): + """ + Loop through each table of the database, restoring the pickled data + to the appropriate database file. + + :param database: database instance to backup + :type database: DbDir + """ + for (base, tbl) in __build_tbl_map(database): + backup_name = __mk_backup_name(database, base) + backup_table = open(backup_name, 'rb') + __load_tbl_txn(database, backup_table, tbl) + + database.rebuild_secondary() + +def __load_tbl_txn(database, backup_table, tbl): + """ + Return the temporary backup name of the database table + + :param database: database instance + :type database: DbDir + :param backup_table: file containing the backup data + :type backup_table: file + :param tbl: Berkeley db database table + :type tbl: Berkeley db database table + """ + try: + while True: + data = pickle.load(backup_table) + txn = database.env.txn_begin() + tbl.put(data[0], data[1], txn=txn) + txn.commit() + except EOFError: + backup_table.close() + +def __build_tbl_map(database): + """ + Builds a table map of names to database tables. + + :param database: database instance to backup + :type database: DbDir + """ + return [ + ( PERSON_TBL, database.person_map.db), + ( FAMILY_TBL, database.family_map.db), + ( PLACES_TBL, database.place_map.db), + ( SOURCES_TBL, database.source_map.db), + ( CITATIONS_TBL, database.citation_map.db), + ( REPO_TBL, database.repository_map.db), + ( NOTE_TBL, database.note_map.db), + ( MEDIA_TBL, database.media_map.db), + ( EVENTS_TBL, database.event_map.db), + ( TAG_TBL, database.tag_map.db), + ( META, database.metadata.db), + ] diff --git a/gramps/gen/db/bsddbtxn.py b/gramps/plugins/database/bsddb_support/bsddbtxn.py similarity index 100% rename from gramps/gen/db/bsddbtxn.py rename to gramps/plugins/database/bsddb_support/bsddbtxn.py diff --git a/gramps/gen/db/cursor.py b/gramps/plugins/database/bsddb_support/cursor.py similarity index 100% rename from gramps/gen/db/cursor.py rename to gramps/plugins/database/bsddb_support/cursor.py diff --git a/gramps/gen/db/dictionary.py b/gramps/plugins/database/bsddb_support/dictionary.py similarity index 98% rename from gramps/gen/db/dictionary.py rename to gramps/plugins/database/bsddb_support/dictionary.py index 36a8cd5fa..2d785ef6e 100644 --- a/gramps/gen/db/dictionary.py +++ b/gramps/plugins/database/bsddb_support/dictionary.py @@ -40,21 +40,18 @@ from . import (PERSON_KEY, NOTE_KEY, TAG_KEY) -from ..const import GRAMPS_LOCALE as glocale -_ = glocale.translation.gettext -from ..errors import DbError -from ..utils.id import create_id -from ..lib.researcher import Researcher -from ..lib.mediaobj import MediaObject -from ..lib.person import Person -from ..lib.family import Family -from ..lib.src import Source -from ..lib.citation import Citation -from ..lib.event import Event -from ..lib.place import Place -from ..lib.repo import Repository -from ..lib.note import Note -from ..lib.tag import Tag +from gramps.gen.utils.id import create_id +from gramps.gen.lib.researcher import Researcher +from gramps.gen.lib.mediaobj import MediaObject +from gramps.gen.lib.person import Person +from gramps.gen.lib.family import Family +from gramps.gen.lib.src import Source +from gramps.gen.lib.citation import Citation +from gramps.gen.lib.event import Event +from gramps.gen.lib.place import Place +from gramps.gen.lib.repo import Repository +from gramps.gen.lib.note import Note +from gramps.gen.lib.tag import Tag class Cursor(object): """ diff --git a/gramps/gen/db/read.py b/gramps/plugins/database/bsddb_support/read.py similarity index 98% rename from gramps/gen/db/read.py rename to gramps/plugins/database/bsddb_support/read.py index 87dc112d6..d51810581 100644 --- a/gramps/gen/db/read.py +++ b/gramps/plugins/database/bsddb_support/read.py @@ -53,30 +53,32 @@ import logging # GRAMPS libraries # #------------------------------------------------------------------------- -from ..lib.mediaobj import MediaObject -from ..lib.person import Person -from ..lib.family import Family -from ..lib.src import Source -from ..lib.citation import Citation -from ..lib.event import Event -from ..lib.place import Place -from ..lib.repo import Repository -from ..lib.note import Note -from ..lib.tag import Tag -from ..lib.genderstats import GenderStats -from ..lib.researcher import Researcher -from ..lib.nameorigintype import NameOriginType +from gramps.gen.lib.mediaobj import MediaObject +from gramps.gen.lib.person import Person +from gramps.gen.lib.family import Family +from gramps.gen.lib.src import Source +from gramps.gen.lib.citation import Citation +from gramps.gen.lib.event import Event +from gramps.gen.lib.place import Place +from gramps.gen.lib.repo import Repository +from gramps.gen.lib.note import Note +from gramps.gen.lib.tag import Tag +from gramps.gen.lib.genderstats import GenderStats +from gramps.gen.lib.researcher import Researcher +from gramps.gen.lib.nameorigintype import NameOriginType -from .dbconst import * -from ..utils.callback import Callback -from ..utils.cast import conv_dbstr_to_unicode -from . import (BsddbBaseCursor, DbReadBase) -from ..utils.id import create_id -from ..errors import DbError -from ..constfunc import handle2internal, get_env_var -from ..const import GRAMPS_LOCALE as glocale +from gramps.gen.utils.callback import Callback +from gramps.gen.utils.cast import conv_dbstr_to_unicode +from . import BsddbBaseCursor +from gramps.gen.db.base import DbReadBase +from gramps.gen.utils.id import create_id +from gramps.gen.errors import DbError +from gramps.gen.constfunc import handle2internal, get_env_var +from gramps.gen.const import GRAMPS_LOCALE as glocale _ = glocale.translation.gettext +from gramps.gen.db.dbconst import * + LOG = logging.getLogger(DBLOGNAME) LOG = logging.getLogger(".citation") #------------------------------------------------------------------------- @@ -84,7 +86,6 @@ LOG = logging.getLogger(".citation") # constants # #------------------------------------------------------------------------- -from .dbconst import * _SIGBASE = ('person', 'family', 'source', 'citation', 'event', 'media', 'place', 'repository', diff --git a/gramps/gen/db/undoredo.py b/gramps/plugins/database/bsddb_support/undoredo.py similarity index 98% rename from gramps/gen/db/undoredo.py rename to gramps/plugins/database/bsddb_support/undoredo.py index 7fed4d876..da6b3368e 100644 --- a/gramps/gen/db/undoredo.py +++ b/gramps/plugins/database/bsddb_support/undoredo.py @@ -43,7 +43,7 @@ except: DBPageNotFoundError = 0 DBInvalidArgError = 0 -from ..const import GRAMPS_LOCALE as glocale +from gramps.gen.const import GRAMPS_LOCALE as glocale _ = glocale.translation.gettext #------------------------------------------------------------------------- @@ -51,10 +51,10 @@ _ = glocale.translation.gettext # Gramps modules # #------------------------------------------------------------------------- -from ..constfunc import conv_to_unicode, handle2internal, win -from .dbconst import * +from gramps.gen.constfunc import conv_to_unicode, handle2internal, win +from gramps.gen.db.dbconst import * from . import BSDDBTxn -from ..errors import DbError +from gramps.gen.errors import DbError #------------------------------------------------------------------------- # diff --git a/gramps/gen/db/upgrade.py b/gramps/plugins/database/bsddb_support/upgrade.py similarity index 98% rename from gramps/gen/db/upgrade.py rename to gramps/plugins/database/bsddb_support/upgrade.py index c720a9547..1f1cd6719 100644 --- a/gramps/gen/db/upgrade.py +++ b/gramps/plugins/database/bsddb_support/upgrade.py @@ -39,23 +39,24 @@ from bsddb3 import db # Gramps modules # #------------------------------------------------------------------------- -from ..const import GRAMPS_LOCALE as glocale +from gramps.gen.const import GRAMPS_LOCALE as glocale _ = glocale.translation.gettext -from ..constfunc import handle2internal -from ..lib.markertype import MarkerType -from ..lib.nameorigintype import NameOriginType -from ..lib.place import Place -from ..lib.placeref import PlaceRef -from ..lib.placetype import PlaceType -from ..lib.placename import PlaceName -from ..lib.eventtype import EventType -from ..lib.tag import Tag -from ..utils.file import create_checksum -from ..utils.id import create_id +from gramps.gen.constfunc import handle2internal +from gramps.gen.lib.markertype import MarkerType +from gramps.gen.lib.nameorigintype import NameOriginType +from gramps.gen.lib.place import Place +from gramps.gen.lib.placeref import PlaceRef +from gramps.gen.lib.placetype import PlaceType +from gramps.gen.lib.placename import PlaceName +from gramps.gen.lib.eventtype import EventType +from gramps.gen.lib.tag import Tag +from gramps.gen.utils.file import create_checksum +from gramps.gen.utils.id import create_id from . import BSDDBTxn from .write import _mkname, SURNAMES -from .dbconst import (PERSON_KEY, FAMILY_KEY, EVENT_KEY, - MEDIA_KEY, PLACE_KEY, REPOSITORY_KEY, SOURCE_KEY) +from gramps.gen.db.dbconst import (PERSON_KEY, FAMILY_KEY, EVENT_KEY, + MEDIA_KEY, PLACE_KEY, REPOSITORY_KEY, + SOURCE_KEY) from gramps.gui.dialog import (InfoDialog) LOG = logging.getLogger(".upgrade") @@ -359,7 +360,7 @@ def upgrade_datamap_17(datamap): """ new_srcattr_list = [] private = False - from ..lib.srcattrtype import SrcAttributeType + from gramps.gen.lib.srcattrtype import SrcAttributeType for (key, value) in datamap.items(): the_type = SrcAttributeType(key).serialize() new_srcattr_list.append((private, the_type, value)) diff --git a/gramps/gen/db/write.py b/gramps/plugins/database/bsddb_support/write.py similarity index 99% rename from gramps/gen/db/write.py rename to gramps/plugins/database/bsddb_support/write.py index 607735ec8..768db02d9 100644 --- a/gramps/gen/db/write.py +++ b/gramps/plugins/database/bsddb_support/write.py @@ -56,33 +56,34 @@ except: # Gramps modules # #------------------------------------------------------------------------- -from ..lib.person import Person -from ..lib.family import Family -from ..lib.src import Source -from ..lib.citation import Citation -from ..lib.event import Event -from ..lib.place import Place -from ..lib.repo import Repository -from ..lib.mediaobj import MediaObject -from ..lib.note import Note -from ..lib.tag import Tag -from ..lib.genderstats import GenderStats -from ..lib.researcher import Researcher +from gramps.gen.lib.person import Person +from gramps.gen.lib.family import Family +from gramps.gen.lib.src import Source +from gramps.gen.lib.citation import Citation +from gramps.gen.lib.event import Event +from gramps.gen.lib.place import Place +from gramps.gen.lib.repo import Repository +from gramps.gen.lib.mediaobj import MediaObject +from gramps.gen.lib.note import Note +from gramps.gen.lib.tag import Tag +from gramps.gen.lib.genderstats import GenderStats +from gramps.gen.lib.researcher import Researcher from . import (DbBsddbRead, DbWriteBase, BSDDBTxn, DbTxn, BsddbBaseCursor, BsddbDowngradeError, DbVersionError, DbEnvironmentError, DbUpgradeRequiredError, find_surname, - find_byte_surname, find_surname_name, DbUndoBSDDB as DbUndo, - exceptions) -from .dbconst import * -from ..utils.callback import Callback -from ..utils.cast import conv_dbstr_to_unicode -from ..utils.id import create_id -from ..updatecallback import UpdateCallback -from ..errors import DbError -from ..constfunc import (win, conv_to_unicode, handle2internal, + find_byte_surname, find_surname_name, DbUndoBSDDB as DbUndo) + +from gramps.gen.db import exceptions +from gramps.gen.db.dbconst import * +from gramps.gen.utils.callback import Callback +from gramps.gen.utils.cast import conv_dbstr_to_unicode +from gramps.gen.utils.id import create_id +from gramps.gen.updatecallback import UpdateCallback +from gramps.gen.errors import DbError +from gramps.gen.constfunc import (win, conv_to_unicode, handle2internal, get_env_var) -from ..const import HOME_DIR, GRAMPS_LOCALE as glocale +from gramps.gen.const import HOME_DIR, GRAMPS_LOCALE as glocale _ = glocale.translation.gettext _LOG = logging.getLogger(DBLOGNAME)