Database backend as a plugin: this set of changes moves most or

all of Bsddb from gramps.gen.db to gramps.plugins.database. The
id of the plugin is 'bsddb' which can be loaded using the
make_database(id, dbstate) API (for now).

Next step is to add an identifying text in the directory to
indicate which database backend to use.

gramps/cli/arghandler.py

@@ -44,7 +44,7 @@ import sys
 #-------------------------------------------------------------------------
 from gramps.gen.recentfiles import recent_files
 from gramps.gen.utils.file import rm_tempdir, get_empty_tempdir
-from gramps.gen.db import DbBsddb
+from gramps.gen.db import make_database
 from .clidbman import CLIDbManager, NAME_FILE, find_locker_name
 
 from gramps.gen.plug import BasePluginManager
@@ -491,7 +491,8 @@ class ArgHandler(object):
                     self.imp_db_path, title = self.dbman.create_new_db_cli()
                 else:
                     self.imp_db_path = get_empty_tempdir("import_dbdir")
-                    newdb = DbBsddb()
+
+                    newdb = make_database("bsddb", self.dbstate)
                     newdb.write_version(self.imp_db_path)
                 
                 try:

gramps/cli/clidbman.py

@@ -54,7 +54,7 @@ _LOG = logging.getLogger(DBLOGNAME)
 #-------------------------------------------------------------------------
 from gramps.gen.const import GRAMPS_LOCALE as glocale
 _ = glocale.translation.gettext
-from gramps.gen.db import DbBsddb
+from gramps.gen.db import make_database
 from gramps.gen.plug import BasePluginManager
 from gramps.gen.config import config
 from gramps.gen.constfunc import win, conv_to_unicode
@@ -294,7 +294,8 @@ class CLIDbManager(object):
 
         if create_db:
             # write the version number into metadata
-            newdb = DbBsddb()
+            
+            newdb = make_database("bsddb", self.dbstate)
             newdb.write_version(new_path)
 
         (tval, last) = time_val(new_path)
@@ -360,8 +361,8 @@ class CLIDbManager(object):
     
                 # Create a new database
                 self.__start_cursor(_("Importing data..."))
-                dbclass = DbBsddb
+
-                dbase = dbclass()
+                dbase = make_database("bsddb", self.dbstate)
                 dbase.load(new_path, user.callback)
     
                 import_function = plugin.get_import_function()

gramps/cli/grampscli.py

@@ -49,7 +49,7 @@ from gramps.gen.config import config
 from gramps.gen.const import PLUGINS_DIR, USER_PLUGINS
 from gramps.gen.errors import DbError
 from gramps.gen.dbstate import DbState
-from gramps.gen.db import DbBsddb
+from gramps.gen.db import make_database
 from gramps.gen.db.exceptions import (DbUpgradeRequiredError, 
                                       BsddbDowngradeError, 
                                       DbVersionError, 
@@ -152,9 +152,9 @@ class CLIDbLoader(object):
         else:
             mode = 'w'
 
-        dbclass = DbBsddb
+        db = make_database("bsddb", self.dbstate)
         
-        self.dbstate.change_database(dbclass())
+        self.dbstate.change_database(db)
         self.dbstate.db.disable_signals()
 
         self._begin_progress()

gramps/gen/db/__init__.py

@@ -86,11 +86,26 @@ More details can be found in the manual's
 
 from .base import *
 from .dbconst import *
-from .cursor import *
+#from .cursor import *
-from .read import *
+#from .read import *
-from .bsddbtxn import *
+#from .bsddbtxn import *
 from .txn import *
-from .undoredo import *
+#from .undoredo import *
 from .exceptions import *
-from .write import *
+#from .write import *
-from .backup import backup, restore
+#from .backup import backup, restore
+
+def make_database(id, dbstate):
+    from gramps.cli.grampscli import CLIManager
+    from gramps.gen.plug import BasePluginManager
+    from gramps.cli.user import User
+
+    climanager = CLIManager(dbstate, setloader=False, user=User()) # do not load db_loader
+    climanager.do_reg_plugins(dbstate, None)
+    pmgr = BasePluginManager.get_instance()
+    pdata = pmgr.get_plugin(id)
+
+    mod = pmgr.load_plugin(pdata)
+    database = getattr(mod, pdata.databaseclass)
+    return database()
+

gramps/gen/db/backup.py

@@ -1,213 +1,8 @@
-#
-# Gramps - a GTK+/GNOME based genealogy program
-#
-# Copyright (C) 2007  Donald N. Allingham
-# Copyright (C) 2011       Tim G L Lyons
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-#
-# gen/db/backup.py
-
-"""
-Description
-===========
-
-This module Provides backup and restore functions for a database. The
-backup function saves the data into backup files, while the restore
-function loads the data back into a database.
-
-You should only restore the data into an empty database.
-
-Implementation
-==============
-
-Not all of the database tables need to be backed up, since many are
-automatically generated from the others. The tables that are backed up
-are the primary tables and the metadata table.
-
-The database consists of a table of "pickled" tuples. Each of the
-primary tables is "walked", and the pickled tuple is extracted, and
-written to the backup file.
-
-Restoring the data is just as simple. The backup file is parsed an
-entry at a time, and inserted into the associated database table. The
-derived tables are built automatically as the items are entered into
-db.
-"""
-
-#-------------------------------------------------------------------------
-#
-# load standard python libraries
-#
-#-------------------------------------------------------------------------
-import os
-import pickle
-
-#------------------------------------------------------------------------
-#
-# Gramps libs
-#
-#------------------------------------------------------------------------
-from .exceptions import DbException
-from .write import FAMILY_TBL, PLACES_TBL, SOURCES_TBL, MEDIA_TBL, \
-    EVENTS_TBL, PERSON_TBL, REPO_TBL, NOTE_TBL, TAG_TBL, META, CITATIONS_TBL
-
-#------------------------------------------------------------------------
-#
-# Set up logging
-#
-#------------------------------------------------------------------------
 import logging
 LOG = logging.getLogger(".Backup")
 
 def backup(database):
-    """
+    print("FIXME")
-    Exports the database to a set of backup files. These files consist
-    of the pickled database tables, one file for each table.
-
-    The heavy lifting is done by the private :py:func:`__do__export` function.
-    The purpose of this function is to catch any exceptions that occur.
-
-    :param database: database instance to backup
-    :type database: DbDir
-    """
-    try:
-        __do_export(database)
-    except (OSError, IOError) as msg:
-        raise DbException(str(msg))
-
-def __mk_backup_name(database, base):
-    """
-    Return the backup name of the database table
-
-    :param database: database instance 
-    :type database: DbDir
-    :param base: base name of the table
-    :type base: str
-    """
-    return os.path.join(database.get_save_path(), base + ".gbkp")
-
-def __mk_tmp_name(database, base):
-    """
-    Return the temporary backup name of the database table
-
-    :param database: database instance 
-    :type database: DbDir
-    :param base: base name of the table
-    :type base: str
-    """
-    return os.path.join(database.get_save_path(), base + ".gbkp.new")
-
-def __do_export(database):
-    """
-    Loop through each table of the database, saving the pickled data
-    a file.
-
-    :param database: database instance to backup
-    :type database: DbDir
-    """
-    try:
-        for (base, tbl) in __build_tbl_map(database):
-            backup_name = __mk_tmp_name(database, base)
-            backup_table = open(backup_name, 'wb')
-    
-            cursor = tbl.cursor()
-            data = cursor.first()
-            while data:
-                pickle.dump(data, backup_table, 2)
-                data = cursor.next()
-            cursor.close()
-            backup_table.close()
-    except (IOError,OSError):
-        return
-
-    for (base, tbl) in __build_tbl_map(database):
-        new_name = __mk_backup_name(database, base)
-        old_name = __mk_tmp_name(database, base)
-        if os.path.isfile(new_name):
-            os.unlink(new_name)
-        os.rename(old_name, new_name)
 
 def restore(database):
-    """
+    print("FIXME")
-    Restores the database to a set of backup files. These files consist
-    of the pickled database tables, one file for each table.
-
-    The heavy lifting is done by the private :py:func:`__do__restore` function.
-    The purpose of this function is to catch any exceptions that occur.
-
-    :param database: database instance to restore
-    :type database: DbDir
-    """
-    try:
-        __do_restore(database)
-    except (OSError, IOError) as msg:
-        raise DbException(str(msg))
-
-def __do_restore(database):
-    """
-    Loop through each table of the database, restoring the pickled data
-    to the appropriate database file.
-
-    :param database: database instance to backup
-    :type database: DbDir
-    """
-    for (base, tbl) in __build_tbl_map(database):
-        backup_name = __mk_backup_name(database, base)
-        backup_table = open(backup_name, 'rb')
-        __load_tbl_txn(database, backup_table, tbl)
-
-    database.rebuild_secondary()
-
-def __load_tbl_txn(database, backup_table, tbl):
-    """
-    Return the temporary backup name of the database table
-
-    :param database: database instance 
-    :type database: DbDir
-    :param backup_table: file containing the backup data
-    :type backup_table: file
-    :param tbl: Berkeley db database table
-    :type tbl: Berkeley db database table
-    """
-    try:
-        while True:
-            data = pickle.load(backup_table)
-            txn = database.env.txn_begin()
-            tbl.put(data[0], data[1], txn=txn)
-            txn.commit()
-    except EOFError:
-        backup_table.close()
-
-def __build_tbl_map(database):
-    """
-    Builds a table map of names to database tables.
-
-    :param database: database instance to backup
-    :type database: DbDir
-    """
-    return [
-        ( PERSON_TBL,  database.person_map.db),
-        ( FAMILY_TBL,  database.family_map.db),
-        ( PLACES_TBL,  database.place_map.db),
-        ( SOURCES_TBL, database.source_map.db),
-        ( CITATIONS_TBL, database.citation_map.db),
-        ( REPO_TBL,    database.repository_map.db),
-        ( NOTE_TBL,    database.note_map.db),
-        ( MEDIA_TBL,   database.media_map.db),
-        ( EVENTS_TBL,  database.event_map.db),
-        ( TAG_TBL,     database.tag_map.db),
-        ( META,        database.metadata.db),
-        ]

gramps/gen/dbstate.py

@@ -23,8 +23,8 @@
 Provide the database state class
 """
 
-from .db import DbBsddbRead
 from .db import DbReadBase
+from .db import make_database
 from .proxy.proxybase import ProxyDbBase
 from .utils.callback import Callback
 from .config import config
@@ -45,7 +45,7 @@ class DbState(Callback):
         just a place holder until a real DB is assigned.
         """
         Callback.__init__(self)
-        self.db      = DbBsddbRead()
+        self.db      = make_database("bsddb", self)
         self.open    = False
         self.stack = []
 
@@ -88,7 +88,7 @@ class DbState(Callback):
         """
         self.emit('no-database', ())
         self.db.close()
-        self.db = DbBsddbRead()
+        self.db = make_database("bsddb", self)
         self.db.db_is_open = False
         self.open = False
         self.emit('database-changed', (self.db, ))

gramps/gen/plug/_gramplet.py

@@ -70,7 +70,6 @@ class Gramplet(object):
         self.connect(self.gui.textview, "motion-notify-event", 
                      self.gui.on_motion)
         self.connect_signal('Person', self._active_changed)
-        
         self._db_changed(self.dbstate.db)
         active_person = self.get_active('Person')
         if active_person: # already changed

gramps/gen/plug/_manager.py

@@ -412,6 +412,11 @@ class BasePluginManager(object):
         """
         return self.__pgr.sidebar_plugins()
     
+    def get_reg_databases(self):
+        """ Return list of registered database backends
+        """
+        return self.__pgr.database_plugins()
+    
     def get_external_opt_dict(self):
         """ Return the dictionary of external options. """
         return self.__external_opt_dict

gramps/gen/plug/_pluginreg.py

@@ -70,8 +70,9 @@ VIEW        = 8
 RELCALC     = 9
 GRAMPLET    = 10
 SIDEBAR     = 11
+DATABASE    = 12
 PTYPE       = [REPORT , QUICKREPORT, TOOL, IMPORT, EXPORT, DOCGEN, GENERAL,
-               MAPSERVICE, VIEW, RELCALC, GRAMPLET, SIDEBAR]
+               MAPSERVICE, VIEW, RELCALC, GRAMPLET, SIDEBAR, DATABASE]
 PTYPE_STR   = {
         REPORT: _('Report') , 
         QUICKREPORT: _('Quickreport'), 
@@ -85,6 +86,7 @@ PTYPE_STR   = {
         RELCALC: _('Relationships'), 
         GRAMPLET: _('Gramplet'),
         SIDEBAR: _('Sidebar'),
+        DATABASE: _('Database'),
         }
 
 #possible report categories
@@ -206,7 +208,7 @@ class PluginData(object):
        The python path where the plugin implementation can be found
     .. attribute:: ptype
        The plugin type. One of REPORT , QUICKREPORT, TOOL, IMPORT,
-        EXPORT, DOCGEN, GENERAL, MAPSERVICE, VIEW, GRAMPLET
+        EXPORT, DOCGEN, GENERAL, MAPSERVICE, VIEW, GRAMPLET, DATABASE
     .. attribute:: authors
        List of authors of the plugin, default=[]
     .. attribute:: authors_email
@@ -349,6 +351,11 @@ class PluginData(object):
        the plugin is appended to the list of plugins. If START, then the
        plugin is prepended. Only set START if you want a plugin to be the
        first in the order of plugins
+
+    Attributes for DATABASE plugins
+
+    .. attribute:: databaseclass
+       The class in the module that is the database class
     """
 
     def __init__(self):
@@ -421,6 +428,8 @@ class PluginData(object):
         self._menu_label = ''
         #VIEW and SIDEBAR attr
         self._order = END
+        #DATABASE attr
+        self._databaseclass = None
         #GENERAL attr
         self._data = []
         self._process = None
@@ -931,6 +940,17 @@ class PluginData(object):
 
     order = property(_get_order, _set_order)
   
+    #DATABASE attributes
+    def _set_databaseclass(self, databaseclass):
+        if not self._ptype == DATABASE:
+            raise ValueError('databaseclass may only be set for DATABASE plugins')
+        self._databaseclass = databaseclass
+
+    def _get_databaseclass(self):
+        return self._databaseclass
+
+    databaseclass = property(_get_databaseclass, _set_databaseclass)
+
     #GENERAL attr
     def _set_data(self, data):
         if not self._ptype in (GENERAL,):
@@ -1032,6 +1052,7 @@ def make_environment(**kwargs):
         'REPORT_MODE_CLI': REPORT_MODE_CLI,
         'TOOL_MODE_GUI': TOOL_MODE_GUI, 
         'TOOL_MODE_CLI': TOOL_MODE_CLI,
+        'DATABASE': DATABASE,
         'GRAMPSVERSION': GRAMPSVERSION,
         'START': START,
         'END': END,
@@ -1297,6 +1318,12 @@ class PluginRegister(object):
         """
         return self.type_plugins(SIDEBAR)
 
+    def database_plugins(self):
+        """
+        Return a list of :class:`PluginData` that are of type DATABASE
+        """
+        return self.type_plugins(DATABASE)
+
     def filter_load_on_reg(self):
         """
         Return a list of :class:`PluginData` that have load_on_reg == True

gramps/gen/utils/callman.py

@@ -303,6 +303,7 @@ class CallbackManager(object):
         Do a custom db connect signal outside of the primary object ones
         managed automatically.
         """
+        if self.database:
             self.custom_signal_keys.append(self.database.connect(name, callback))
     
     def __callbackcreator(self, signal, noarg=False):

gramps/gui/dbloader.py

@@ -55,7 +55,7 @@ from gramps.gen.const import GRAMPS_LOCALE as glocale
 _ = glocale.translation.gettext
 from gramps.cli.grampscli import CLIDbLoader
 from gramps.gen.config import config
-from gramps.gen.db import DbBsddb
+from gramps.gen.db import make_database
 from gramps.gen.db.exceptions import (DbUpgradeRequiredError, 
                                       BsddbDowngradeError, 
                                       DbVersionError, 
@@ -305,7 +305,7 @@ class DbLoader(CLIDbLoader):
         else:
             mode = 'w'
 
-        db = DbBsddb()
+        db = make_database("bsddb", self.dbstate)
         db.disable_signals()
         self.dbstate.no_database()
 

gramps/gui/dbman.py

@@ -73,7 +73,7 @@ _ = glocale.translation.gettext
 from gramps.gen.const import URL_WIKISTRING
 from .user import User
 from .dialog import ErrorDialog, QuestionDialog, QuestionDialog2
-from gramps.gen.db import DbBsddb
+from gramps.gen.db import make_database
 from .pluginmanager import GuiPluginManager
 from gramps.cli.clidbman import CLIDbManager, NAME_FILE, time_val
 from .ddtargets import DdTargets
@@ -531,8 +531,8 @@ class DbManager(CLIDbManager):
         new_path, newname = self._create_new_db("%s : %s" % (parent_name, name))
         
         self.__start_cursor(_("Extracting archive..."))
-        dbclass = DbBsddb
+
-        dbase = dbclass()
+        dbase = make_database("bsddb", self.dbstate)
         dbase.load(new_path, None)
         
         self.__start_cursor(_("Importing archive..."))
@@ -719,11 +719,10 @@ class DbManager(CLIDbManager):
                 fname = os.path.join(dirname, filename)
                 os.unlink(fname)
 
-        newdb = DbBsddb()
+        newdb = make_database("bsddb", self.dbstate)
         newdb.write_version(dirname)
 
-        dbclass = DbBsddb
+        dbase = make_database("bsddb", self.dbstate)
-        dbase = dbclass()
         dbase.set_save_path(dirname)
         dbase.load(dirname, None)
 

gramps/gui/pluginmanager.py

@@ -205,7 +205,12 @@ class GuiPluginManager(Callback):
         return [plg for plg in self.basemgr.get_reg_docgens()
                                 if plg.id not in self.__hidden_plugins]
 
+    def get_reg_databases(self):
+        """ Return list of non hidden registered database backends
+        """
+        return [plg for plg in self.basemgr.get_reg_databases()
+                                if plg.id not in self.__hidden_plugins]
+
     def get_reg_general(self, category=None):
         return [plg for plg in self.basemgr.get_reg_general(category)
                                 if plg.id not in self.__hidden_plugins]
-

gramps/plugins/database/bsddb.gpr.py

@@ -0,0 +1,31 @@
+#
+# Gramps - a GTK+/GNOME based genealogy program
+#
+# Copyright (C) 2015 Douglas Blank <doug.blank@gmail.com>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+
+plg = newplugin()
+plg.id    = 'bsddb'
+plg.name  = _("BSDDB Database Backend")
+plg.name_accell  = _("_BSDDB Database Backend")
+plg.description =  _("Berkeley Software Distribution Database Backend")
+plg.version = '1.0'
+plg.gramps_target_version = "4.2"
+plg.status = STABLE
+plg.fname = 'bsddb.py'
+plg.ptype = DATABASE
+plg.databaseclass = 'DbBsddb'

gramps/plugins/database/bsddb.py

@@ -0,0 +1,3 @@
+
+from bsddb_support import DbBsddb
+

gramps/plugins/database/bsddb_support/__init__.py

@@ -0,0 +1,96 @@
+#
+# Gramps - a GTK+/GNOME based genealogy program
+#
+# Copyright (C) 2000-2007  Donald N. Allingham
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+
+"""
+Gramps Database API.
+
+Database Architecture
+=====================
+
+Access to the database is made through Python classes. Exactly
+what functionality you have is dependent on the properties of the
+database. For example, if you are accessing a read-only view, then
+you will only have access to a subset of the methods available.
+
+At the root of any database interface is either :py:class:`.DbReadBase` and/or
+:py:class:`.DbWriteBase`. These define the methods to read and write to a
+database, respectively.
+
+The full database hierarchy is:
+
+- :py:class:`.DbBsddb` - read and write implementation to BSDDB databases
+
+  * :py:class:`.DbWriteBase` - virtual and implementation-independent methods
+    for reading data
+
+  * :py:class:`.DbBsddbRead` - read-only (accessors, getters) implementation
+    to BSDDB databases
+
+    + :py:class:`.DbReadBase` - virtual and implementation-independent
+      methods for reading data
+
+    + :py:class:`.Callback` - callback and signal functions
+
+  * :py:class:`.UpdateCallback` - callback functionality
+
+- :py:class:`.DbDjango` - read and write implementation to Django-based
+  databases
+
+  * :py:class:`.DbWriteBase` - virtual and implementation-independent methods
+    for reading data
+
+  * :py:class:`.DbReadBase` - virtual and implementation-independent methods
+    for reading data
+
+DbBsddb
+=======
+
+The :py:class:`.DbBsddb` interface defines a hierarchical database
+(non-relational) written in
+`PyBSDDB <http://www.jcea.es/programacion/pybsddb.htm>`_. There is no
+such thing as a database schema, and the meaning of the data is
+defined in the Python classes above. The data is stored as pickled
+tuples and unserialized into the primary data types (below).
+
+DbDjango
+========
+
+The DbDjango interface defines the Gramps data in terms of
+*models* and *relations* from the
+`Django project <http://www.djangoproject.com/>`_. The database
+backend can be any implementation that supports Django, including
+such popular SQL implementations as sqlite, MySQL, Postgresql, and
+Oracle. The data is retrieved from the SQL fields, serialized and
+then unserialized into the primary data types (below).
+
+More details can be found in the manual's
+`Using database API <http://www.gramps-project.org/wiki/index.php?title=Using_database_API>`_.
+"""
+
+from gramps.gen.db.base import *
+from gramps.gen.db.dbconst import *
+from .cursor import *
+from .read import *
+from .bsddbtxn import *
+from gramps.gen.db.txn import *
+from .undoredo import *
+from gramps.gen.db.exceptions import *
+from .write import *
+from .backup import backup, restore

gramps/plugins/database/bsddb_support/backup.py

@@ -0,0 +1,213 @@
+#
+# Gramps - a GTK+/GNOME based genealogy program
+#
+# Copyright (C) 2007  Donald N. Allingham
+# Copyright (C) 2011       Tim G L Lyons
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# gen/db/backup.py
+
+"""
+Description
+===========
+
+This module Provides backup and restore functions for a database. The
+backup function saves the data into backup files, while the restore
+function loads the data back into a database.
+
+You should only restore the data into an empty database.
+
+Implementation
+==============
+
+Not all of the database tables need to be backed up, since many are
+automatically generated from the others. The tables that are backed up
+are the primary tables and the metadata table.
+
+The database consists of a table of "pickled" tuples. Each of the
+primary tables is "walked", and the pickled tuple is extracted, and
+written to the backup file.
+
+Restoring the data is just as simple. The backup file is parsed an
+entry at a time, and inserted into the associated database table. The
+derived tables are built automatically as the items are entered into
+db.
+"""
+
+#-------------------------------------------------------------------------
+#
+# load standard python libraries
+#
+#-------------------------------------------------------------------------
+import os
+import pickle
+
+#------------------------------------------------------------------------
+#
+# Gramps libs
+#
+#------------------------------------------------------------------------
+from gramps.gen.db.exceptions import DbException
+from .write import FAMILY_TBL, PLACES_TBL, SOURCES_TBL, MEDIA_TBL, \
+    EVENTS_TBL, PERSON_TBL, REPO_TBL, NOTE_TBL, TAG_TBL, META, CITATIONS_TBL
+
+#------------------------------------------------------------------------
+#
+# Set up logging
+#
+#------------------------------------------------------------------------
+import logging
+LOG = logging.getLogger(".Backup")
+
+def backup(database):
+    """
+    Exports the database to a set of backup files. These files consist
+    of the pickled database tables, one file for each table.
+
+    The heavy lifting is done by the private :py:func:`__do__export` function.
+    The purpose of this function is to catch any exceptions that occur.
+
+    :param database: database instance to backup
+    :type database: DbDir
+    """
+    try:
+        __do_export(database)
+    except (OSError, IOError) as msg:
+        raise DbException(str(msg))
+
+def __mk_backup_name(database, base):
+    """
+    Return the backup name of the database table
+
+    :param database: database instance 
+    :type database: DbDir
+    :param base: base name of the table
+    :type base: str
+    """
+    return os.path.join(database.get_save_path(), base + ".gbkp")
+
+def __mk_tmp_name(database, base):
+    """
+    Return the temporary backup name of the database table
+
+    :param database: database instance 
+    :type database: DbDir
+    :param base: base name of the table
+    :type base: str
+    """
+    return os.path.join(database.get_save_path(), base + ".gbkp.new")
+
+def __do_export(database):
+    """
+    Loop through each table of the database, saving the pickled data
+    a file.
+
+    :param database: database instance to backup
+    :type database: DbDir
+    """
+    try:
+        for (base, tbl) in __build_tbl_map(database):
+            backup_name = __mk_tmp_name(database, base)
+            backup_table = open(backup_name, 'wb')
+    
+            cursor = tbl.cursor()
+            data = cursor.first()
+            while data:
+                pickle.dump(data, backup_table, 2)
+                data = cursor.next()
+            cursor.close()
+            backup_table.close()
+    except (IOError,OSError):
+        return
+
+    for (base, tbl) in __build_tbl_map(database):
+        new_name = __mk_backup_name(database, base)
+        old_name = __mk_tmp_name(database, base)
+        if os.path.isfile(new_name):
+            os.unlink(new_name)
+        os.rename(old_name, new_name)
+
+def restore(database):
+    """
+    Restores the database to a set of backup files. These files consist
+    of the pickled database tables, one file for each table.
+
+    The heavy lifting is done by the private :py:func:`__do__restore` function.
+    The purpose of this function is to catch any exceptions that occur.
+
+    :param database: database instance to restore
+    :type database: DbDir
+    """
+    try:
+        __do_restore(database)
+    except (OSError, IOError) as msg:
+        raise DbException(str(msg))
+
+def __do_restore(database):
+    """
+    Loop through each table of the database, restoring the pickled data
+    to the appropriate database file.
+
+    :param database: database instance to backup
+    :type database: DbDir
+    """
+    for (base, tbl) in __build_tbl_map(database):
+        backup_name = __mk_backup_name(database, base)
+        backup_table = open(backup_name, 'rb')
+        __load_tbl_txn(database, backup_table, tbl)
+
+    database.rebuild_secondary()
+
+def __load_tbl_txn(database, backup_table, tbl):
+    """
+    Return the temporary backup name of the database table
+
+    :param database: database instance 
+    :type database: DbDir
+    :param backup_table: file containing the backup data
+    :type backup_table: file
+    :param tbl: Berkeley db database table
+    :type tbl: Berkeley db database table
+    """
+    try:
+        while True:
+            data = pickle.load(backup_table)
+            txn = database.env.txn_begin()
+            tbl.put(data[0], data[1], txn=txn)
+            txn.commit()
+    except EOFError:
+        backup_table.close()
+
+def __build_tbl_map(database):
+    """
+    Builds a table map of names to database tables.
+
+    :param database: database instance to backup
+    :type database: DbDir
+    """
+    return [
+        ( PERSON_TBL,  database.person_map.db),
+        ( FAMILY_TBL,  database.family_map.db),
+        ( PLACES_TBL,  database.place_map.db),
+        ( SOURCES_TBL, database.source_map.db),
+        ( CITATIONS_TBL, database.citation_map.db),
+        ( REPO_TBL,    database.repository_map.db),
+        ( NOTE_TBL,    database.note_map.db),
+        ( MEDIA_TBL,   database.media_map.db),
+        ( EVENTS_TBL,  database.event_map.db),
+        ( TAG_TBL,     database.tag_map.db),
+        ( META,        database.metadata.db),
+        ]

gramps/plugins/database/bsddb_support/dictionary.py

@@ -40,21 +40,18 @@ from . import (PERSON_KEY,
                NOTE_KEY,
                TAG_KEY)
 
-from ..const import GRAMPS_LOCALE as glocale
+from gramps.gen.utils.id import create_id
-_ = glocale.translation.gettext
+from gramps.gen.lib.researcher import Researcher
-from ..errors import DbError
+from gramps.gen.lib.mediaobj import MediaObject
-from ..utils.id import create_id
+from gramps.gen.lib.person import Person
-from ..lib.researcher import Researcher
+from gramps.gen.lib.family import Family
-from ..lib.mediaobj import MediaObject
+from gramps.gen.lib.src import Source
-from ..lib.person import Person
+from gramps.gen.lib.citation import Citation
-from ..lib.family import Family
+from gramps.gen.lib.event import Event
-from ..lib.src import Source
+from gramps.gen.lib.place import Place
-from ..lib.citation import Citation
+from gramps.gen.lib.repo import Repository
-from ..lib.event import Event
+from gramps.gen.lib.note import Note
-from ..lib.place import Place
+from gramps.gen.lib.tag import Tag
-from ..lib.repo import Repository
-from ..lib.note import Note
-from ..lib.tag import Tag
 
 class Cursor(object):
     """

gramps/plugins/database/bsddb_support/read.py

@@ -53,30 +53,32 @@ import logging
 # GRAMPS libraries
 #
 #-------------------------------------------------------------------------
-from ..lib.mediaobj import MediaObject
+from gramps.gen.lib.mediaobj import MediaObject
-from ..lib.person import Person
+from gramps.gen.lib.person import Person
-from ..lib.family import Family
+from gramps.gen.lib.family import Family
-from ..lib.src import Source
+from gramps.gen.lib.src import Source
-from ..lib.citation import Citation
+from gramps.gen.lib.citation import Citation
-from ..lib.event import Event
+from gramps.gen.lib.event import Event
-from ..lib.place import Place
+from gramps.gen.lib.place import Place
-from ..lib.repo import Repository
+from gramps.gen.lib.repo import Repository
-from ..lib.note import Note
+from gramps.gen.lib.note import Note
-from ..lib.tag import Tag
+from gramps.gen.lib.tag import Tag
-from ..lib.genderstats import GenderStats
+from gramps.gen.lib.genderstats import GenderStats
-from ..lib.researcher import Researcher 
+from gramps.gen.lib.researcher import Researcher 
-from ..lib.nameorigintype import NameOriginType
+from gramps.gen.lib.nameorigintype import NameOriginType
 
-from .dbconst import *
+from gramps.gen.utils.callback import Callback
-from ..utils.callback import Callback
+from gramps.gen.utils.cast import conv_dbstr_to_unicode
-from ..utils.cast import conv_dbstr_to_unicode
+from . import BsddbBaseCursor
-from . import (BsddbBaseCursor, DbReadBase)
+from gramps.gen.db.base import DbReadBase
-from ..utils.id import create_id
+from gramps.gen.utils.id import create_id
-from ..errors import DbError
+from gramps.gen.errors import DbError
-from ..constfunc import handle2internal, get_env_var
+from gramps.gen.constfunc import handle2internal, get_env_var
-from ..const import GRAMPS_LOCALE as glocale
+from gramps.gen.const import GRAMPS_LOCALE as glocale
 _ = glocale.translation.gettext
 
+from gramps.gen.db.dbconst import *
+
 LOG = logging.getLogger(DBLOGNAME)
 LOG = logging.getLogger(".citation")
 #-------------------------------------------------------------------------
@@ -84,7 +86,6 @@ LOG = logging.getLogger(".citation")
 # constants
 #
 #-------------------------------------------------------------------------
-from .dbconst import *
 
 _SIGBASE = ('person', 'family', 'source', 'citation', 
             'event',  'media', 'place', 'repository',

gramps/plugins/database/bsddb_support/undoredo.py

@@ -43,7 +43,7 @@ except:
         DBPageNotFoundError = 0
         DBInvalidArgError = 0
         
-from ..const import GRAMPS_LOCALE as glocale
+from gramps.gen.const import GRAMPS_LOCALE as glocale
 _ = glocale.translation.gettext
 
 #-------------------------------------------------------------------------
@@ -51,10 +51,10 @@ _ = glocale.translation.gettext
 # Gramps modules
 #
 #-------------------------------------------------------------------------
-from ..constfunc import conv_to_unicode, handle2internal, win
+from gramps.gen.constfunc import conv_to_unicode, handle2internal, win
-from .dbconst import *
+from gramps.gen.db.dbconst import *
 from . import BSDDBTxn
-from ..errors import DbError
+from gramps.gen.errors import DbError
 
 #-------------------------------------------------------------------------
 #

gramps/plugins/database/bsddb_support/upgrade.py

@@ -39,23 +39,24 @@ from bsddb3 import db
 # Gramps modules
 #
 #-------------------------------------------------------------------------
-from ..const import GRAMPS_LOCALE as glocale
+from gramps.gen.const import GRAMPS_LOCALE as glocale
 _ = glocale.translation.gettext
-from ..constfunc import handle2internal
+from gramps.gen.constfunc import handle2internal
-from ..lib.markertype import MarkerType
+from gramps.gen.lib.markertype import MarkerType
-from ..lib.nameorigintype import NameOriginType
+from gramps.gen.lib.nameorigintype import NameOriginType
-from ..lib.place import Place
+from gramps.gen.lib.place import Place
-from ..lib.placeref import PlaceRef
+from gramps.gen.lib.placeref import PlaceRef
-from ..lib.placetype import PlaceType
+from gramps.gen.lib.placetype import PlaceType
-from ..lib.placename import PlaceName
+from gramps.gen.lib.placename import PlaceName
-from ..lib.eventtype import EventType
+from gramps.gen.lib.eventtype import EventType
-from ..lib.tag import Tag
+from gramps.gen.lib.tag import Tag
-from ..utils.file import create_checksum
+from gramps.gen.utils.file import create_checksum
-from ..utils.id import create_id
+from gramps.gen.utils.id import create_id
 from . import BSDDBTxn
 from .write import _mkname, SURNAMES
-from .dbconst import (PERSON_KEY, FAMILY_KEY, EVENT_KEY, 
+from gramps.gen.db.dbconst import (PERSON_KEY, FAMILY_KEY, EVENT_KEY, 
-                      MEDIA_KEY, PLACE_KEY, REPOSITORY_KEY, SOURCE_KEY)
+                                   MEDIA_KEY, PLACE_KEY, REPOSITORY_KEY, 
+                                   SOURCE_KEY)
 from gramps.gui.dialog import (InfoDialog)
 
 LOG = logging.getLogger(".upgrade")
@@ -359,7 +360,7 @@ def upgrade_datamap_17(datamap):
     """
     new_srcattr_list = []
     private = False
-    from ..lib.srcattrtype import SrcAttributeType
+    from gramps.gen.lib.srcattrtype import SrcAttributeType
     for (key, value) in datamap.items():
         the_type = SrcAttributeType(key).serialize()
         new_srcattr_list.append((private, the_type, value))

gramps/plugins/database/bsddb_support/write.py

@@ -56,33 +56,34 @@ except:
 # Gramps modules
 #
 #-------------------------------------------------------------------------
-from ..lib.person import Person
+from gramps.gen.lib.person import Person
-from ..lib.family import Family
+from gramps.gen.lib.family import Family
-from ..lib.src import Source
+from gramps.gen.lib.src import Source
-from ..lib.citation import Citation
+from gramps.gen.lib.citation import Citation
-from ..lib.event import Event
+from gramps.gen.lib.event import Event
-from ..lib.place import Place
+from gramps.gen.lib.place import Place
-from ..lib.repo import Repository
+from gramps.gen.lib.repo import Repository
-from ..lib.mediaobj import MediaObject
+from gramps.gen.lib.mediaobj import MediaObject
-from ..lib.note import Note
+from gramps.gen.lib.note import Note
-from ..lib.tag import Tag
+from gramps.gen.lib.tag import Tag
-from ..lib.genderstats import GenderStats
+from gramps.gen.lib.genderstats import GenderStats
-from ..lib.researcher import Researcher 
+from gramps.gen.lib.researcher import Researcher 
 
 from . import (DbBsddbRead, DbWriteBase, BSDDBTxn, 
                     DbTxn, BsddbBaseCursor, BsddbDowngradeError, DbVersionError,
                     DbEnvironmentError, DbUpgradeRequiredError, find_surname,
-                    find_byte_surname, find_surname_name, DbUndoBSDDB as DbUndo,
+                    find_byte_surname, find_surname_name, DbUndoBSDDB as DbUndo)
-                    exceptions)
+
-from .dbconst import *
+from gramps.gen.db import exceptions
-from ..utils.callback import Callback
+from gramps.gen.db.dbconst import *
-from ..utils.cast import conv_dbstr_to_unicode
+from gramps.gen.utils.callback import Callback
-from ..utils.id import create_id
+from gramps.gen.utils.cast import conv_dbstr_to_unicode
-from ..updatecallback import UpdateCallback
+from gramps.gen.utils.id import create_id
-from ..errors import DbError
+from gramps.gen.updatecallback import UpdateCallback
-from ..constfunc import (win, conv_to_unicode, handle2internal,
+from gramps.gen.errors import DbError
+from gramps.gen.constfunc import (win, conv_to_unicode, handle2internal,
                          get_env_var)
-from ..const import HOME_DIR, GRAMPS_LOCALE as glocale
+from gramps.gen.const import HOME_DIR, GRAMPS_LOCALE as glocale
 _ = glocale.translation.gettext
 
 _LOG = logging.getLogger(DBLOGNAME)