Only BSDDB plugin needs bsddb3; back/restore moved to db

gramps/cli/clidbman.py

@@ -137,57 +137,8 @@ class CLIDbManager(object):
         current DB.
         Returns ("Unknown", "Unknown", "Unknown") if invalid DB or other error.
         """
-        from bsddb3 import dbshelve, db
+        ## Maybe return the txt file contents, for now
-
+        return ("Unknown", "Unknown", "Unknown")
-        from gramps.gen.db import META, PERSON_TBL
-        from  gramps.gen.db.dbconst import BDBVERSFN
-
-        bdbversion_file = os.path.join(dirpath, BDBVERSFN)
-        if os.path.isfile(bdbversion_file):
-            vers_file = open(bdbversion_file)
-            bsddb_version = vers_file.readline().strip()
-        else:
-            return "Unknown", "Unknown", "Unknown"
-        
-        current_bsddb_version = str(db.version())
-        if bsddb_version != current_bsddb_version:
-            return "Unknown", bsddb_version, "Unknown"
-        
-        env = db.DBEnv()
-        flags = db.DB_CREATE | db.DB_PRIVATE |\
-            db.DB_INIT_MPOOL |\
-            db.DB_INIT_LOG | db.DB_INIT_TXN
-        try:
-            env.open(dirpath, flags)
-        except Exception as msg:
-            LOG.warning("Error opening db environment for '%s': %s" %
-                        (name, str(msg)))
-            try:
-                env.close()
-            except Exception as msg:
-                LOG.warning("Error closing db environment for '%s': %s" %
-                        (name, str(msg)))
-            return "Unknown", bsddb_version, "Unknown"
-        dbmap1 = dbshelve.DBShelf(env)
-        fname = os.path.join(dirpath, META + ".db")
-        try:
-            dbmap1.open(fname, META, db.DB_HASH, db.DB_RDONLY)
-        except:
-            env.close()
-            return "Unknown", bsddb_version, "Unknown"
-        schema_version = dbmap1.get(b'version', default=None)
-        dbmap1.close()
-        dbmap2 = dbshelve.DBShelf(env)
-        fname = os.path.join(dirpath, PERSON_TBL + ".db")
-        try:
-            dbmap2.open(fname, PERSON_TBL, db.DB_HASH, db.DB_RDONLY)
-        except:
-            env.close()
-            return "Unknown", bsddb_version, schema_version
-        count = len(dbmap2)
-        dbmap2.close()
-        env.close()
-        return (count, bsddb_version, schema_version)
 
     def family_tree_summary(self):
         """

gramps/gen/db/__init__.py

@@ -86,11 +86,5 @@ More details can be found in the manual's
 
 from .base import *
 from .dbconst import *
-#from .cursor import *
-#from .read import *
-#from .bsddbtxn import *
 from .txn import *
-#from .undoredo import *
 from .exceptions import *
-#from .write import *
-#from .backup import backup, restore

gramps/gen/db/backup.py

@@ -1,8 +0,0 @@
-import logging
-LOG = logging.getLogger(".Backup")
-
-def backup(database):
-    print("FIXME")
-
-def restore(database):
-    print("FIXME")

gramps/gen/db/dbconst.py

@@ -31,8 +31,7 @@ Declare constants used by database modules
 __all__ = (
             ('DBPAGE', 'DBMODE', 'DBCACHE', 'DBLOCKS', 'DBOBJECTS', 'DBUNDO',
              'DBEXT', 'DBMODE_R', 'DBMODE_W', 'DBUNDOFN', 'DBLOCKFN',
-             'DBRECOVFN','BDBVERSFN', 'DBLOGNAME', 'DBFLAGS_O',  'DBFLAGS_R',
+             'DBRECOVFN','BDBVERSFN', 'DBLOGNAME', 'SCHVERSFN', 'PCKVERSFN'
-             'DBFLAGS_D', 'SCHVERSFN', 'PCKVERSFN'
             ) +
             
             ('PERSON_KEY', 'FAMILY_KEY', 'SOURCE_KEY', 'CITATION_KEY',
@@ -60,18 +59,6 @@ DBLOCKS   = 100000          # Maximum number of locks supported
 DBOBJECTS = 100000          # Maximum number of simultaneously locked objects
 DBUNDO    = 1000            # Maximum size of undo buffer
 
-try:
-    from bsddb3.db import DB_CREATE, DB_AUTO_COMMIT, DB_DUP, DB_DUPSORT, DB_RDONLY
-    DBFLAGS_O = DB_CREATE | DB_AUTO_COMMIT  # Default flags for database open
-    DBFLAGS_R = DB_RDONLY                   # Flags to open a database read-only
-    DBFLAGS_D = DB_DUP | DB_DUPSORT         # Default flags for duplicate keys
-except:
-    print("WARNING: no bsddb support")
-    # FIXME: make this more abstract to deal with other backends, or do not import
-    DBFLAGS_O = DB_CREATE = DB_AUTO_COMMIT = 0
-    DBFLAGS_R = DB_RDONLY = 0
-    DBFLAGS_D = DB_DUP = DB_DUPSORT = 0
-
 PERSON_KEY     = 0
 FAMILY_KEY     = 1
 SOURCE_KEY     = 2

gramps/gen/plug/_gramplet.py

@@ -320,8 +320,6 @@ class Gramplet(object):
             self._idle_id = 0
             LOG.debug("gramplet updater: %s : One time, done!" % self.gui.title)
             return False
-        # FIXME: find out why Data Entry has this error, or just ignore it
-        import bsddb3 as bsddb
         try:
             retval = next(self._generator)
             if not retval:
@@ -332,10 +330,6 @@ class Gramplet(object):
             LOG.debug("gramplet updater: %s: return %s" % 
                       (self.gui.title, retval))
             return retval
-        except bsddb.db.DBCursorClosedError:
-            # not sure why---caused by Data Entry Gramplet
-            LOG.warn("bsddb.db.DBCursorClosedError in: %s" % self.gui.title)
-            return False
         except StopIteration:
             self._idle_id = 0
             self._generator.close()

gramps/grampsapp.py

@@ -136,14 +136,6 @@ if not sys.version_info >= MIN_PYTHON_VERSION :
              'v3': MIN_PYTHON_VERSION[2]})
     sys.exit(1)
 
-try:
-    import bsddb3
-except ImportError:
-    logging.warning(_("\nYou don't have the python3 bsddb3 package installed."
-        " This package is needed to start Gramps.\n\n"
-         "Gramps will terminate now."))
-    sys.exit(1)
-    
 #-------------------------------------------------------------------------
 #
 # Gramps libraries

gramps/gui/aboutdialog.py

@@ -28,7 +28,12 @@
 import os
 import sys
 import io
-import bsddb3 as bsddb
+
+try:
+    import bsddb3 as bsddb ## ok, in try/except
+    BSDDB_STR = ellipses(str(bsddb.__version__) + " " + str(bsddb.db.version()))
+except:
+    BSDDB_STR = 'not found'
 
 ##import logging
 ##_LOG = logging.getLogger(".GrampsAboutDialog")
@@ -125,7 +130,7 @@ class GrampsAboutDialog(Gtk.AboutDialog):
                  "Distribution: %s")
                 % (ellipses(str(VERSION)),
                    ellipses(str(sys.version).replace('\n','')),
-                   ellipses(str(bsddb.__version__) + " " + str(bsddb.db.version())),
+                   BSDDB_STR,
                    ellipses(get_env_var('LANG','')),
                    ellipses(operatingsystem),
                    ellipses(distribution)))

gramps/gui/dbman.py

@@ -78,7 +78,6 @@ from gramps.cli.clidbman import CLIDbManager, NAME_FILE, time_val
 from .ddtargets import DdTargets
 from gramps.gen.recentfiles import rename_filename, remove_filename
 from .glade import Glade
-from gramps.gen.db.backup import restore
 from gramps.gen.db.exceptions import DbException
 
 
@@ -728,7 +727,7 @@ class DbManager(CLIDbManager):
         self.__start_cursor(_("Rebuilding database from backup files"))
         
         try:
-            restore(dbase)
+            dbase.restore()
         except DbException as msg:
             DbManager.ERROR(_("Error restoring backup data"), msg)
 

gramps/gui/editors/editfamily.py

@@ -26,7 +26,6 @@
 # python modules
 #
 #-------------------------------------------------------------------------
-from bsddb3 import db as bsddb_db
 import pickle
 
 #-------------------------------------------------------------------------
@@ -1026,10 +1025,11 @@ class EditFamily(EditPrimary):
                )
             
     def save(self, *obj):
-        try:
+        ## FIXME: how to catch a specific error?
-            self.__do_save()
+        #try:
-        except bsddb_db.DBRunRecoveryError as msg:
+        self.__do_save()
-            RunDatabaseRepair(msg[1])
+        #except bsddb_db.DBRunRecoveryError as msg:
+        #    RunDatabaseRepair(msg[1])
 
     def __do_save(self):
         self.ok_button.set_sensitive(False)

gramps/gui/logger/_errorreportassistant.py

@@ -30,7 +30,12 @@ from gi.repository import GdkPixbuf
 from gi.repository import GObject
 import cairo
 import sys, os
-import bsddb3 as bsddb
+
+try:
+    import bsddb3 as bsddb # ok, in try/except
+    BSDDB_STR = str(bsddb.__version__) + " " + str(bsddb.db.version())
+except:
+    BSDDB_STR = 'not found'
     
 #-------------------------------------------------------------------------
 #
@@ -166,7 +171,7 @@ class ErrorReportAssistant(Gtk.Assistant):
                "gobject version: %s\n"\
                "cairo version  : %s"\
                % (str(sys.version).replace('\n',''),
-                  str(bsddb.__version__) + " " + str(bsddb.db.version()),
+                  BSDDB_STR,
                   str(VERSION),
                   get_env_var('LANG',''),
                   operatingsystem,

gramps/gui/viewmanager.py

@@ -87,7 +87,6 @@ from gramps.gen.utils.file import media_path_full
 from .dbloader import DbLoader
 from .display import display_help, display_url
 from .configure import GrampsPreferences
-from gramps.gen.db.backup import backup
 from gramps.gen.db.exceptions import DbException
 from .aboutdialog import GrampsAboutDialog
 from .navigator import Navigator
@@ -762,7 +761,7 @@ class ViewManager(CLIManager):
             self.uistate.progress.show()
             self.uistate.push_message(self.dbstate, _("Autobackup..."))
             try:
-                backup(self.dbstate.db)
+                self.dbstate.db.backup()
             except DbException as msg:
                 ErrorDialog(_("Error saving backup data"), msg)
             self.uistate.set_busy_cursor(False)

gramps/plugins/database/bsddb_support/__init__.py

@@ -93,4 +93,3 @@ from gramps.gen.db.txn import *
 from .undoredo import *
 from gramps.gen.db.exceptions import *
 from .write import *
-from .backup import backup, restore

gramps/plugins/database/bsddb_support/backup.py

@@ -72,142 +72,3 @@ from .write import FAMILY_TBL, PLACES_TBL, SOURCES_TBL, MEDIA_TBL, \
 import logging
 LOG = logging.getLogger(".Backup")
 
-def backup(database):
-    """
-    Exports the database to a set of backup files. These files consist
-    of the pickled database tables, one file for each table.
-
-    The heavy lifting is done by the private :py:func:`__do__export` function.
-    The purpose of this function is to catch any exceptions that occur.
-
-    :param database: database instance to backup
-    :type database: DbDir
-    """
-    try:
-        __do_export(database)
-    except (OSError, IOError) as msg:
-        raise DbException(str(msg))
-
-def __mk_backup_name(database, base):
-    """
-    Return the backup name of the database table
-
-    :param database: database instance 
-    :type database: DbDir
-    :param base: base name of the table
-    :type base: str
-    """
-    return os.path.join(database.get_save_path(), base + ".gbkp")
-
-def __mk_tmp_name(database, base):
-    """
-    Return the temporary backup name of the database table
-
-    :param database: database instance 
-    :type database: DbDir
-    :param base: base name of the table
-    :type base: str
-    """
-    return os.path.join(database.get_save_path(), base + ".gbkp.new")
-
-def __do_export(database):
-    """
-    Loop through each table of the database, saving the pickled data
-    a file.
-
-    :param database: database instance to backup
-    :type database: DbDir
-    """
-    try:
-        for (base, tbl) in __build_tbl_map(database):
-            backup_name = __mk_tmp_name(database, base)
-            backup_table = open(backup_name, 'wb')
-    
-            cursor = tbl.cursor()
-            data = cursor.first()
-            while data:
-                pickle.dump(data, backup_table, 2)
-                data = cursor.next()
-            cursor.close()
-            backup_table.close()
-    except (IOError,OSError):
-        return
-
-    for (base, tbl) in __build_tbl_map(database):
-        new_name = __mk_backup_name(database, base)
-        old_name = __mk_tmp_name(database, base)
-        if os.path.isfile(new_name):
-            os.unlink(new_name)
-        os.rename(old_name, new_name)
-
-def restore(database):
-    """
-    Restores the database to a set of backup files. These files consist
-    of the pickled database tables, one file for each table.
-
-    The heavy lifting is done by the private :py:func:`__do__restore` function.
-    The purpose of this function is to catch any exceptions that occur.
-
-    :param database: database instance to restore
-    :type database: DbDir
-    """
-    try:
-        __do_restore(database)
-    except (OSError, IOError) as msg:
-        raise DbException(str(msg))
-
-def __do_restore(database):
-    """
-    Loop through each table of the database, restoring the pickled data
-    to the appropriate database file.
-
-    :param database: database instance to backup
-    :type database: DbDir
-    """
-    for (base, tbl) in __build_tbl_map(database):
-        backup_name = __mk_backup_name(database, base)
-        backup_table = open(backup_name, 'rb')
-        __load_tbl_txn(database, backup_table, tbl)
-
-    database.rebuild_secondary()
-
-def __load_tbl_txn(database, backup_table, tbl):
-    """
-    Return the temporary backup name of the database table
-
-    :param database: database instance 
-    :type database: DbDir
-    :param backup_table: file containing the backup data
-    :type backup_table: file
-    :param tbl: Berkeley db database table
-    :type tbl: Berkeley db database table
-    """
-    try:
-        while True:
-            data = pickle.load(backup_table)
-            txn = database.env.txn_begin()
-            tbl.put(data[0], data[1], txn=txn)
-            txn.commit()
-    except EOFError:
-        backup_table.close()
-
-def __build_tbl_map(database):
-    """
-    Builds a table map of names to database tables.
-
-    :param database: database instance to backup
-    :type database: DbDir
-    """
-    return [
-        ( PERSON_TBL,  database.person_map.db),
-        ( FAMILY_TBL,  database.family_map.db),
-        ( PLACES_TBL,  database.place_map.db),
-        ( SOURCES_TBL, database.source_map.db),
-        ( CITATIONS_TBL, database.citation_map.db),
-        ( REPO_TBL,    database.repository_map.db),
-        ( NOTE_TBL,    database.note_map.db),
-        ( MEDIA_TBL,   database.media_map.db),
-        ( EVENTS_TBL,  database.event_map.db),
-        ( TAG_TBL,     database.tag_map.db),
-        ( META,        database.metadata.db),
-        ]

gramps/plugins/database/bsddb_support/summary.py

@@ -0,0 +1,62 @@
+## Removed from clidbman.py
+## specific to bsddb
+
+from bsddb3 import dbshelve, db
+import os
+
+from gramps.gen.db import META, PERSON_TBL
+from  gramps.gen.db.dbconst import BDBVERSFN
+
+def get_dbdir_summary(dirpath, name):
+    """
+    Returns (people_count, bsddb_version, schema_version) of
+    current DB.
+    Returns ("Unknown", "Unknown", "Unknown") if invalid DB or other error.
+    """
+
+    bdbversion_file = os.path.join(dirpath, BDBVERSFN)
+    if os.path.isfile(bdbversion_file):
+        vers_file = open(bdbversion_file)
+        bsddb_version = vers_file.readline().strip()
+    else:
+        return "Unknown", "Unknown", "Unknown"
+        
+    current_bsddb_version = str(db.version())
+    if bsddb_version != current_bsddb_version:
+        return "Unknown", bsddb_version, "Unknown"
+        
+    env = db.DBEnv()
+    flags = db.DB_CREATE | db.DB_PRIVATE |\
+        db.DB_INIT_MPOOL |\
+        db.DB_INIT_LOG | db.DB_INIT_TXN
+    try:
+        env.open(dirpath, flags)
+    except Exception as msg:
+        LOG.warning("Error opening db environment for '%s': %s" %
+                    (name, str(msg)))
+        try:
+            env.close()
+        except Exception as msg:
+            LOG.warning("Error closing db environment for '%s': %s" %
+                    (name, str(msg)))
+        return "Unknown", bsddb_version, "Unknown"
+    dbmap1 = dbshelve.DBShelf(env)
+    fname = os.path.join(dirpath, META + ".db")
+    try:
+        dbmap1.open(fname, META, db.DB_HASH, db.DB_RDONLY)
+    except:
+        env.close()
+        return "Unknown", bsddb_version, "Unknown"
+    schema_version = dbmap1.get(b'version', default=None)
+    dbmap1.close()
+    dbmap2 = dbshelve.DBShelf(env)
+    fname = os.path.join(dirpath, PERSON_TBL + ".db")
+    try:
+        dbmap2.open(fname, PERSON_TBL, db.DB_HASH, db.DB_RDONLY)
+    except:
+        env.close()
+        return "Unknown", bsddb_version, schema_version
+    count = len(dbmap2)
+    dbmap2.close()
+    env.close()
+    return (count, bsddb_version, schema_version)

gramps/plugins/database/bsddb_support/write.py

@@ -40,16 +40,12 @@ from functools import wraps
 import logging
 from sys import maxsize, getfilesystemencoding, version_info
 
-try:
+from bsddb3 import dbshelve, db
-    from bsddb3 import dbshelve, db
+from bsddb3.db import DB_CREATE, DB_AUTO_COMMIT, DB_DUP, DB_DUPSORT, DB_RDONLY
-except:
+
-    # FIXME: make this more abstract to deal with other backends
+DBFLAGS_O = DB_CREATE | DB_AUTO_COMMIT  # Default flags for database open
-    class db:
+DBFLAGS_R = DB_RDONLY                   # Flags to open a database read-only
-        DB_HASH = 0
+DBFLAGS_D = DB_DUP | DB_DUPSORT         # Default flags for duplicate keys
-        DBRunRecoveryError = 0
-        DBAccessError = 0
-        DBPageNotFoundError = 0
-        DBInvalidArgError = 0
 
 #-------------------------------------------------------------------------
 #
@@ -2450,6 +2446,146 @@ class DbBsddb(DbBsddbRead, DbWriteBase, UpdateCallback):
         """
         return DbTxn
 
+    def backup(self):
+        """
+        Exports the database to a set of backup files. These files consist
+        of the pickled database tables, one file for each table.
+        
+        The heavy lifting is done by the private :py:func:`__do__export` function.
+        The purpose of this function is to catch any exceptions that occur.
+        
+        :param database: database instance to backup
+        :type database: DbDir
+        """
+        try:
+            do_export(self)
+        except (OSError, IOError) as msg:
+            raise DbException(str(msg))
+
+    def restore(self):
+        """
+        Restores the database to a set of backup files. These files consist
+        of the pickled database tables, one file for each table.
+        
+        The heavy lifting is done by the private :py:func:`__do__restore` function.
+        The purpose of this function is to catch any exceptions that occur.
+        
+        :param database: database instance to restore
+        :type database: DbDir
+        """
+        try:
+            do_restore(self)
+        except (OSError, IOError) as msg:
+            raise DbException(str(msg))
+
+def mk_backup_name(database, base):
+    """
+    Return the backup name of the database table
+
+    :param database: database instance 
+    :type database: DbDir
+    :param base: base name of the table
+    :type base: str
+    """
+    return os.path.join(database.get_save_path(), base + ".gbkp")
+
+def mk_tmp_name(database, base):
+    """
+    Return the temporary backup name of the database table
+
+    :param database: database instance 
+    :type database: DbDir
+    :param base: base name of the table
+    :type base: str
+    """
+    return os.path.join(database.get_save_path(), base + ".gbkp.new")
+
+def do_export(database):
+    """
+    Loop through each table of the database, saving the pickled data
+    a file.
+
+    :param database: database instance to backup
+    :type database: DbDir
+    """
+    try:
+        for (base, tbl) in build_tbl_map(database):
+            backup_name = mk_tmp_name(database, base)
+            backup_table = open(backup_name, 'wb')
+    
+            cursor = tbl.cursor()
+            data = cursor.first()
+            while data:
+                pickle.dump(data, backup_table, 2)
+                data = cursor.next()
+            cursor.close()
+            backup_table.close()
+    except (IOError,OSError):
+        return
+
+    for (base, tbl) in build_tbl_map(database):
+        new_name = mk_backup_name(database, base)
+        old_name = mk_tmp_name(database, base)
+        if os.path.isfile(new_name):
+            os.unlink(new_name)
+        os.rename(old_name, new_name)
+
+def do_restore(database):
+    """
+    Loop through each table of the database, restoring the pickled data
+    to the appropriate database file.
+
+    :param database: database instance to backup
+    :type database: DbDir
+    """
+    for (base, tbl) in build_tbl_map(database):
+        backup_name = mk_backup_name(database, base)
+        backup_table = open(backup_name, 'rb')
+        load_tbl_txn(database, backup_table, tbl)
+
+    database.rebuild_secondary()
+
+def load_tbl_txn(database, backup_table, tbl):
+    """
+    Return the temporary backup name of the database table
+
+    :param database: database instance 
+    :type database: DbDir
+    :param backup_table: file containing the backup data
+    :type backup_table: file
+    :param tbl: Berkeley db database table
+    :type tbl: Berkeley db database table
+    """
+    try:
+        while True:
+            data = pickle.load(backup_table)
+            txn = database.env.txn_begin()
+            tbl.put(data[0], data[1], txn=txn)
+            txn.commit()
+    except EOFError:
+        backup_table.close()
+
+def build_tbl_map(database):
+    """
+    Builds a table map of names to database tables.
+
+    :param database: database instance to backup
+    :type database: DbDir
+    """
+    return [
+        ( PERSON_TBL,  database.person_map.db),
+        ( FAMILY_TBL,  database.family_map.db),
+        ( PLACES_TBL,  database.place_map.db),
+        ( SOURCES_TBL, database.source_map.db),
+        ( CITATIONS_TBL, database.citation_map.db),
+        ( REPO_TBL,    database.repository_map.db),
+        ( NOTE_TBL,    database.note_map.db),
+        ( MEDIA_TBL,   database.media_map.db),
+        ( EVENTS_TBL,  database.event_map.db),
+        ( TAG_TBL,     database.tag_map.db),
+        ( META,        database.metadata.db),
+        ]
+
 def _mkname(path, name):
     return os.path.join(path, name + DBEXT)
 

gramps/plugins/gramplet/leak.py

@@ -31,7 +31,6 @@ Show uncollected objects in a window.
 #------------------------------------------------------------------------
 from gramps.gen.const import GRAMPS_LOCALE as glocale
 _ = glocale.translation.gettext
-from bsddb3.db import DBError
 
 #------------------------------------------------------------------------
 #
@@ -155,6 +154,13 @@ class Leak(Gramplet):
                         parent=self.uistate.window)
 
     def display(self):
+        try:
+            from bsddb3.db import DBError
+        except:
+            class DBError(Exception):
+                """
+                Dummy.
+                """
         gc.collect(2)
         self.model.clear()
         count = 0