0006529: Cancelling database upgrade can corrupt the database. Check whether the bsddb version has changed (or was originally unknown). If it is an upgrade, ask the user whether he has made a backup, and is ready to upgrade. If so, made a zip backup and open the database with the new bsddb version. Make messages on the dialogues more explicit with the version numbers mentioned.

svn: r21850

src/cli/arghandler.py

@@ -370,12 +370,8 @@ class ArgHandler(object):
                 if not self.check_db(db_path, self.force_unlock):
                     sys.exit(0)
                 # Add the file to the recent items
-                path = os.path.join(db_path, "name.txt")
+                title = self.dbstate.db.get_dbname()
-                try:
+                if not title:
-                    ifile = open(path)
-                    title = ifile.readline().strip()
-                    ifile.close()
-                except:
                     title = db_path
                 RecentFiles.recent_files(db_path, title)
                 self.open = db_path

src/cli/clidbman.py

@@ -46,6 +46,8 @@ from gen.ggettext import gettext as _
 #-------------------------------------------------------------------------
 import logging
 LOG = logging.getLogger(".clidbman")
+from gen.db.dbconst import DBLOGNAME
+_LOG = logging.getLogger(DBLOGNAME)
 
 #-------------------------------------------------------------------------
 #
@@ -345,6 +347,11 @@ class CLIDbManager(object):
                 self.__start_cursor(_("Importing data..."))
                 dbclass = gen.db.DbBsddb
                 dbase = dbclass()
+                from  gen.db.dbconst import BDBVERSFN
+                versionpath = os.path.join(name, BDBVERSFN)
+                _LOG.debug("Write bsddb version %s" % str(dbase.version()))
+                with open(versionpath, "w") as version_file:
+                    version_file.write(str(dbase.version()))
                 dbase.load(new_path, callback)
     
                 import_function = plugin.get_import_function()

src/cli/grampscli.py

@@ -154,6 +154,15 @@ class CLIDbLoader(object):
         try:
             self.dbstate.db.load(filename, self._pulse_progress, mode)
             self.dbstate.db.set_save_path(filename)
+        except gen.db.exceptions.DbEnvironmentError, msg:
+            self.dbstate.no_database()
+            self._errordialog( _("Cannot open database"), str(msg))
+        except gen.db.exceptions.BsddbUpgradeRequiredError, msg:
+            self.dbstate.no_database()
+            self._errordialog( _("Cannot open database"), str(msg))
+        except gen.db.exceptions.BsddbDowngradeError, msg:
+            self.dbstate.no_database()
+            self._errordialog( _("Cannot open database"), str(msg))
         except gen.db.exceptions.DbUpgradeRequiredError, msg:
             self.dbstate.no_database()
             self._errordialog( _("Cannot open database"), str(msg))

src/gen/db/base.py

@@ -1014,7 +1014,8 @@ class DbReadBase(object):
         """
         raise NotImplementedError
 
-    def load(self, name, callback, mode=None, upgrade=False):
+    def load(self, name, callback, mode=None, force_schema_upgrade=False,
+             force_bsddb_upgrade=False):
         """
         Open the specified database.
         """
@@ -1399,7 +1400,7 @@ class DbWriteBase(DbReadBase):
         """
         raise NotImplementedError
 
-    def need_upgrade(self):
+    def need_schema_upgrade(self):
         """
         Return True if database needs to be upgraded
         """

src/gen/db/exceptions.py

@@ -71,13 +71,23 @@ class DbVersionError(Exception):
     Error used to report that a file could not be read because it is written 
     in an unsupported version of the file format.
     """
-    def __init__(self):
+    def __init__(self, tree_vers, min_vers, max_vers):
         Exception.__init__(self)
+        self.tree_vers = tree_vers
+        self.min_vers = min_vers
+        self.max_vers = max_vers
 
     def __str__(self):
-        return _("The database version is not supported by this version of "
+        return _("The schema version is not supported by this version of "
-                 "Gramps.\nPlease upgrade to the corresponding version or use "
+                 "Gramps.\n\n"
-                 "XML for porting data between different database versions.")
+                 "This Family tree is schema version %(tree_vers)s, and this "
+                 "version of Gramps supports versions %(min_vers)s to "
+                 "%(max_vers)s\n\n"
+                 "Please upgrade to the corresponding version or use "
+                 "XML for porting data between different database versions.") %\
+                 {'tree_vers': self.tree_vers,
+                  'min_vers': self.min_vers,
+                  'max_vers': self.max_vers}
     
 class BsddbDowngradeError(Exception):
     """
@@ -103,6 +113,30 @@ class BsddbDowngradeError(Exception):
                  'intend to use.') % {'env_version': self.env_version,
                  'bdb_version': self.bdb_version}
 
+class BsddbUpgradeRequiredError(Exception):
+    """
+    Error used to report that the Berkeley database used to create the family
+    tree is of a version that is too new to be supported by the current version.
+    """
+    def __init__(self, env_version, bsddb_version):
+        Exception.__init__(self)
+        self.env_version = str(env_version)
+        self.bsddb_version = str(bsddb_version)
+
+    def __str__(self):
+        return _('The BSDDB version of the Family Tree you are trying to open '
+              'needs to be upgraded from %(env_version)s to %(bdb_version)s.\n\n'
+              'This probably means that the Family Tree was created with '
+              'an old version of Gramps. Opening the tree with this version '
+              'of Gramps may irretrievably corrupt your tree. You are '
+              'strongly advised to backup your tree before proceeding, '
+              'see: \n'
+              'http://www.gramps-project.org/wiki/index.php?title=How_to_make_a_backup\n\n'
+              'If you have made a backup, then you can get Gramps to try '
+              'to open the tree and upgrade it') % \
+              {'env_version': self.env_version,
+               'bdb_version': self.bsddb_version}
+
 class DbEnvironmentError(Exception):
     """
     Error used to report that the database 'environment' could not be opened.

src/gen/db/read.py

@@ -1752,7 +1752,7 @@ class DbBsddbRead(DbReadBase, Callback):
         filepath = os.path.join(self.path, "name.txt")
         try:
             name_file = open(filepath, "r")
-            name = name_file.read()
+            name = name_file.readline().strip()
             name_file.close()
         except (OSError, IOError), msg:
             self.__log_error()

src/gen/db/test/db_test.py

@@ -211,7 +211,7 @@ class DbTest(object):
         "commit_source",
         "commit_tag",
         "delete_primary_from_reference_map",
-        "need_upgrade",
+        "need_schema_upgrade",
         "rebuild_secondary",
         "reindex_reference_map",
         "remove_event",

src/gen/db/write.py

@@ -40,7 +40,7 @@ import locale
 import bisect
 from functools import wraps
 import logging
-from sys import maxint
+from sys import maxint, getfilesystemencoding
 
 from gen.ggettext import gettext as _
 import config
@@ -59,7 +59,7 @@ from gen.lib import (GenderStats, Person, Family, Event, Place, Source,
 from gen.db import (DbBsddbRead, DbWriteBase, BSDDBTxn, 
                     DbTxn, BsddbBaseCursor, BsddbDowngradeError, DbVersionError,
                     DbEnvironmentError, DbUpgradeRequiredError, find_surname,
-                    find_surname_name, DbUndoBSDDB as DbUndo)
+                    find_surname_name, DbUndoBSDDB as DbUndo, exceptions)
 from gen.db.dbconst import *
 from gen.utils.callback import Callback
 from gen.updatecallback import UpdateCallback
@@ -357,26 +357,71 @@ class DbBsddb(DbBsddbRead, DbWriteBase, UpdateCallback):
             with BSDDBTxn(self.env, self.metadata) as txn:
                 txn.put('mediapath', path)            
 
-    def __check_bdb_version(self, name):
+    def __make_zip_backup(self, dirname):
-        """Older version of Berkeley DB can't read data created by a newer
+        import zipfile
-        version."""
+        title = self.get_dbname()
+        
+        if not os.access(dirname, os.W_OK):
+            _LOG.warning("Can't write technical DB backup for %s" % title)
+            return
+        (grampsdb_path, db_code) = os.path.split(dirname)
+        dotgramps_path = os.path.dirname(grampsdb_path)
+        zipname = title + time.strftime("%Y-%m-%d %H-%M-%S") + ".zip"
+        zipname = zipname.encode(getfilesystemencoding())
+        zippath = os.path.join(dotgramps_path, zipname)
+        myzip = zipfile.ZipFile(zippath, 'w')
+        for filename in os.listdir(dirname):
+            pathname = os.path.join(dirname, filename)
+            myzip.write(pathname, os.path.join(db_code, filename))
+        myzip.close()
+        _LOG.warning("If upgrade and loading the Family Tree works, you can "
+                     "delete the zip file at %s" %
+                     zippath)
+    
+    def __check_bdb_version(self, name, force_bsddb_upgrade=False):
+        """
+        Older version of Berkeley DB can't read data created by a newer
+        version.
+        name: Directory path of the database files
+        force_bsddb_upgrade: whether the user has requested that the database be
+        upgraded
+        """
         bdb_version = db.version()
-        env_version = (0, 0, 0)
         versionpath = os.path.join(self.path, BDBVERSFN)
-        try:
+        # Compare the current version of the database (bsddb_version) with the
+        # version of the database code (env_version). If it is a downgrade,
+        # raise an exception because we can't do anything. If they are the same,
+        # return. If it is an upgrade, raise an exception unless  the user has
+        # already told us we can upgrade.
+        if os.path.isfile(versionpath):
             with open(versionpath, "r") as version_file:
-                env_version = version_file.read().strip()
+                bsddb_version = version_file.read().strip()
-                env_version = tuple(map(int, env_version[1:-1].split(', ')))
+                env_version = tuple(map(int, bsddb_version[1:-1].split(', ')))
-        except:
+            if (env_version[0] > bdb_version[0]) or \
-            # Just assume that the Berkeley DB version is OK.
+                (env_version[0] == bdb_version[0] and
-            pass
+                 env_version[1] > bdb_version[1]):
-        if (env_version[0] > bdb_version[0]) or \
+                clear_lock_file(name)
-            (env_version[0] == bdb_version[0] and
+                raise BsddbDowngradeError(env_version, bdb_version)
-             env_version[1] > bdb_version[1]):
+            elif env_version == bdb_version:
-            clear_lock_file(name)
+                return
-            raise BsddbDowngradeError(env_version, bdb_version)
+        else:
-        elif env_version != bdb_version and not self.readonly:
+            # bsddb version is unknown
+            bsddb_version = "Unknown"
+
+        # An upgrade is needed, raise an exception unless the user has allowed
+        # an upgrade
+        if not force_bsddb_upgrade:
+            _LOG.debug("Bsddb upgrade required from %s to %s" %
+                       (bsddb_version, str(bdb_version)))
+            raise exceptions.BsddbUpgradeRequiredError(bsddb_version,
+                                                       str(bdb_version))
+        
+        if not self.readonly:
+            _LOG.warning("Bsddb upgrade requested from %s to %s" %
+                         (bsddb_version, str(bdb_version)))
             self.update_env_version = True
+        # Make a backup of the database files anyway 
+        self.__make_zip_backup(name)   
 
     @catch_db_error
     def version_supported(self):
@@ -384,7 +429,7 @@ class DbBsddb(DbBsddbRead, DbWriteBase, UpdateCallback):
         return ((dbversion <= _DBVERSION) and (dbversion >= _MINVERSION))
 
     @catch_db_error
-    def need_upgrade(self):
+    def need_schema_upgrade(self):
         dbversion = self.metadata.get('version', default=0)
         return not self.readonly and dbversion < _DBVERSION
 
@@ -410,7 +455,8 @@ class DbBsddb(DbBsddbRead, DbWriteBase, UpdateCallback):
         return False
 
     @catch_db_error
-    def load(self, name, callback, mode=DBMODE_W, upgrade=False):
+    def load(self, name, callback, mode=DBMODE_W, force_schema_upgrade=False,
+             force_bsddb_upgrade=False):
 
         if self.__check_readonly(name):
             mode = DBMODE_R
@@ -430,7 +476,7 @@ class DbBsddb(DbBsddbRead, DbWriteBase, UpdateCallback):
         self.path = self.full_name
         self.brief_name = os.path.basename(name)
 
-        self.__check_bdb_version(name)
+        self.__check_bdb_version(name, force_bsddb_upgrade)
 
         # Set up database environment
         self.env = db.DBEnv()
@@ -482,8 +528,9 @@ class DbBsddb(DbBsddbRead, DbWriteBase, UpdateCallback):
         # If we cannot work with this DB version,
         # it makes no sense to go further
         if not self.version_supported():
+            tree_vers = self.metadata.get(b'version', default=0)
             self.__close_early()
-            raise DbVersionError()
+            raise DbVersionError(tree_vers, _MINVERSION, _DBVERSION)
 
         self.__load_metadata()
         gstats = self.metadata.get('gender_stats', default=None)
@@ -530,12 +577,23 @@ class DbBsddb(DbBsddbRead, DbWriteBase, UpdateCallback):
         self.name_group = self.__open_db(self.full_name, NAME_GROUP,
                               db.DB_HASH, db.DB_DUP)
 
+        # We have now successfully opened the database, so if the BSDDB version
+        # has changed, we update the DBSDB version file. 
+        
+        if self.update_env_version:
+            versionpath = os.path.join(name, BDBVERSFN)
+            with open(versionpath, "w") as version_file:
+                version_file.write(str(db.version()))
+            _LOG.debug("Updated BDBVERSFN file to %s" % str(db.version()))
+
         # Here we take care of any changes in the tables related to new code.
         # If secondary indices change, then they should removed
         # or rebuilt by upgrade as well. In any case, the
         # self.secondary_connected flag should be set accordingly.
-        if self.need_upgrade():
+        if self.need_schema_upgrade():
-            if upgrade == True:
+            _LOG.debug("Schema upgrade required from %s to %s" %
+                       (self.metadata.get('version', default=0), _DBVERSION))
+            if force_schema_upgrade == True or force_bsddb_upgrade == True:
                 self.gramps_upgrade(callback)
             else:
                 self.__close_early()
@@ -1142,15 +1200,6 @@ class DbBsddb(DbBsddbRead, DbWriteBase, UpdateCallback):
         self.undo_history_callback = None
         self.undodb = None
 
-        if self.update_env_version:
-            versionpath = os.path.join(self.path, BDBVERSFN)
-            try:
-                with open(versionpath, "w") as version_file:
-                    version_file.write(str(db.version()))
-            except:
-                # Storing the version of Berkeley Db is not really vital.
-                pass
-
         try:
             clear_lock_file(self.get_save_path())
         except IOError:
@@ -1973,9 +2022,15 @@ class DbBsddb(DbBsddbRead, DbWriteBase, UpdateCallback):
 
         self.metadata  = self.__open_shelf(full_name, META)
         
+        _LOG.debug("Write schema version %s" % _DBVERSION)
         with BSDDBTxn(self.env, self.metadata) as txn:
             txn.put('version', _DBVERSION)
         
+        versionpath = os.path.join(name, BDBVERSFN)
+        _LOG.debug("Write bsddb version %s" % str(db.version()))
+        with open(versionpath, "w") as version_file:
+            version_file.write(str(db.version()))
+
         self.metadata.close()
         self.env.close()
   

src/gui/dbloader.py

@@ -295,20 +295,38 @@ class DbLoader(CLIDbLoader):
 
         self._begin_progress()
         
+        force_schema_upgrade = False
+        force_bsddb_upgrade = False
         try:
-            try:
+            while True:
-                self.dbstate.db.load(filename, self._pulse_progress, 
+                try:
-                                     mode, upgrade=False)
-            except gen.db.exceptions.DbUpgradeRequiredError, msg:
-                if QuestionDialog2(_("Need to upgrade database!"), 
-                                   str(msg), 
-                                   _("Upgrade now"), 
-                                   _("Cancel")).run():
                     self.dbstate.db.load(filename, self._pulse_progress, 
-                                         mode, upgrade=True)
+                                         mode, force_schema_upgrade,
+                                         force_bsddb_upgrade)
                     self.dbstate.db.set_save_path(filename)
-                else:
+                    break
-                    self.dbstate.no_database()
+                except gen.db.exceptions.DbUpgradeRequiredError, msg:
+                    if QuestionDialog2(_("Need to upgrade database!"), 
+                                       str(msg), 
+                                       _("Upgrade now"), 
+                                       _("Cancel")).run():
+                        force_schema_upgrade = True
+                        force_bsddb_upgrade = False
+                    else:
+                        self.dbstate.no_database()
+                        break
+                except gen.db.exceptions.BsddbUpgradeRequiredError, msg:
+                    if QuestionDialog2(_("Need to upgrade BSDDB database!"), 
+                                       str(msg), 
+                                       _("I have made a backup, "
+                                         "please upgrade my tree"), 
+                                       _("Cancel")).run():
+                        force_schema_upgrade = False
+                        force_bsddb_upgrade = True
+                    else:
+                        self.dbstate.no_database()
+                        break
+        # Get here is there is an exception the while loop does not handle
         except gen.db.exceptions.BsddbDowngradeError, msg:
             self.dbstate.no_database()
             self._errordialog( _("Cannot open database"), str(msg))

src/plugins/lib/libgrdb.py

@@ -287,7 +287,7 @@ class DbGrdb(Callback):
         """Return True when the file has a supported version."""
         return True
 
-    def need_upgrade(self):
+    def need_scema_upgrade(self):
         return False
 
     def gramps_upgrade(self):