diff --git a/gramps/cli/arghandler.py b/gramps/cli/arghandler.py index 23b4a8d19..6c6ca0501 100644 --- a/gramps/cli/arghandler.py +++ b/gramps/cli/arghandler.py @@ -376,12 +376,8 @@ class ArgHandler(object): if not self.check_db(db_path, self.force_unlock): sys.exit(0) # Add the file to the recent items - path = os.path.join(db_path, "name.txt") - try: - ifile = open(path) - title = ifile.readline().strip() - ifile.close() - except: + title = self.dbstate.db.get_dbname() + if not title: title = db_path recent_files(db_path, title) self.open = db_path diff --git a/gramps/cli/clidbman.py b/gramps/cli/clidbman.py index ad1cd3f18..c9bc4df4e 100644 --- a/gramps/cli/clidbman.py +++ b/gramps/cli/clidbman.py @@ -51,6 +51,8 @@ import tempfile #------------------------------------------------------------------------- import logging LOG = logging.getLogger(".clidbman") +from gramps.gen.db.dbconst import DBLOGNAME +_LOG = logging.getLogger(DBLOGNAME) #------------------------------------------------------------------------- # @@ -350,6 +352,11 @@ class CLIDbManager(object): # write locally: temp_fp.write(data) url_fp.close() + from gen.db.dbconst import BDBVERSFN + versionpath = os.path.join(name, BDBVERSFN) + _LOG.debug("Write bsddb version %s" % str(dbase.version())) + with open(versionpath, "w") as version_file: + version_file.write(str(dbase.version())) temp_fp.close() (name, ext) = os.path.splitext(os.path.basename(filename)) diff --git a/gramps/cli/grampscli.py b/gramps/cli/grampscli.py index a4f2b9a42..8987bb357 100644 --- a/gramps/cli/grampscli.py +++ b/gramps/cli/grampscli.py @@ -158,6 +158,15 @@ class CLIDbLoader(object): try: self.dbstate.db.load(filename, self._pulse_progress, mode) self.dbstate.db.set_save_path(filename) + except gen.db.exceptions.DbEnvironmentError as msg: + self.dbstate.no_database() + self._errordialog( _("Cannot open database"), str(msg)) + except gen.db.exceptions.BsddbUpgradeRequiredError as msg: + self.dbstate.no_database() + self._errordialog( _("Cannot open database"), str(msg)) + except gen.db.exceptions.BsddbDowngradeError as msg: + self.dbstate.no_database() + self._errordialog( _("Cannot open database"), str(msg)) except gen.db.exceptions.DbUpgradeRequiredError as msg: self.dbstate.no_database() self._errordialog( _("Cannot open database"), str(msg)) diff --git a/gramps/gen/db/base.py b/gramps/gen/db/base.py index ce995a75b..a282fc511 100644 --- a/gramps/gen/db/base.py +++ b/gramps/gen/db/base.py @@ -1030,7 +1030,8 @@ class DbReadBase(object): """ raise NotImplementedError - def load(self, name, callback, mode=None, upgrade=False): + def load(self, name, callback, mode=None, force_schema_upgrade=False, + force_bsddb_upgrade=False): """ Open the specified database. """ @@ -1415,7 +1416,7 @@ class DbWriteBase(DbReadBase): """ raise NotImplementedError - def need_upgrade(self): + def need_schema_upgrade(self): """ Return True if database needs to be upgraded """ diff --git a/gramps/gen/db/exceptions.py b/gramps/gen/db/exceptions.py index ef4d0910f..da128b8e5 100644 --- a/gramps/gen/db/exceptions.py +++ b/gramps/gen/db/exceptions.py @@ -72,13 +72,23 @@ class DbVersionError(Exception): Error used to report that a file could not be read because it is written in an unsupported version of the file format. """ - def __init__(self): + def __init__(self, tree_vers, min_vers, max_vers): Exception.__init__(self) + self.tree_vers = tree_vers + self.min_vers = min_vers + self.max_vers = max_vers def __str__(self): - return _("The database version is not supported by this version of " - "Gramps.\nPlease upgrade to the corresponding version or use " - "XML for porting data between different database versions.") + return _("The schema version is not supported by this version of " + "Gramps.\n\n" + "This Family tree is schema version %(tree_vers)s, and this " + "version of Gramps supports versions %(min_vers)s to " + "%(max_vers)s\n\n" + "Please upgrade to the corresponding version or use " + "XML for porting data between different database versions.") %\ + {'tree_vers': self.tree_vers, + 'min_vers': self.min_vers, + 'max_vers': self.max_vers} class BsddbDowngradeError(Exception): """ @@ -104,6 +114,30 @@ class BsddbDowngradeError(Exception): 'intend to use.') % {'env_version': self.env_version, 'bdb_version': self.bdb_version} +class BsddbUpgradeRequiredError(Exception): + """ + Error used to report that the Berkeley database used to create the family + tree is of a version that is too new to be supported by the current version. + """ + def __init__(self, env_version, bsddb_version): + Exception.__init__(self) + self.env_version = str(env_version) + self.bsddb_version = str(bsddb_version) + + def __str__(self): + return _('The BSDDB version of the Family Tree you are trying to open ' + 'needs to be upgraded from %(env_version)s to %(bdb_version)s.\n\n' + 'This probably means that the Family Tree was created with ' + 'an old version of Gramps. Opening the tree with this version ' + 'of Gramps may irretrievably corrupt your tree. You are ' + 'strongly advised to backup your tree before proceeding, ' + 'see: \n' + 'http://www.gramps-project.org/wiki/index.php?title=How_to_make_a_backup\n\n' + 'If you have made a backup, then you can get Gramps to try ' + 'to open the tree and upgrade it') % \ + {'env_version': self.env_version, + 'bdb_version': self.bsddb_version} + class DbEnvironmentError(Exception): """ Error used to report that the database 'environment' could not be opened. diff --git a/gramps/gen/db/read.py b/gramps/gen/db/read.py index bdd5783be..bd43de967 100644 --- a/gramps/gen/db/read.py +++ b/gramps/gen/db/read.py @@ -1900,7 +1900,7 @@ class DbBsddbRead(DbReadBase, Callback): filepath = os.path.join(self.path, "name.txt") try: name_file = open(filepath, "r") - name = name_file.read() + name = name_file.readline().strip() name_file.close() except (OSError, IOError) as msg: self.__log_error() diff --git a/gramps/gen/db/test/db_test.py b/gramps/gen/db/test/db_test.py index 9ab340ef2..d72224539 100644 --- a/gramps/gen/db/test/db_test.py +++ b/gramps/gen/db/test/db_test.py @@ -213,7 +213,7 @@ class DbTest(object): "commit_source", "commit_tag", "delete_primary_from_reference_map", - "need_upgrade", + "need_schema_upgrade", "rebuild_secondary", "reindex_reference_map", "remove_event", diff --git a/gramps/gen/db/write.py b/gramps/gen/db/write.py index 6af04b633..67ba5af1d 100644 --- a/gramps/gen/db/write.py +++ b/gramps/gen/db/write.py @@ -43,7 +43,7 @@ import time import bisect from functools import wraps import logging -from sys import maxsize +from sys import getfilesystemencoding from ..config import config if config.get('preferences.use-bsddb3') or sys.version_info[0] >= 3: @@ -72,7 +72,8 @@ from ..lib.researcher import Researcher from . import (DbBsddbRead, DbWriteBase, BSDDBTxn, DbTxn, BsddbBaseCursor, BsddbDowngradeError, DbVersionError, DbEnvironmentError, DbUpgradeRequiredError, find_surname, - find_byte_surname, find_surname_name, DbUndoBSDDB as DbUndo) + find_byte_surname, find_surname_name, DbUndoBSDDB as DbUndo, + exceptions) from .dbconst import * from ..utils.callback import Callback from ..utils.cast import conv_dbstr_to_unicode @@ -397,29 +398,67 @@ class DbBsddb(DbBsddbRead, DbWriteBase, UpdateCallback): with BSDDBTxn(self.env, self.metadata) as txn: txn.put(b'mediapath', path) - def __check_bdb_version(self, name): + def __make_zip_backup(self, dirname): + import zipfile + title = self.get_dbname() + + if not os.access(dirname, os.W_OK): + _LOG.warning("Can't write technical DB backup for %s" % title) + return + (grampsdb_path, db_code) = os.path.split(dirname) + dotgramps_path = os.path.dirname(grampsdb_path) + zipname = title + time.strftime("%Y-%m-%d %H-%M-%S") + ".zip" + if sys.version_info[0] < 3: + zipname = zipname.encode(glocale.getfilesystemencoding()) + zippath = os.path.join(dotgramps_path, zipname) + myzip = zipfile.ZipFile(zippath, 'w') + for filename in os.listdir(dirname): + pathname = os.path.join(dirname, filename) + myzip.write(pathname, os.path.join(db_code, filename)) + myzip.close() + _LOG.warning("If upgrade and loading the Family Tree works, you can " + "delete the zip file at %s" % + zippath) + + def __check_bdb_version(self, name, force_bsddb_upgrade=False): """Older version of Berkeley DB can't read data created by a newer version.""" bdb_version = db.version() - env_version = (0, 0, 0) versionpath = os.path.join(self.path, BDBVERSFN) - try: + # Compare the current version of the database (bsddb_version) with the + # version of the database code (env_version). If it is a downgrade, + # raise an exception because we can't do anything. If they are the same, + # return. If it is an upgrade, raise an exception unless the user has + # already told us we can upgrade. + if os.path.isfile(versionpath): with open(versionpath, "r") as version_file: - env_version = version_file.read().strip() - env_version = tuple(map(int, env_version[1:-1].split(', '))) - except: - # Just assume that the Berkeley DB version is OK. - pass - if not env_version: - #empty file, assume it is ok to open - env_version = (0, 0, 0) - if (env_version[0] > bdb_version[0]) or \ - (env_version[0] == bdb_version[0] and - env_version[1] > bdb_version[1]): - clear_lock_file(name) - raise BsddbDowngradeError(env_version, bdb_version) - elif env_version != bdb_version and not self.readonly: + bsddb_version = version_file.read().strip() + env_version = tuple(map(int, bsddb_version[1:-1].split(', '))) + if (env_version[0] > bdb_version[0]) or \ + (env_version[0] == bdb_version[0] and + env_version[1] > bdb_version[1]): + clear_lock_file(name) + raise BsddbDowngradeError(env_version, bdb_version) + elif env_version == bdb_version: + return + else: + # bsddb version is unknown + bsddb_version = "Unknown" + + # An upgrade is needed, raise an exception unless the user has allowed + # an upgrade + if not force_bsddb_upgrade: + _LOG.debug("Bsddb upgrade required from %s to %s" % + (bsddb_version, str(bdb_version))) + raise exceptions.BsddbUpgradeRequiredError(bsddb_version, + str(bdb_version)) + + if not self.readonly: + _LOG.warning("Bsddb upgrade requested from %s to %s" % + (bsddb_version, str(bdb_version))) self.update_env_version = True + # Make a backup of the database files anyway + self.__make_zip_backup(name) @catch_db_error def version_supported(self): @@ -427,7 +466,7 @@ class DbBsddb(DbBsddbRead, DbWriteBase, UpdateCallback): return ((dbversion <= _DBVERSION) and (dbversion >= _MINVERSION)) @catch_db_error - def need_upgrade(self): + def need_schema_upgrade(self): dbversion = self.metadata.get(b'version', default=0) return not self.readonly and dbversion < _DBVERSION @@ -453,7 +492,8 @@ class DbBsddb(DbBsddbRead, DbWriteBase, UpdateCallback): return False @catch_db_error - def load(self, name, callback, mode=DBMODE_W, upgrade=False): + def load(self, name, callback, mode=DBMODE_W, force_schema_upgrade=False, + force_bsddb_upgrade=False): if self.__check_readonly(name): mode = DBMODE_R @@ -473,7 +513,7 @@ class DbBsddb(DbBsddbRead, DbWriteBase, UpdateCallback): self.path = self.full_name self.brief_name = os.path.basename(name) - self.__check_bdb_version(name) + self.__check_bdb_version(name, force_bsddb_upgrade) # Set up database environment self.env = db.DBEnv() @@ -525,9 +565,10 @@ class DbBsddb(DbBsddbRead, DbWriteBase, UpdateCallback): # If we cannot work with this DB version, # it makes no sense to go further if not self.version_supported(): + tree_vers = self.metadata.get(b'version', default=0) self.__close_early() - raise DbVersionError() - + raise DbVersionError(tree_vers, _MINVERSION, _DBVERSION) + self.__load_metadata() gstats = self.metadata.get(b'gender_stats', default=None) @@ -573,12 +614,27 @@ class DbBsddb(DbBsddbRead, DbWriteBase, UpdateCallback): self.name_group = self.__open_db(self.full_name, NAME_GROUP, db.DB_HASH, db.DB_DUP) + # We have now successfully opened the database, so if the BSDDB version + # has changed, we update the DBSDB version file. + + if self.update_env_version: + versionpath = os.path.join(name, BDBVERSFN) + with open(versionpath, "w") as version_file: + version = str(db.version()) + if sys.version_info[0] < 3: + if isinstance(version, UNITYPE): + version = version.encode('utf-8') + version_file.write(version) + _LOG.debug("Updated BDBVERSFN file to %s" % str(db.version())) + # Here we take care of any changes in the tables related to new code. # If secondary indices change, then they should removed # or rebuilt by upgrade as well. In any case, the # self.secondary_connected flag should be set accordingly. - if self.need_upgrade(): - if upgrade == True: + if self.need_schema_upgrade(): + _LOG.debug("Schema upgrade required from %s to %s" % + (self.metadata.get('version', default=0), _DBVERSION)) + if force_schema_upgrade == True or force_bsddb_upgrade == True: self.gramps_upgrade(callback) else: self.__close_early() @@ -1210,20 +1266,6 @@ class DbBsddb(DbBsddbRead, DbWriteBase, UpdateCallback): self.undo_history_callback = None self.undodb = None - if self.update_env_version: - versionpath = os.path.join(self.path, BDBVERSFN) - try: - with open(versionpath, "w") as version_file: - version = str(db.version()) - if sys.version_info[0] < 3: - if isinstance(version, UNITYPE): - version = version.encode('utf-8') - version_file.write(version) - except: - # Storing the version of Berkeley Db is not really vital. - print ("Error storing berkeley db version") - pass - try: clear_lock_file(self.get_save_path()) except IOError: @@ -2074,9 +2116,19 @@ class DbBsddb(DbBsddbRead, DbWriteBase, UpdateCallback): self.metadata = self.__open_shelf(full_name, META) + _LOG.debug("Write schema version %s" % _DBVERSION) with BSDDBTxn(self.env, self.metadata) as txn: txn.put(b'version', _DBVERSION) + versionpath = os.path.join(name, BDBVERSFN) + version = str(db.version()) + if sys.version_info[0] < 3: + if isinstance(version, UNITYPE): + version = version.encode('utf-8') + _LOG.debug("Write bsddb version %s" % version) + with open(versionpath, "w") as version_file: + version_file.write(version) + self.metadata.close() self.env.close() diff --git a/gramps/gui/dbloader.py b/gramps/gui/dbloader.py index b889b2fd3..83fbbd881 100644 --- a/gramps/gui/dbloader.py +++ b/gramps/gui/dbloader.py @@ -61,7 +61,8 @@ from gramps.gen.db import DbBsddb from gramps.gen.db.exceptions import (DbUpgradeRequiredError, BsddbDowngradeError, DbVersionError, - DbEnvironmentError) + DbEnvironmentError, + BsddbUpgradeRequiredError) from gramps.gen.constfunc import STRTYPE from gramps.gen.utils.file import get_unicode_path_from_file_chooser from .pluginmanager import GuiPluginManager @@ -304,24 +305,39 @@ class DbLoader(CLIDbLoader): self._begin_progress() + force_schema_upgrade = False + force_bsddb_upgrade = False try: - try: - db.load(filename, self._pulse_progress, - mode, upgrade=False) - self.dbstate.change_database(db) - except DbUpgradeRequiredError as msg: - if QuestionDialog2(_("Need to upgrade database!"), - str(msg), - _("Upgrade now"), - _("Cancel")).run(): - db = DbBsddb() - db.disable_signals() + while True: + try: db.load(filename, self._pulse_progress, - mode, upgrade=True) + mode, force_schema_upgrade, + force_bsddb_upgrade) db.set_save_path(filename) self.dbstate.change_database(db) - else: - self.dbstate.no_database() + break + except DbUpgradeRequiredError as msg: + if QuestionDialog2(_("Need to upgrade database!"), + str(msg), + _("Upgrade now"), + _("Cancel")).run(): + force_schema_upgrade = True + force_bsddb_upgrade = False + else: + self.dbstate.no_database() + break + except BsddbUpgradeRequiredError as msg: + if QuestionDialog2(_("Need to upgrade BSDDB database!"), + str(msg), + _("I have made a backup, " + "please upgrade my tree"), + _("Cancel")).run(): + force_schema_upgrade = False + force_bsddb_upgrade = True + else: + self.dbstate.no_database() + break + # Get here is there is an exception the while loop does not handle except BsddbDowngradeError as msg: self.dbstate.no_database() self._errordialog( _("Cannot open database"), str(msg))