Fix surname index for autocompletion

Fix check tool and start with importgrdb upgrade


svn: r15949

src/gen/db/read.py

@@ -46,7 +46,7 @@ import logging
 #
 #-------------------------------------------------------------------------
 from gen.lib import (MediaObject, Person, Family, Source, Event, Place, 
-                     Repository, Note, GenderStats, Researcher)
+                     Repository, Note, GenderStats, Researcher, NameOriginType)
 from gen.db.dbconst import *
 from gen.utils.callback import Callback
 from gen.db import (BsddbBaseCursor, DbReadBase)
@@ -67,6 +67,41 @@ _SIGBASE = ('person', 'family', 'source', 'event',
 DBERRS      = (db.DBRunRecoveryError, db.DBAccessError, 
                db.DBPageNotFoundError, db.DBInvalidArgError)
 
+#-------------------------------------------------------------------------
+#
+# Helper functions
+#
+#-------------------------------------------------------------------------  
+def find_surname(key, data):
+    """
+    Creating a surname from raw data of a person, to use for sort and index
+    """
+    return __index_surname(data[3][5])
+
+def find_surname_name(key, data):
+    """
+    Creating a surname from raw name, to use for sort and index
+    """
+    return __index_surname(data[5])
+
+def __index_surname(surn_list):
+    """
+    All non pa/matronymic surnames are used in indexing.
+    pa/matronymic not as they change for every generation!
+    """
+    if surn_list:
+        surn = " ".join([x[0] for x in surn_list if not (x[3][0] in [
+                    NameOriginType.PATRONYMIC, NameOriginType.MATRONYMIC]) ])
+    else:
+        surn = ""
+    return str(surn)
+    
+
+#-------------------------------------------------------------------------
+#
+# class DbBookmarks
+#
+#-------------------------------------------------------------------------  
 class DbBookmarks(object):
     def __init__(self, default=[]):
         self.bookmarks = list(default) # want a copy (not an alias)
@@ -1356,12 +1391,8 @@ class DbBsddbRead(DbReadBase, Callback):
         return self.__has_handle(self.source_map, handle)
 
     def __sortbyperson_key(self, person):
-        surnlist = self.person_map.get(str(person))[3][5]
+        return locale.strxfrm(find_surname(str(person), 
-        if surnlist:
+                                           self.person_map.get(str(person))))
-            surn = " ".join([x[0] for x in surnlist])
-        else:
-            surn = ""
-        return locale.strxfrm(surn)
 
     def __sortbyplace(self, first, second):
         return locale.strcoll(self.place_map.get(str(first))[2], 

src/gen/db/upgrade.py

@@ -25,17 +25,17 @@ from __future__ import with_statement
 """
 methods to upgrade a database from version 13 to current version
 """
-
+from bsddb import db
 from gen.db import BSDDBTxn
 from gen.lib.nameorigintype import NameOriginType
-
+from gen.db.write import _mkname, SURNAMES
 
 def gramps_upgrade_15(self):
     """Upgrade database from version 14 to 15. This upgrade adds:
          * tagging
          * surname list
     """
-    length = len(self.person_map)
+    length = len(self.person_map)+10
     self.set_total(length)
 
     # ---------------------------------
@@ -94,7 +94,13 @@ def gramps_upgrade_15(self):
                       )
         with BSDDBTxn(self.env, self.person_map) as txn:
             txn.put(str(handle), new_person)
-        self.update()
+        self.update(length)
+    #surname is now different, remove secondary index with names
+    _db = db.DB(self.env)
+    try:
+        _db.remove(_mkname(self.full_name, SURNAMES), SURNAMES)
+    except db.DBNoSuchFileError:
+        pass
 
     # Bump up database version. Separate transaction to save metadata.
     with BSDDBTxn(self.env, self.metadata) as txn:

src/gen/db/write.py

@@ -52,7 +52,7 @@ from gen.lib import (GenderStats, Person, Family, Event, Place, Source,
                      MediaObject, Repository, Note)
 from gen.db import (DbBsddbRead, DbWriteBase, BSDDBTxn, 
                     DbTxn, BsddbBaseCursor, DbVersionError, 
-                    DbUpgradeRequiredError, 
+                    DbUpgradeRequiredError, find_surname, find_surname_name,
                     DbUndoBSDDB as DbUndo)
 from gen.db.dbconst import *
 from gen.utils.callback import Callback
@@ -122,9 +122,6 @@ KEY_TO_CLASS_MAP = {PERSON_KEY: Person.__name__,
 #
 #-------------------------------------------------------------------------
 
-def find_surname(key, data):
-    return str(data[3][5])
-
 def find_idmap(key, data):
     return str(data[1])
 
@@ -1315,7 +1312,8 @@ class DbBsddb(DbBsddbRead, DbWriteBase, UpdateCallback):
         """
         Build surname list for use in autocompletion
         """
-        self.surname_list = sorted(map(unicode, set(self.surnames.keys())), key=locale.strxfrm)
+        self.surname_list = sorted(map(unicode, set(self.surnames.keys())), 
+                                   key=locale.strxfrm)
 
     def add_to_surname_list(self, person, batch_transaction):
         """
@@ -1323,7 +1321,8 @@ class DbBsddb(DbBsddbRead, DbWriteBase, UpdateCallback):
         """
         if batch_transaction:
             return
-        name = unicode(person.get_primary_name().get_surname())
+        name = unicode(find_surname_name(person.handle, 
+                                        person.get_primary_name().serialize()))
         i = bisect.bisect(self.surname_list, name)
         if 0 < i <= len(self.surname_list):
             if self.surname_list[i-1] != name:
@@ -1340,7 +1339,8 @@ class DbBsddb(DbBsddbRead, DbWriteBase, UpdateCallback):
         If not then we need to remove the name from the list.
         The function must be overridden in the derived class.
         """
-        name = str(person.get_primary_name().get_surname())
+        name = str(find_surname_name(person.handle, 
+                                     person.get_primary_name().serialize()))
         try:
             cursor = self.surnames.cursor(txn=self.txn)
             cursor.set(name)
@@ -1400,7 +1400,10 @@ class DbBsddb(DbBsddbRead, DbWriteBase, UpdateCallback):
                 self.genderStats.count_person(person)
 
             # Update surname list if necessary
-            if (old_person.primary_name.surname !=person.primary_name.surname):
+            if (find_surname_name(old_person.handle, 
+                                  old_person.primary_name.serialize()) != 
+                    find_surname_name(person.handle,
+                                  person.primary_name.serialize())):
                 self.remove_from_surname_list(old_person)
                 self.add_to_surname_list(person, transaction.batch)
         else:

src/plugins/import/ImportGrdb.py

@@ -77,8 +77,9 @@ def find_surname(key, data):
     """
     Return the surname from the data stream. Used for building a secondary 
     index.
+    This function is not needed, as we don't use the secondary index.
     """
-    return str(data[3][5])
+    return str("a")
 
 def find_idmap(key, data):
     """
@@ -138,6 +139,10 @@ class GrampsBSDDB(DbGrdb, UpdateCallback):
         This is replaced for internal use by gen/db/dbdir.py
         However, this class is still used for import of the 2.2.x 
         GRDB format. In 3.0+ this format is no longer used.
+        
+        We only need to upgrade the old main tables.
+        That will be used to append data to the database this GrampsBSDDB is 
+        imported to.
     """
 
     def __init__(self, use_txn = True):
@@ -1247,19 +1252,6 @@ class GrampsBSDDB(DbGrdb, UpdateCallback):
         self.surname_list = list(set(self.surnames.keys()))
         self.sort_surname_list()
 
-    def remove_from_surname_list(self, person):
-        """
-        Check whether there are persons with the same surname left in
-        the database. If not then we need to remove the name from the list.
-        The function must be overridden in the derived class.
-        """
-        name = str(person.get_primary_name().get_surname())
-        try:
-            if self.surnames.keys().count(name) == 1:
-                self.surname_list.remove(unicode(name))
-        except ValueError:
-            pass
-
     def __get_obj_from_gramps_id(self, val, tbl, class_init, prim_tbl):
         if tbl.has_key(str(val)):
         #if str(val) in tbl:
@@ -1563,6 +1555,8 @@ class GrampsBSDDB(DbGrdb, UpdateCallback):
             self.gramps_upgrade_13()
         if version < 14:
             self.gramps_upgrade_14()
+        if version < 15:
+            self.gramps_upgrade_15()
         LOG.debug("Upgrade time: %s %s", int(time.time()-t), "seconds")
 
     def gramps_upgrade_10(self):
@@ -2589,6 +2583,108 @@ class GrampsBSDDB(DbGrdb, UpdateCallback):
                 name_type, prefix, patronymic,
                 group_as, sort_as, display_as, call)
 
+    def gramps_upgrade_15(self):
+        """Upgrade database from version 14 to 15. This upgrade adds:
+             * tagging
+             * surname list
+        """
+        length = len(self.person_map)+10
+        self.set_total(length)
+
+        # ---------------------------------
+        # Modify Person
+        # ---------------------------------
+        for handle in self.person_map.keys():
+            person = self.person_map[handle]
+            (junk_handle,        #  0
+             gramps_id,          #  1
+             gender,             #  2
+             primary_name,       #  3
+             alternate_names,    #  4
+             death_ref_index,    #  5
+             birth_ref_index,    #  6
+             event_ref_list,     #  7
+             family_list,        #  8
+             parent_family_list, #  9
+             media_list,         # 10
+             address_list,       # 11
+             attribute_list,     # 12
+             urls,               # 13
+             ord_list,           # 14
+             psource_list,       # 15
+             pnote_list,         # 16
+             change,             # 17
+             marker,             # 18
+             pprivate,           # 19
+             person_ref_list,    # 20
+             ) = person
+
+            new_primary_name = self.convert_name_15(primary_name)
+            new_alternate_names = [self.convert_name_15(altname) for altname in 
+                                                                alternate_names]
+            new_person = (junk_handle,        #  0
+                          gramps_id,          #  1
+                          gender,             #  2
+                          new_primary_name,   #  3
+                          new_alternate_names,#  4
+                          death_ref_index,    #  5
+                          birth_ref_index,    #  6
+                          event_ref_list,     #  7
+                          family_list,        #  8
+                          parent_family_list, #  9
+                          media_list,         # 10
+                          address_list,       # 11
+                          attribute_list,     # 12
+                          urls,               # 13
+                          ord_list,           # 14
+                          psource_list,       # 15
+                          pnote_list,         # 16
+                          change,             # 17
+                          marker,             # 18
+                          pprivate,           # 19
+                          person_ref_list,    # 20
+                          []                  # 21, tags
+                          )
+            the_txn = self.env.txn_begin()
+            self.person_map.put(str(handle), new_person, txn=the_txn)
+            the_txn.commit()
+            self.update(length)
+
+        #surname is now different, normally remove secondary index with names
+        #we skip this, as this database will not be used after the import
+
+        # Bump up database version. Separate transaction to save metadata.
+        the_txn = self.env.txn_begin()
+        self.metadata.put('version', 15, txn=the_txn)
+        the_txn.commit()
+
+    def convert_name_15(self, name):
+        (privacy, source_list, note_list, date, 
+         first_name, surname, suffix, title,
+         name_type, prefix, patronymic,
+         group_as, sort_as, display_as, call) = name
+        
+        connector = u""
+        origintype = (NameOriginType.NONE, u"")
+        patorigintype = (NameOriginType.PATRONYMIC, u"")
+        
+        if patronymic.strip() == u"":
+            #no patronymic, create a single surname
+            surname_list = [(surname, prefix, True, origintype, connector)]
+        else:
+            #a patronymic, if no surname or equal as patronymic, a single surname
+            if (surname.strip() == u"") or (surname == patronymic and prefix == u""):
+                surname_list = [(patronymic, prefix, True, patorigintype, connector)]
+            else:
+                #two surnames, first patronymic, then surname which is primary
+                surname_list = [(patronymic, u"", False, patorigintype, u""),
+                                (surname, prefix, True, origintype, connector)]
+        
+        #return new value, add two empty strings for nick and family nick
+        return (privacy, source_list, note_list, date,
+             first_name, surname_list, suffix, title, name_type,
+             group_as, sort_as, display_as, call, u"", u"")
+
     def set_auto_remove(self):
         """
         BSDDB change log settings using new method with renamed attributes

src/plugins/tool/Check.py

@@ -878,6 +878,7 @@ class CheckIntegrity(object):
              marker,             # 18
              pprivate,           # 19
              person_ref_list,    # 20
+             tags,               # 21
              ) = person
             # Take apart person reference list:
             new_person_ref_list = []
@@ -935,6 +936,7 @@ class CheckIntegrity(object):
                               marker,             # 18
                               pprivate,           # 19
                               new_person_ref_list,    # 20
+                              tags,               # 21
                               )
                 p = gen.lib.Person(new_person)
                 self.db.commit_person(p, self.trans)