9742: Rewrite cursors to avoid using table maps

gramps/gen/db/dbconst.py

@@ -56,6 +56,7 @@ DBCACHE   = 0x4000000       # Size of the shared memory buffer pool
 DBLOCKS   = 100000          # Maximum number of locks supported
 DBOBJECTS = 100000          # Maximum number of simultaneously locked objects
 DBUNDO    = 1000            # Maximum size of undo buffer
+ARRAYSIZE = 1000            # The arraysize for a SQL cursor
 
 PERSON_KEY     = 0
 FAMILY_KEY     = 1

gramps/gen/db/generic.py

@@ -2,6 +2,7 @@
 # Gramps - a GTK+/GNOME based genealogy program
 #
 # Copyright (C) 2015-2016 Gramps Development Team
+# Copyright (C) 2016      Nick Hall
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -305,14 +306,14 @@ class MetaCursor:
         pass
 
 class Cursor:
-    def __init__(self, map):
+    def __init__(self, iterator):
-        self.map = map
+        self.iterator = iterator
         self._iter = self.__iter__()
     def __enter__(self):
         return self
     def __iter__(self):
-        for item in self.map.keys():
+        for handle, data in self.iterator():
-            yield (item, self.map[item])
+            yield (handle, data)
     def __next__(self):
         try:
             return self._iter.__next__()
@@ -321,8 +322,8 @@ class Cursor:
     def __exit__(self, *args, **kwargs):
         pass
     def iter(self):
-        for item in self.map.keys():
+        for handle, data in self.iterator():
-            yield (item, self.map[item])
+            yield (handle, data)
     def first(self):
         self._iter = self.__iter__()
         try:
@@ -1295,37 +1296,37 @@ class DbGeneric(DbWriteBase, DbReadBase, UpdateCallback, Callback):
         return Note.create(data)
 
     def get_place_cursor(self):
-        return Cursor(self.place_map)
+        return Cursor(self._iter_raw_place_data)
 
     def get_place_tree_cursor(self, *args, **kwargs):
         return TreeCursor(self, self.place_map)
 
     def get_person_cursor(self):
-        return Cursor(self.person_map)
+        return Cursor(self._iter_raw_person_data)
 
     def get_family_cursor(self):
-        return Cursor(self.family_map)
+        return Cursor(self._iter_raw_family_data)
 
     def get_event_cursor(self):
-        return Cursor(self.event_map)
+        return Cursor(self._iter_raw_event_data)
 
     def get_note_cursor(self):
-        return Cursor(self.note_map)
+        return Cursor(self._iter_raw_note_data)
 
     def get_tag_cursor(self):
-        return Cursor(self.tag_map)
+        return Cursor(self._iter_raw_tag_data)
 
     def get_repository_cursor(self):
-        return Cursor(self.repository_map)
+        return Cursor(self._iter_raw_repository_data)
 
     def get_media_cursor(self):
-        return Cursor(self.media_map)
+        return Cursor(self._iter_raw_media_data)
 
     def get_citation_cursor(self):
-        return Cursor(self.citation_map)
+        return Cursor(self._iter_raw_citation_data)
 
     def get_source_cursor(self):
-        return Cursor(self.source_map)
+        return Cursor(self._iter_raw_source_data)
 
     def add_person(self, person, trans, set_gid=True):
         if not person.handle:

gramps/plugins/db/dbapi/dbapi.py

@@ -2,6 +2,7 @@
 # Gramps - a GTK+/GNOME based genealogy program
 #
 # Copyright (C) 2015-2016 Douglas S. Blank <doug.blank@gmail.com>
+# Copyright (C) 2016      Nick Hall
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -1370,6 +1371,80 @@ class DBAPI(DbGeneric):
         for row in rows:
             yield row[0]
 
+    def _iter_raw_data(self, obj_key):
+        """
+        Return an iterator over raw data in the database.
+        """
+        table = KEY_TO_NAME_MAP[obj_key]
+        sql = "SELECT handle, blob_data FROM %s" % table
+        with self.dbapi.cursor() as cursor:
+            cursor.execute(sql)
+            rows = cursor.fetchmany()
+            while rows:
+                for row in rows:
+                    yield (row[0].encode('utf8'), pickle.loads(row[1]))
+                rows = cursor.fetchmany()
+
+    def _iter_raw_person_data(self):
+        """
+        Return an iterator over raw Person data.
+        """
+        return self._iter_raw_data(PERSON_KEY)
+
+    def _iter_raw_family_data(self):
+        """
+        Return an iterator over raw Family data.
+        """
+        return self._iter_raw_data(FAMILY_KEY)
+
+    def _iter_raw_event_data(self):
+        """
+        Return an iterator over raw Event data.
+        """
+        return self._iter_raw_data(EVENT_KEY)
+
+    def _iter_raw_place_data(self):
+        """
+        Return an iterator over raw Place data.
+        """
+        return self._iter_raw_data(PLACE_KEY)
+
+    def _iter_raw_repository_data(self):
+        """
+        Return an iterator over raw Repository data.
+        """
+        return self._iter_raw_data(REPOSITORY_KEY)
+
+    def _iter_raw_source_data(self):
+        """
+        Return an iterator over raw Source data.
+        """
+        return self._iter_raw_data(SOURCE_KEY)
+
+    def _iter_raw_citation_data(self):
+        """
+        Return an iterator over raw Citation data.
+        """
+        return self._iter_raw_data(CITATION_KEY)
+
+    def _iter_raw_media_data(self):
+        """
+        Return an iterator over raw Media data.
+        """
+        return self._iter_raw_data(MEDIA_KEY)
+
+    def _iter_raw_note_data(self):
+        """
+        Return an iterator over raw Note data.
+        """
+        return self._iter_raw_data(NOTE_KEY)
+
+    def _iter_raw_tag_data(self):
+        """
+        Return an iterator over raw Tag data.
+        """
+        return self._iter_raw_data(TAG_KEY)
+
     def reindex_reference_map(self, callback):
         """
         Reindex all primary records in the database.

gramps/plugins/db/dbapi/mysql.py

@@ -2,6 +2,7 @@
 # Gramps - a GTK+/GNOME based genealogy program
 #
 # Copyright (C) 2015-2016 Douglas S. Blank <doug.blank@gmail.com>
+# Copyright (C) 2016      Nick Hall
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -18,9 +19,21 @@
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 #
 
+#-------------------------------------------------------------------------
+#
+# Standard python modules
+#
+#-------------------------------------------------------------------------
 import MySQLdb
 import re
 
+#-------------------------------------------------------------------------
+#
+# Gramps modules
+#
+#-------------------------------------------------------------------------
+from gramps.gen.db.dbconst import ARRAYSIZE
+
 MySQLdb.paramstyle = 'qmark' ## Doesn't work
 
 class MySQL:
@@ -40,9 +53,9 @@ class MySQL:
         return summary
 
     def __init__(self, *args, **kwargs):
-        self.connection = MySQLdb.connect(*args, **kwargs)
+        self.__connection = MySQLdb.connect(*args, **kwargs)
-        self.connection.autocommit(True)
+        self.__connection.autocommit(True)
-        self.cursor = self.connection.cursor()
+        self.__cursor = self.__connection.cursor()
 
     def _hack_query(self, query):
         ## Workaround: no qmark support:
@@ -70,27 +83,61 @@ class MySQL:
 
     def execute(self, query, args=[]):
         query = self._hack_query(query)
-        self.cursor.execute(query, args)
+        self.__cursor.execute(query, args)
 
     def fetchone(self):
-        return self.cursor.fetchone()
+        return self.__cursor.fetchone()
 
     def fetchall(self):
-        return self.cursor.fetchall()
+        return self.__cursor.fetchall()
 
     def commit(self):
-        self.cursor.execute("COMMIT;");
+        self.__cursor.execute("COMMIT;");
 
     def begin(self):
-        self.cursor.execute("BEGIN;");
+        self.__cursor.execute("BEGIN;");
 
     def rollback(self):
-        self.connection.rollback()
+        self.__connection.rollback()
 
     def table_exists(self, table):
-        self.cursor.execute("SELECT COUNT(*) FROM information_schema.tables "
+        self.__cursor.execute("SELECT COUNT(*) "
-                            "WHERE table_name='%s';" % table)
+                              "FROM information_schema.tables "
+                              "WHERE table_name='%s';" % table)
         return self.fetchone()[0] != 0
 
     def close(self):
-        self.connection.close()
+        self.__connection.close()
+    def cursor(self):
+        return Cursor(self.__connection)
+
+
+class Cursor:
+    def __init__(self, connection):
+        self.__connection = connection
+
+    def __enter__(self):
+        self.__cursor = self.__connection.cursor()
+        self.__cursor.arraysize = ARRAYSIZE
+        return self
+
+    def __exit__(self, *args, **kwargs):
+        self.__cursor.close()
+
+    def execute(self, *args, **kwargs):
+        """
+        Executes an SQL statement.
+
+        :param args: arguments to be passed to the sqlite3 execute statement
+        :type args: list
+        :param kwargs: arguments to be passed to the sqlite3 execute statement
+        :type kwargs: list
+        """
+        self.__cursor.execute(*args, **kwargs)
+
+    def fetchmany(self):
+        """
+        Fetches the next set of rows of a query result, returning a list. An
+        empty list is returned when no more rows are available.
+        """
+        return self.__cursor.fetchmany()

gramps/plugins/db/dbapi/postgresql.py

@@ -2,6 +2,7 @@
 # Gramps - a GTK+/GNOME based genealogy program
 #
 # Copyright (C) 2015-2016 Douglas S. Blank <doug.blank@gmail.com>
+# Copyright (C) 2016      Nick Hall
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -18,9 +19,21 @@
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 #
 
+#-------------------------------------------------------------------------
+#
+# Standard python modules
+#
+#-------------------------------------------------------------------------
 import psycopg2
 import re
 
+#-------------------------------------------------------------------------
+#
+# Gramps modules
+#
+#-------------------------------------------------------------------------
+from gramps.gen.db.dbconst import ARRAYSIZE
+
 psycopg2.paramstyle = 'format'
 
 class Postgresql:
@@ -40,9 +53,9 @@ class Postgresql:
         return summary
 
     def __init__(self, *args, **kwargs):
-        self.connection = psycopg2.connect(*args, **kwargs)
+        self.__connection = psycopg2.connect(*args, **kwargs)
-        self.connection.autocommit = True
+        self.__connection.autocommit = True
-        self.cursor = self.connection.cursor()
+        self.__cursor = self.__connection.cursor()
 
     def _hack_query(self, query):
         query = query.replace("?", "%s")
@@ -71,33 +84,71 @@ class Postgresql:
         else:
             args = None
         try:
-            self.cursor.execute(sql, args, **kwargs)
+            self.__cursor.execute(sql, args, **kwargs)
         except:
-            self.cursor.execute("rollback")
+            self.__cursor.execute("rollback")
             raise
 
     def fetchone(self):
         try:
-            return self.cursor.fetchone()
+            return self.__cursor.fetchone()
         except:
             return None
 
     def fetchall(self):
-        return self.cursor.fetchall()
+        return self.__cursor.fetchall()
 
     def begin(self):
-        self.cursor.execute("BEGIN;")
+        self.__cursor.execute("BEGIN;")
 
     def commit(self):
-        self.cursor.execute("COMMIT;")
+        self.__cursor.execute("COMMIT;")
 
     def rollback(self):
-        self.connection.rollback()
+        self.__connection.rollback()
 
     def table_exists(self, table):
-        self.cursor.execute("SELECT COUNT(*) FROM information_schema.tables "
+        self.__cursor.execute("SELECT COUNT(*) "
-                            "WHERE table_name=?;", [table])
+                              "FROM information_schema.tables "
+                              "WHERE table_name=?;", [table])
         return self.fetchone()[0] != 0
 
     def close(self):
-        self.connection.close()
+        self.__connection.close()
+
+    def cursor(self):
+        return Cursor(self.__connection)
+
+
+class Cursor:
+    def __init__(self, connection):
+        self.__connection = connection
+
+    def __enter__(self):
+        self.__cursor = self.__connection.cursor()
+        self.__cursor.arraysize = ARRAYSIZE
+        return self
+
+    def __exit__(self, *args, **kwargs):
+        self.__cursor.close()
+
+    def execute(self, *args, **kwargs):
+        """
+        Executes an SQL statement.
+
+        :param args: arguments to be passed to the sqlite3 execute statement
+        :type args: list
+        :param kwargs: arguments to be passed to the sqlite3 execute statement
+        :type kwargs: list
+        """
+        self.__cursor.execute(*args, **kwargs)
+
+    def fetchmany(self):
+        """
+        Fetches the next set of rows of a query result, returning a list. An
+        empty list is returned when no more rows are available.
+        """
+        try:
+            return self.__cursor.fetchmany()
+        except:
+            return None

gramps/plugins/db/dbapi/sqlite.py

@@ -2,6 +2,7 @@
 # Gramps - a GTK+/GNOME based genealogy program
 #
 # Copyright (C) 2015-2016 Douglas S. Blank <doug.blank@gmail.com>
+# Copyright (C) 2016      Nick Hall
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -31,6 +32,13 @@ import sqlite3
 import logging
 import re
 
+#-------------------------------------------------------------------------
+#
+# Gramps modules
+#
+#-------------------------------------------------------------------------
+from gramps.gen.db.dbconst import ARRAYSIZE
+
 sqlite3.paramstyle = 'qmark'
 
 #-------------------------------------------------------------------------
@@ -72,10 +80,9 @@ class Sqlite:
         :type kwargs: list
         """
         self.log = logging.getLogger(".sqlite")
-        self.connection = sqlite3.connect(*args, **kwargs)
+        self.__connection = sqlite3.connect(*args, **kwargs)
-        self.cursor = self.connection.cursor()
+        self.__cursor = self.__connection.cursor()
-        self.queries = {}
+        self.__connection.create_function("regexp", 2, regexp)
-        self.connection.create_function("regexp", 2, regexp)
 
     def execute(self, *args, **kwargs):
         """
@@ -87,21 +94,21 @@ class Sqlite:
         :type kwargs: list
         """
         self.log.debug(args)
-        self.cursor.execute(*args, **kwargs)
+        self.__cursor.execute(*args, **kwargs)
 
     def fetchone(self):
         """
         Fetches the next row of a query result set, returning a single sequence,
         or None when no more data is available.
         """
-        return self.cursor.fetchone()
+        return self.__cursor.fetchone()
 
     def fetchall(self):
         """
         Fetches the next set of rows of a query result, returning a list. An
         empty list is returned when no more rows are available.
         """
-        return self.cursor.fetchall()
+        return self.__cursor.fetchall()
 
     def begin(self):
         """
@@ -116,14 +123,14 @@ class Sqlite:
         Commit the current transaction.
         """
         self.log.debug("COMMIT;")
-        self.connection.commit()
+        self.__connection.commit()
 
     def rollback(self):
         """
         Roll back any changes to the database since the last call to commit().
         """
         self.log.debug("ROLLBACK;")
-        self.connection.rollback()
+        self.__connection.rollback()
 
     def table_exists(self, table):
         """
@@ -134,7 +141,8 @@ class Sqlite:
         :returns: True if the table exists, false otherwise.
         :rtype: bool
         """
-        self.execute("SELECT COUNT(*) FROM sqlite_master "
+        self.execute("SELECT COUNT(*) "
+                     "FROM sqlite_master "
                      "WHERE type='table' AND name='%s';" % table)
         return self.fetchone()[0] != 0
 
@@ -143,7 +151,50 @@ class Sqlite:
         Close the current database.
         """
         self.log.debug("closing database...")
-        self.connection.close()
+        self.__connection.close()
+
+    def cursor(self):
+        """
+        Return a new cursor.
+        """
+        return Cursor(self.__connection)
+
+
+#-------------------------------------------------------------------------
+#
+# Cursor class
+#
+#-------------------------------------------------------------------------
+class Cursor:
+    def __init__(self, connection):
+        self.__connection = connection
+
+    def __enter__(self):
+        self.__cursor = self.__connection.cursor()
+        self.__cursor.arraysize = ARRAYSIZE
+        return self
+
+    def __exit__(self, *args, **kwargs):
+        self.__cursor.close()
+
+    def execute(self, *args, **kwargs):
+        """
+        Executes an SQL statement.
+
+        :param args: arguments to be passed to the sqlite3 execute statement
+        :type args: list
+        :param kwargs: arguments to be passed to the sqlite3 execute statement
+        :type kwargs: list
+        """
+        self.__cursor.execute(*args, **kwargs)
+
+    def fetchmany(self):
+        """
+        Fetches the next set of rows of a query result, returning a list. An
+        empty list is returned when no more rows are available.
+        """
+        return self.__cursor.fetchmany()
+
 
 def regexp(expr, value):
     """

gramps/plugins/db/dbapi/test/db_test.py

@@ -342,6 +342,57 @@ class DbTest(unittest.TestCase):
         for gramps_id in self.gids['Note']:
             self.assertTrue(self.db.has_note_gramps_id(gramps_id))
 
+    ################################################################
+    #
+    # Test get_*_cursor methods
+    #
+    ################################################################
+    def __get_cursor_test(self, cursor_func, raw_func):
+        with cursor_func() as cursor:
+            for handle, data1 in cursor:
+                data2 = raw_func(handle)
+                self.assertEqual(data1, data2)
+
+    def test_get_person_cursor(self):
+        self.__get_cursor_test(self.db.get_person_cursor,
+                               self.db.get_raw_person_data)
+
+    def test_get_family_cursor(self):
+        self.__get_cursor_test(self.db.get_family_cursor,
+                               self.db.get_raw_family_data)
+
+    def test_get_event_cursor(self):
+        self.__get_cursor_test(self.db.get_event_cursor,
+                               self.db.get_raw_event_data)
+
+    def test_get_place_cursor(self):
+        self.__get_cursor_test(self.db.get_place_cursor,
+                               self.db.get_raw_place_data)
+
+    def test_get_repository_cursor(self):
+        self.__get_cursor_test(self.db.get_repository_cursor,
+                               self.db.get_raw_repository_data)
+
+    def test_get_source_cursor(self):
+        self.__get_cursor_test(self.db.get_source_cursor,
+                               self.db.get_raw_source_data)
+
+    def test_get_citation_cursor(self):
+        self.__get_cursor_test(self.db.get_citation_cursor,
+                               self.db.get_raw_citation_data)
+
+    def test_get_media_cursor(self):
+        self.__get_cursor_test(self.db.get_media_cursor,
+                               self.db.get_raw_media_data)
+
+    def test_get_note_cursor(self):
+        self.__get_cursor_test(self.db.get_note_cursor,
+                               self.db.get_raw_note_data)
+
+    def test_get_tag_cursor(self):
+        self.__get_cursor_test(self.db.get_tag_cursor,
+                               self.db.get_raw_tag_data)
+
 
 if __name__ == "__main__":
     unittest.main()