diff --git a/gramps/gen/db/dbconst.py b/gramps/gen/db/dbconst.py index 543381ca8..8aa3d0044 100644 --- a/gramps/gen/db/dbconst.py +++ b/gramps/gen/db/dbconst.py @@ -56,6 +56,7 @@ DBCACHE = 0x4000000 # Size of the shared memory buffer pool DBLOCKS = 100000 # Maximum number of locks supported DBOBJECTS = 100000 # Maximum number of simultaneously locked objects DBUNDO = 1000 # Maximum size of undo buffer +ARRAYSIZE = 1000 # The arraysize for a SQL cursor PERSON_KEY = 0 FAMILY_KEY = 1 diff --git a/gramps/gen/db/generic.py b/gramps/gen/db/generic.py index 431bb9139..e373b699f 100644 --- a/gramps/gen/db/generic.py +++ b/gramps/gen/db/generic.py @@ -2,6 +2,7 @@ # Gramps - a GTK+/GNOME based genealogy program # # Copyright (C) 2015-2016 Gramps Development Team +# Copyright (C) 2016 Nick Hall # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -305,14 +306,14 @@ class MetaCursor: pass class Cursor: - def __init__(self, map): - self.map = map + def __init__(self, iterator): + self.iterator = iterator self._iter = self.__iter__() def __enter__(self): return self def __iter__(self): - for item in self.map.keys(): - yield (item, self.map[item]) + for handle, data in self.iterator(): + yield (handle, data) def __next__(self): try: return self._iter.__next__() @@ -321,8 +322,8 @@ class Cursor: def __exit__(self, *args, **kwargs): pass def iter(self): - for item in self.map.keys(): - yield (item, self.map[item]) + for handle, data in self.iterator(): + yield (handle, data) def first(self): self._iter = self.__iter__() try: @@ -1295,37 +1296,37 @@ class DbGeneric(DbWriteBase, DbReadBase, UpdateCallback, Callback): return Note.create(data) def get_place_cursor(self): - return Cursor(self.place_map) + return Cursor(self._iter_raw_place_data) def get_place_tree_cursor(self, *args, **kwargs): return TreeCursor(self, self.place_map) def get_person_cursor(self): - return Cursor(self.person_map) + return Cursor(self._iter_raw_person_data) def get_family_cursor(self): - return Cursor(self.family_map) + return Cursor(self._iter_raw_family_data) def get_event_cursor(self): - return Cursor(self.event_map) + return Cursor(self._iter_raw_event_data) def get_note_cursor(self): - return Cursor(self.note_map) + return Cursor(self._iter_raw_note_data) def get_tag_cursor(self): - return Cursor(self.tag_map) + return Cursor(self._iter_raw_tag_data) def get_repository_cursor(self): - return Cursor(self.repository_map) + return Cursor(self._iter_raw_repository_data) def get_media_cursor(self): - return Cursor(self.media_map) + return Cursor(self._iter_raw_media_data) def get_citation_cursor(self): - return Cursor(self.citation_map) + return Cursor(self._iter_raw_citation_data) def get_source_cursor(self): - return Cursor(self.source_map) + return Cursor(self._iter_raw_source_data) def add_person(self, person, trans, set_gid=True): if not person.handle: diff --git a/gramps/plugins/db/dbapi/dbapi.py b/gramps/plugins/db/dbapi/dbapi.py index 4c28811b9..05db9faed 100644 --- a/gramps/plugins/db/dbapi/dbapi.py +++ b/gramps/plugins/db/dbapi/dbapi.py @@ -2,6 +2,7 @@ # Gramps - a GTK+/GNOME based genealogy program # # Copyright (C) 2015-2016 Douglas S. Blank +# Copyright (C) 2016 Nick Hall # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -1370,6 +1371,80 @@ class DBAPI(DbGeneric): for row in rows: yield row[0] + def _iter_raw_data(self, obj_key): + """ + Return an iterator over raw data in the database. + """ + table = KEY_TO_NAME_MAP[obj_key] + sql = "SELECT handle, blob_data FROM %s" % table + with self.dbapi.cursor() as cursor: + cursor.execute(sql) + rows = cursor.fetchmany() + while rows: + for row in rows: + yield (row[0].encode('utf8'), pickle.loads(row[1])) + rows = cursor.fetchmany() + + def _iter_raw_person_data(self): + """ + Return an iterator over raw Person data. + """ + return self._iter_raw_data(PERSON_KEY) + + def _iter_raw_family_data(self): + """ + Return an iterator over raw Family data. + """ + return self._iter_raw_data(FAMILY_KEY) + + def _iter_raw_event_data(self): + """ + Return an iterator over raw Event data. + """ + return self._iter_raw_data(EVENT_KEY) + + def _iter_raw_place_data(self): + """ + Return an iterator over raw Place data. + """ + return self._iter_raw_data(PLACE_KEY) + + def _iter_raw_repository_data(self): + """ + Return an iterator over raw Repository data. + """ + return self._iter_raw_data(REPOSITORY_KEY) + + def _iter_raw_source_data(self): + """ + Return an iterator over raw Source data. + """ + return self._iter_raw_data(SOURCE_KEY) + + def _iter_raw_citation_data(self): + """ + Return an iterator over raw Citation data. + """ + return self._iter_raw_data(CITATION_KEY) + + def _iter_raw_media_data(self): + """ + Return an iterator over raw Media data. + """ + return self._iter_raw_data(MEDIA_KEY) + + def _iter_raw_note_data(self): + """ + Return an iterator over raw Note data. + """ + return self._iter_raw_data(NOTE_KEY) + + def _iter_raw_tag_data(self): + """ + Return an iterator over raw Tag data. + """ + return self._iter_raw_data(TAG_KEY) + def reindex_reference_map(self, callback): """ Reindex all primary records in the database. diff --git a/gramps/plugins/db/dbapi/mysql.py b/gramps/plugins/db/dbapi/mysql.py index c25630f92..0bca9ec35 100644 --- a/gramps/plugins/db/dbapi/mysql.py +++ b/gramps/plugins/db/dbapi/mysql.py @@ -2,6 +2,7 @@ # Gramps - a GTK+/GNOME based genealogy program # # Copyright (C) 2015-2016 Douglas S. Blank +# Copyright (C) 2016 Nick Hall # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -18,9 +19,21 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # +#------------------------------------------------------------------------- +# +# Standard python modules +# +#------------------------------------------------------------------------- import MySQLdb import re +#------------------------------------------------------------------------- +# +# Gramps modules +# +#------------------------------------------------------------------------- +from gramps.gen.db.dbconst import ARRAYSIZE + MySQLdb.paramstyle = 'qmark' ## Doesn't work class MySQL: @@ -40,9 +53,9 @@ class MySQL: return summary def __init__(self, *args, **kwargs): - self.connection = MySQLdb.connect(*args, **kwargs) - self.connection.autocommit(True) - self.cursor = self.connection.cursor() + self.__connection = MySQLdb.connect(*args, **kwargs) + self.__connection.autocommit(True) + self.__cursor = self.__connection.cursor() def _hack_query(self, query): ## Workaround: no qmark support: @@ -70,27 +83,61 @@ class MySQL: def execute(self, query, args=[]): query = self._hack_query(query) - self.cursor.execute(query, args) + self.__cursor.execute(query, args) def fetchone(self): - return self.cursor.fetchone() + return self.__cursor.fetchone() def fetchall(self): - return self.cursor.fetchall() + return self.__cursor.fetchall() def commit(self): - self.cursor.execute("COMMIT;"); + self.__cursor.execute("COMMIT;"); def begin(self): - self.cursor.execute("BEGIN;"); + self.__cursor.execute("BEGIN;"); def rollback(self): - self.connection.rollback() + self.__connection.rollback() def table_exists(self, table): - self.cursor.execute("SELECT COUNT(*) FROM information_schema.tables " - "WHERE table_name='%s';" % table) + self.__cursor.execute("SELECT COUNT(*) " + "FROM information_schema.tables " + "WHERE table_name='%s';" % table) return self.fetchone()[0] != 0 def close(self): - self.connection.close() + self.__connection.close() + def cursor(self): + return Cursor(self.__connection) + + +class Cursor: + def __init__(self, connection): + self.__connection = connection + + def __enter__(self): + self.__cursor = self.__connection.cursor() + self.__cursor.arraysize = ARRAYSIZE + return self + + def __exit__(self, *args, **kwargs): + self.__cursor.close() + + def execute(self, *args, **kwargs): + """ + Executes an SQL statement. + + :param args: arguments to be passed to the sqlite3 execute statement + :type args: list + :param kwargs: arguments to be passed to the sqlite3 execute statement + :type kwargs: list + """ + self.__cursor.execute(*args, **kwargs) + + def fetchmany(self): + """ + Fetches the next set of rows of a query result, returning a list. An + empty list is returned when no more rows are available. + """ + return self.__cursor.fetchmany() diff --git a/gramps/plugins/db/dbapi/postgresql.py b/gramps/plugins/db/dbapi/postgresql.py index fecd023a1..e02fc2fe4 100644 --- a/gramps/plugins/db/dbapi/postgresql.py +++ b/gramps/plugins/db/dbapi/postgresql.py @@ -2,6 +2,7 @@ # Gramps - a GTK+/GNOME based genealogy program # # Copyright (C) 2015-2016 Douglas S. Blank +# Copyright (C) 2016 Nick Hall # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -18,9 +19,21 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # +#------------------------------------------------------------------------- +# +# Standard python modules +# +#------------------------------------------------------------------------- import psycopg2 import re +#------------------------------------------------------------------------- +# +# Gramps modules +# +#------------------------------------------------------------------------- +from gramps.gen.db.dbconst import ARRAYSIZE + psycopg2.paramstyle = 'format' class Postgresql: @@ -40,9 +53,9 @@ class Postgresql: return summary def __init__(self, *args, **kwargs): - self.connection = psycopg2.connect(*args, **kwargs) - self.connection.autocommit = True - self.cursor = self.connection.cursor() + self.__connection = psycopg2.connect(*args, **kwargs) + self.__connection.autocommit = True + self.__cursor = self.__connection.cursor() def _hack_query(self, query): query = query.replace("?", "%s") @@ -71,33 +84,71 @@ class Postgresql: else: args = None try: - self.cursor.execute(sql, args, **kwargs) + self.__cursor.execute(sql, args, **kwargs) except: - self.cursor.execute("rollback") + self.__cursor.execute("rollback") raise def fetchone(self): try: - return self.cursor.fetchone() + return self.__cursor.fetchone() except: return None def fetchall(self): - return self.cursor.fetchall() + return self.__cursor.fetchall() def begin(self): - self.cursor.execute("BEGIN;") + self.__cursor.execute("BEGIN;") def commit(self): - self.cursor.execute("COMMIT;") + self.__cursor.execute("COMMIT;") def rollback(self): - self.connection.rollback() + self.__connection.rollback() def table_exists(self, table): - self.cursor.execute("SELECT COUNT(*) FROM information_schema.tables " - "WHERE table_name=?;", [table]) + self.__cursor.execute("SELECT COUNT(*) " + "FROM information_schema.tables " + "WHERE table_name=?;", [table]) return self.fetchone()[0] != 0 def close(self): - self.connection.close() + self.__connection.close() + + def cursor(self): + return Cursor(self.__connection) + + +class Cursor: + def __init__(self, connection): + self.__connection = connection + + def __enter__(self): + self.__cursor = self.__connection.cursor() + self.__cursor.arraysize = ARRAYSIZE + return self + + def __exit__(self, *args, **kwargs): + self.__cursor.close() + + def execute(self, *args, **kwargs): + """ + Executes an SQL statement. + + :param args: arguments to be passed to the sqlite3 execute statement + :type args: list + :param kwargs: arguments to be passed to the sqlite3 execute statement + :type kwargs: list + """ + self.__cursor.execute(*args, **kwargs) + + def fetchmany(self): + """ + Fetches the next set of rows of a query result, returning a list. An + empty list is returned when no more rows are available. + """ + try: + return self.__cursor.fetchmany() + except: + return None diff --git a/gramps/plugins/db/dbapi/sqlite.py b/gramps/plugins/db/dbapi/sqlite.py index dac1ae920..6726a416c 100644 --- a/gramps/plugins/db/dbapi/sqlite.py +++ b/gramps/plugins/db/dbapi/sqlite.py @@ -2,6 +2,7 @@ # Gramps - a GTK+/GNOME based genealogy program # # Copyright (C) 2015-2016 Douglas S. Blank +# Copyright (C) 2016 Nick Hall # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -31,6 +32,13 @@ import sqlite3 import logging import re +#------------------------------------------------------------------------- +# +# Gramps modules +# +#------------------------------------------------------------------------- +from gramps.gen.db.dbconst import ARRAYSIZE + sqlite3.paramstyle = 'qmark' #------------------------------------------------------------------------- @@ -72,10 +80,9 @@ class Sqlite: :type kwargs: list """ self.log = logging.getLogger(".sqlite") - self.connection = sqlite3.connect(*args, **kwargs) - self.cursor = self.connection.cursor() - self.queries = {} - self.connection.create_function("regexp", 2, regexp) + self.__connection = sqlite3.connect(*args, **kwargs) + self.__cursor = self.__connection.cursor() + self.__connection.create_function("regexp", 2, regexp) def execute(self, *args, **kwargs): """ @@ -87,21 +94,21 @@ class Sqlite: :type kwargs: list """ self.log.debug(args) - self.cursor.execute(*args, **kwargs) + self.__cursor.execute(*args, **kwargs) def fetchone(self): """ Fetches the next row of a query result set, returning a single sequence, or None when no more data is available. """ - return self.cursor.fetchone() + return self.__cursor.fetchone() def fetchall(self): """ Fetches the next set of rows of a query result, returning a list. An empty list is returned when no more rows are available. """ - return self.cursor.fetchall() + return self.__cursor.fetchall() def begin(self): """ @@ -116,14 +123,14 @@ class Sqlite: Commit the current transaction. """ self.log.debug("COMMIT;") - self.connection.commit() + self.__connection.commit() def rollback(self): """ Roll back any changes to the database since the last call to commit(). """ self.log.debug("ROLLBACK;") - self.connection.rollback() + self.__connection.rollback() def table_exists(self, table): """ @@ -134,7 +141,8 @@ class Sqlite: :returns: True if the table exists, false otherwise. :rtype: bool """ - self.execute("SELECT COUNT(*) FROM sqlite_master " + self.execute("SELECT COUNT(*) " + "FROM sqlite_master " "WHERE type='table' AND name='%s';" % table) return self.fetchone()[0] != 0 @@ -143,7 +151,50 @@ class Sqlite: Close the current database. """ self.log.debug("closing database...") - self.connection.close() + self.__connection.close() + + def cursor(self): + """ + Return a new cursor. + """ + return Cursor(self.__connection) + + +#------------------------------------------------------------------------- +# +# Cursor class +# +#------------------------------------------------------------------------- +class Cursor: + def __init__(self, connection): + self.__connection = connection + + def __enter__(self): + self.__cursor = self.__connection.cursor() + self.__cursor.arraysize = ARRAYSIZE + return self + + def __exit__(self, *args, **kwargs): + self.__cursor.close() + + def execute(self, *args, **kwargs): + """ + Executes an SQL statement. + + :param args: arguments to be passed to the sqlite3 execute statement + :type args: list + :param kwargs: arguments to be passed to the sqlite3 execute statement + :type kwargs: list + """ + self.__cursor.execute(*args, **kwargs) + + def fetchmany(self): + """ + Fetches the next set of rows of a query result, returning a list. An + empty list is returned when no more rows are available. + """ + return self.__cursor.fetchmany() + def regexp(expr, value): """ diff --git a/gramps/plugins/db/dbapi/test/db_test.py b/gramps/plugins/db/dbapi/test/db_test.py index f3b3d57e1..28f586854 100644 --- a/gramps/plugins/db/dbapi/test/db_test.py +++ b/gramps/plugins/db/dbapi/test/db_test.py @@ -342,6 +342,57 @@ class DbTest(unittest.TestCase): for gramps_id in self.gids['Note']: self.assertTrue(self.db.has_note_gramps_id(gramps_id)) + ################################################################ + # + # Test get_*_cursor methods + # + ################################################################ + def __get_cursor_test(self, cursor_func, raw_func): + with cursor_func() as cursor: + for handle, data1 in cursor: + data2 = raw_func(handle) + self.assertEqual(data1, data2) + + def test_get_person_cursor(self): + self.__get_cursor_test(self.db.get_person_cursor, + self.db.get_raw_person_data) + + def test_get_family_cursor(self): + self.__get_cursor_test(self.db.get_family_cursor, + self.db.get_raw_family_data) + + def test_get_event_cursor(self): + self.__get_cursor_test(self.db.get_event_cursor, + self.db.get_raw_event_data) + + def test_get_place_cursor(self): + self.__get_cursor_test(self.db.get_place_cursor, + self.db.get_raw_place_data) + + def test_get_repository_cursor(self): + self.__get_cursor_test(self.db.get_repository_cursor, + self.db.get_raw_repository_data) + + def test_get_source_cursor(self): + self.__get_cursor_test(self.db.get_source_cursor, + self.db.get_raw_source_data) + + def test_get_citation_cursor(self): + self.__get_cursor_test(self.db.get_citation_cursor, + self.db.get_raw_citation_data) + + def test_get_media_cursor(self): + self.__get_cursor_test(self.db.get_media_cursor, + self.db.get_raw_media_data) + + def test_get_note_cursor(self): + self.__get_cursor_test(self.db.get_note_cursor, + self.db.get_raw_note_data) + + def test_get_tag_cursor(self): + self.__get_cursor_test(self.db.get_tag_cursor, + self.db.get_raw_tag_data) + if __name__ == "__main__": unittest.main()