From ad3dcb8d13bfb268b80d4d1ad1c8e94a7567b222 Mon Sep 17 00:00:00 2001 From: Doug Blank Date: Sun, 28 Feb 2016 15:40:16 -0500 Subject: [PATCH] Hide db._select(); Refactor db._tables; QuerySet; tests and bug fixes db.select() has become db._select() This commit refactors database._tables so that they can work properly after a proxy or filter has been removed. It adds abstract API called QuerySet which allows a variety of selections and ordering of data. Logging was added to sqlite to better see db access. Additional fixes: * clearing GenericDB._directory * test_util_test off-by-one error --- .travis.yml | 2 +- example/gramps/data.gramps | 4 +- gramps/gen/db/base.py | 657 +++++++++++++----- gramps/gen/db/generic.py | 121 ++-- gramps/gen/lib/handle.py | 2 +- gramps/gen/lib/struct.py | 8 +- gramps/gen/lib/test/struct_test.py | 6 +- gramps/gen/merge/diff.py | 16 +- gramps/gen/plug/menu/_enumeratedlist.py | 1 + gramps/gen/proxy/filter.py | 146 +++- gramps/gen/proxy/living.py | 146 +++- gramps/gen/proxy/private.py | 117 +++- gramps/gen/proxy/proxybase.py | 182 ++++- gramps/gen/proxy/referencedbyselection.py | 148 +++- gramps/plugins/database/bsddb_support/read.py | 222 +++--- .../plugins/database/bsddb_support/write.py | 55 +- gramps/plugins/database/dbapi.py | 162 ++--- .../plugins/database/dbapi_support/sqlite.py | 3 + gramps/plugins/database/test/db_test.py | 160 ++++- gramps/test/test/test_util_test.py | 5 +- 20 files changed, 1645 insertions(+), 518 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4e33ebb51..0c7e0ac0f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,7 +31,7 @@ script: - mkdir -p ~/.gramps/grampsdb/ # --exclude=TestUser because of older version of mock # without configure_mock - - nosetests3 --with-coverage --cover-package=gramps --exclude=TestcaseGenerator --exclude=vcard --exclude=merge_ref_test --exclude=user_test gramps + - nosetests3 --nologcapture --with-coverage --cover-package=gramps --exclude=TestcaseGenerator --exclude=vcard --exclude=merge_ref_test --exclude=user_test gramps after_success: - codecov diff --git a/example/gramps/data.gramps b/example/gramps/data.gramps index f240a568e..5f5be95ee 100644 --- a/example/gramps/data.gramps +++ b/example/gramps/data.gramps @@ -3,7 +3,7 @@ "http://gramps-project.org/xml/1.7.1/grampsxml.dtd">
- + Alex Roitman 1122 Boogie Boogie Ave @@ -901,7 +901,7 @@ - + M Edwin Michael diff --git a/gramps/gen/db/base.py b/gramps/gen/db/base.py index 6236def26..4daac2252 100644 --- a/gramps/gen/db/base.py +++ b/gramps/gen/db/base.py @@ -32,6 +32,7 @@ from this class. #------------------------------------------------------------------------- import re import time +from operator import itemgetter #------------------------------------------------------------------------- # @@ -45,6 +46,35 @@ from ..lib.childref import ChildRef from .txn import DbTxn from .exceptions import DbTransactionCancel +def eval_order_by(order_by, obj, db): + """ + Given a list of [[field, DIRECTION], ...] + return the list of values of the fields + """ + values = [] + for (field, direction) in order_by: + values.append(obj.get_field(field, db, ignore_errors=True)) + return values + +def sort_objects(objects, order_by, db): + """ + Python-based sorting. + """ + # first build sort order: + sorted_items = [] + map_items = {} + for obj in objects: + # just use values and handle to keep small: + sorted_items.append((eval_order_by(order_by, obj, db), obj.handle)) + map_items[obj.handle] = obj + # next we sort by fields and direction + pos = len(order_by) - 1 + for (field, order) in reversed(order_by): # sort the lasts parts first + sorted_items.sort(key=itemgetter(pos), reverse=(order=="DESC")) + pos -= 1 + for (order_by_values, handle) in sorted_items: + yield map_items[handle] + #------------------------------------------------------------------------- # # Gramps libraries @@ -67,18 +97,12 @@ class DbReadBase(object): """ self.basedb = self self.__feature = {} # {"feature": VALUE, ...} - self._tables = { - "Citation": {}, - "Event": {}, - "Family": {}, - "Media": {}, - "Note": {}, - "Person": {}, - "Place": {}, - "Repository": {}, - "Source": {}, - "Tag": {}, - } + + def get_table_func(self, table=None, func=None): + """ + Base implementation of get_table_func. + """ + return None def get_feature(self, feature): """ @@ -1234,6 +1258,186 @@ class DbReadBase(object): """ raise NotImplementedError + def _select(self, table, fields=None, start=0, limit=-1, + where=None, order_by=None): + """ + Default implementation of a select for those databases + that don't support SQL. Returns a list of dicts, total, + and time. + + table - Person, Family, etc. + fields - used by object.get_field() + start - position to start + limit - count to get; -1 for all + where - (field, SQL string_operator, value) | + ["AND", [where, where, ...]] | + ["OR", [where, where, ...]] | + ["NOT", where] + order_by - [[fieldname, "ASC" | "DESC"], ...] + """ + def compare(v, op, value): + """ + Compare values in a SQL-like way + """ + if isinstance(v, (list, tuple)) and len(v) > 0: # join, or multi-values + # If any is true: + for item in v: + if compare(item, op, value): + return True + return False + if op == "=": + matched = v == value + elif op == ">": + matched = v > value + elif op == ">=": + matched = v >= value + elif op == "<": + matched = v < value + elif op == "<=": + matched = v <= value + elif op == "IN": + matched = v in value + elif op == "IS": + matched = v is value + elif op == "IS NOT": + matched = v is not value + elif op == "IS NULL": + matched = v is None + elif op == "IS NOT NULL": + matched = v is not None + elif op == "BETWEEN": + matched = value[0] <= v <= value[1] + elif op in ["<>", "!="]: + matched = v != value + elif op == "LIKE": + if value and v: + value = value.replace("%", "(.*)").replace("_", ".") + ## FIXME: allow a case-insensitive version + matched = re.match("^" + value + "$", v, re.MULTILINE) + else: + matched = False + else: + raise Exception("invalid select operator: '%s'" % op) + return True if matched else False + + def evaluate_values(condition, item, db, table, env): + """ + Evaluates the names in all conditions. + """ + if len(condition) == 2: # ["AND" [...]] | ["OR" [...]] | ["NOT" expr] + connector, exprs = condition + if connector in ["AND", "OR"]: + for expr in exprs: + evaluate_values(expr, item, db, table, env) + else: # "NOT" + evaluate_values(exprs, item, db, table, env) + elif len(condition) == 3: # (name, op, value) + (name, op, value) = condition + # just the ones we need for where + hname = self._hash_name(table, name) + if hname not in env: + value = item.get_field(name, db, ignore_errors=True) + env[hname] = value + + def evaluate_truth(condition, item, db, table, env): + if len(condition) == 2: # ["AND"|"OR" [...]] + connector, exprs = condition + if connector == "AND": # all must be true + for expr in exprs: + if not evaluate_truth(expr, item, db, table, env): + return False + return True + elif connector == "OR": # any will return true + for expr in exprs: + if evaluate_truth(expr, item, db, table, env): + return True + return False + elif connector == "NOT": # return not of single value + return not evaluate_truth(exprs, item, db, table, env) + else: + raise Exception("No such connector: '%s'" % connector) + elif len(condition) == 3: # (name, op, value) + (name, op, value) = condition + v = env.get(self._hash_name(table, name)) + return compare(v, op, value) + + # Fields is None or list, maybe containing "*": + if fields is None: + pass # ok + elif not isinstance(fields, (list, tuple)): + raise Exception("fields must be a list/tuple of field names") + elif "*" in fields: + fields.remove("*") + fields.extend(self.get_table_func(table,"class_func").get_schema().keys()) + get_count_only = (fields is not None and fields[0] == "count(1)") + position = 0 + selected = 0 + if get_count_only: + if where or limit != -1 or start != 0: + # no need to order for a count + data = self.get_table_func(table,"iter_func")() + else: + yield self.get_table_func(table,"count_func")() + else: + data = self.get_table_func(table, "iter_func")(order_by=order_by) + if where: + for item in data: + # Go through all fliters and evaluate the fields: + env = {} + evaluate_values(where, item, self, table, env) + matched = evaluate_truth(where, item, self, table, env) + if matched: + if ((selected < limit) or (limit == -1)) and start <= position: + selected += 1 + if not get_count_only: + if fields: + row = {} + for field in fields: + value = item.get_field(field, self, ignore_errors=True) + row[field.replace("__", ".")] = value + yield row + else: + yield item + position += 1 + if get_count_only: + yield selected + else: # no where + for item in data: + if position >= start: + if ((selected >= limit) and (limit != -1)): + break + selected += 1 + if not get_count_only: + if fields: + row = {} + for field in fields: + value = item.get_field(field, self, ignore_errors=True) + row[field.replace("__", ".")] = value + yield row + else: + yield item + position += 1 + if get_count_only: + yield selected + + def _hash_name(self, table, name): + """ + Used in SQL functions to eval expressions involving selected + data. + """ + name = self.get_table_func(table,"class_func").get_field_alias(name) + return name.replace(".", "__") + + Person = property(lambda self:QuerySet(self, "Person")) + Family = property(lambda self:QuerySet(self, "Family")) + Note = property(lambda self:QuerySet(self, "Note")) + Citation = property(lambda self:QuerySet(self, "Citation")) + Source = property(lambda self:QuerySet(self, "Source")) + Repository = property(lambda self:QuerySet(self, "Repository")) + Place = property(lambda self:QuerySet(self, "Place")) + Event = property(lambda self:QuerySet(self, "Event")) + Tag = property(lambda self:QuerySet(self, "Tag")) + class DbWriteBase(DbReadBase): """ Gramps database object. This object is a base class for all @@ -1877,180 +2081,279 @@ class DbWriteBase(DbReadBase): else: raise ValueError("invalid instance type: %s" % instance.__class__.__name__) - def select(self, table, fields=None, start=0, limit=-1, - where=None, order_by=None): + def get_queryset_by_table_name(self, table_name): """ - Default implementation of a select for those databases - that don't support SQL. Returns a list of dicts, total, - and time. + Get Person, Family queryset by name. + """ + return getattr(self, table_name) - table - Person, Family, etc. - fields - used by object.get_field() - start - position to start - limit - count to get; -1 for all - where - (field, SQL string_operator, value) | - ["AND", [where, where, ...]] | - ["OR", [where, where, ...]] | - ["NOT", where] - order_by - [[fieldname, "ASC" | "DESC"], ...] - """ - class Result(list): - """ - A list rows of just matching for this page, with total = all, - time = time to select, expanded (unpickled), query (N/A). - """ - total = 0 - time = 0.0 - expanded = True - query = None - def compare(v, op, value): - """ - Compare values in a SQL-like way - """ - if isinstance(v, (list, tuple)): # join, or multi-values - # If any is true: - for item in v: - if compare(item, op, value): - return True - return False - if op == "=": - matched = v == value - elif op == ">": - matched = v > value - elif op == ">=": - matched = v >= value - elif op == "<": - matched = v < value - elif op == "<=": - matched = v <= value - elif op == "IN": - matched = v in value - elif op == "IS": - matched = v is value - elif op == "IS NOT": - matched = v is not value - elif op == "IS NULL": - matched = v is None - elif op == "IS NOT NULL": - matched = v is not None - elif op == "BETWEEN": - matched = value[0] <= v <= value[1] - elif op in ["<>", "!="]: - matched = v != value - elif op == "LIKE": - if value and v: - value = value.replace("%", "(.*)").replace("_", ".") - ## FIXME: allow a case-insensitive version - matched = re.match("^" + value + "$", v, re.MULTILINE) - else: - matched = False +class Operator(object): + """ + Base for QuerySet operators. + """ + op = "OP" + def __init__(self, *expressions, **kwargs): + if self.op in ["AND", "OR"]: + exprs = [expression.list for expression + in expressions] + for key in kwargs: + exprs.append( + _select_field_operator_value(key, "=", kwargs[key])) + else: # "NOT" + if expressions: + exprs = expressions.list else: - raise Exception("invalid select operator: '%s'" % op) - return True if matched else False + key, value = list(kwargs.items())[0] + exprs = _select_field_operator_value(key, "=", value) + self.list = [self.op, exprs] - def evaluate_values(condition, item, db, table, env): - """ - Evaluates the names in all conditions. - """ - if len(condition) == 2: # ["AND" [...]] | ["OR" [...]] | ["NOT" expr] - connector, exprs = condition - if connector in ["AND", "OR"]: - for expr in exprs: - evaluate_values(expr, item, db, table, env) - else: # "NOT" - evaluate_values(exprs, item, db, table, env) - elif len(condition) == 3: # (name, op, value) - (name, op, value) = condition - # just the ones we need for where - hname = self._hash_name(table, name) - if hname not in env: - value = item.get_field(name, db, ignore_errors=True) - env[hname] = value +class AND(Operator): + op = "AND" - def evaluate_truth(condition, item, db, table, env): - if len(condition) == 2: # ["AND"|"OR" [...]] - connector, exprs = condition - if connector == "AND": # all must be true - for expr in exprs: - if not evaluate_truth(expr, item, db, table, env): - return False - return True - elif connector == "OR": # any will return true - for expr in exprs: - if evaluate_truth(expr, item, db, table, env): - return True - return False - elif connector == "NOT": # return not of single value - return not evaluate_truth(exprs, item, db, table, env) +class OR(Operator): + """ + OR operator for QuerySet logical WHERE expressions. + """ + op = "OR" + +class NOT(Operator): + """ + NOT operator for QuerySet logical WHERE expressions. + """ + op = "NOT" + +class QuerySet(object): + """ + A container for selection criteria before being actually + applied to a database. + """ + def __init__(self, database, table): + self.database = database + self.table = table + self.generator = None + self.where_by = None + self.order_by = None + self.limit_by = -1 + self.start = 0 + self.needs_to_run = False + + def limit(self, start=None, count=None): + """ + Put limits on the selection. + """ + if start is not None: + self.start = start + if count is not None: + self.limit_by = count + self.needs_to_run = True + return self + + def order(self, *args): + """ + Put an ordering on the selection. + """ + for arg in args: + if self.order_by is None: + self.order_by = [] + if arg.startswith("-"): + self.order_by.append((arg[1:], "DESC")) + else: + self.order_by.append((arg, "ASC")) + self.needs_to_run = True + return self + + def _add_where_clause(self, *args, **kwargs): + """ + Add a condition to the where clause. + """ + # First, handle AND, OR, NOT args: + and_expr = [] + for arg in args: + expr = arg.list + and_expr.append(expr) + # Next, handle kwargs: + for keyword in kwargs: + and_expr.append( + _select_field_operator_value( + keyword, "=", kwargs[keyword])) + if and_expr: + if self.where_by: + self.where_by = ["AND", [self.where_by] + and_expr] + elif len(and_expr) == 1: + self.where_by = and_expr[0] + else: + self.where_by = ["AND", and_expr] + self.needs_to_run = True + return self + + def count(self): + """ + Run query with just where, start, limit to get count. + """ + if self.generator and self.needs_to_run: + raise Exception("Queries in invalid order") + elif self.generator: + return len(list(self.generator)) + else: + generator = self.database._select(self.table, + ["count(1)"], + where=self.where_by, + start=self.start, + limit=self.limit_by) + return next(generator) + + def _generate(self, args=None): + """ + Create a generator from current options. + """ + generator = self.database._select(self.table, + args, + order_by=self.order_by, + where=self.where_by, + start=self.start, + limit=self.limit_by) + # Reset all criteria + self.where_by = None + self.order_by = None + self.limit_by = -1 + self.start = 0 + self.needs_to_run = False + return generator + + def select(self, *args): + """ + Actually touch the database. + """ + if len(args) == 0: + args = None + if self.generator and self.needs_to_run: + ## problem + raise Exception("Queries in invalid order") + elif self.generator: + if args: # there is a generator, with args + for i in self.generator: + yield [i.get_field(arg) for arg in args] + else: # generator, no args + for i in self.generator: + yield i + else: # need to run or not + self.generator = self._generate(args) + for i in self.generator: + yield i + + def proxy(self, proxy_name, *args, **kwargs): + """ + Apply a named proxy to the db. + """ + from gramps.gen.proxy import (LivingProxyDb, PrivateProxyDb, + ReferencedBySelectionProxyDb) + if proxy_name == "living": + proxy_class = LivingProxyDb + elif proxy_name == "private": + proxy_class = PrivateProxyDb + elif proxy_name == "referenced": + proxy_class = ReferencedBySelectionProxyDb + else: + raise Exception("No such proxy name: '%s'" % proxy_name) + self.database = proxy_class(self.database, *args, **kwargs) + return self + + def filter(self, *args, **kwargs): + """ + Apply a filter to the database. + """ + from gramps.gen.proxy import FilterProxyDb + from gramps.gen.filters import GenericFilter + for i in range(len(args)): + arg = args[i] + if isinstance(arg, GenericFilter): + self.database = FilterProxyDb(self.database, arg, *args[i+1:]) + if arg.where_by: + self._add_where_clause(arg.where_by) + elif isinstance(arg, Operator): + self._add_where_clause(arg) + elif callable(arg): + if self.generator and self.needs_to_run: + ## error + raise Exception("Queries in invalid order") + elif self.generator: + pass # ok else: - raise Exception("No such connector: '%s'" % connector) - elif len(condition) == 3: # (name, op, value) - (name, op, value) = condition - v = env.get(self._hash_name(table, name)) - return compare(v, op, value) + self.generator = self._generate() + self.generator = filter(arg, self.generator) + else: + pass # ignore, may have been arg from previous Filter + if kwargs: + self._add_where_clause(**kwargs) + return self - # Fields is None or list, maybe containing "*": - if fields is None: - fields = ["*"] - elif not isinstance(fields, (list, tuple)): - raise Exception("fields must be a list/tuple of field names") - if "*" in fields: - fields.remove("*") - fields.extend(self._tables[table]["class_func"].get_schema().keys()) - data = self._tables[table]["iter_func"](order_by=order_by) - position = 0 - selected = 0 - result = Result() - start_time = time.time() - if where: - for item in data: - # have to evaluate all, because there is a where - row = {} - env = {} - # Go through all fliters and evaluate the fields: - evaluate_values(where, item, self, table, env) - matched = evaluate_truth(where, item, self, table, env) - if matched: - if ((selected < limit) or (limit == -1)) and start <= position: - # now, we get all of the fields - for field in fields: - value = item.get_field(field, self, ignore_errors=True) - row[field.replace("__", ".")] = value - selected += 1 - result.append(row) - position += 1 - result.total = position - else: # no where - for item in data: - if position >= start: - if ((selected >= limit) and (limit != -1)): - break - row = {} - for field in fields: - value = item.get_field(field, self, ignore_errors=True) - row[field.replace("__", ".")] = value - result.append(row) - selected += 1 - position += 1 - result.total = self._tables[table]["count_func"]() - result.time = time.time() - start_time - return result + def map(self, f): + """ + Apply the function f to the selected items and return results. + """ + if self.generator and self.needs_to_run: + raise Exception("Queries in invalid order") + elif self.generator: + pass # ok + else: + self.generator = self._generate() + previous_generator = self.generator + def generator(): + for item in previous_generator: + yield f(item) + self.generator = generator() + return self - def _hash_name(self, table, name): + def tag(self, tag_text): """ - Used in SQL functions to eval expressions involving selected - data. + Tag the selected items with the tag name. """ - name = self._tables[table]["class_func"].get_field_alias(name) - return name.replace(".", "__") + if self.generator and self.needs_to_run: + raise Exception("Queries in invalid order") + elif self.generator: + pass # ok + else: + self.generator = self._generate() + tag = self.database.get_tag_from_name(tag_text) + trans_class = self.database.get_transaction_class() + with trans_class("Tag Selected Items", self.database, batch=True) as trans: + if tag is None: + tag = self.database.get_table_func("Tag","class_func")() + tag.set_name(tag_text) + self.database.add_tag(tag, trans) + commit_func = self.database.get_table_func(self.table,"commit_func") + for item in self.generator: + if tag.handle not in item.tag_list: + item.add_tag(tag.handle) + commit_func(item, trans) - def eval_order_by(self, order_by, obj): - """ - Given a list of [[field, DIRECTION], ...] - return the list of values of the fields - """ - values = [] - for (field, direction) in order_by: - values.append(obj.get_field(field, self, ignore_errors=True)) - return values +def _to_dot_format(field): + """ + Convert a field keyword arg into a proper + dotted field name. + """ + return field.replace("__", ".") +def _select_field_operator_value(field, op, value): + """ + Convert a field keyword arg into proper + field, op, and value. + """ + alias = { + "LT": "<", + "GT": ">", + "LTE": "<=", + "GTE": ">=", + "IS_NOT": "IS NOT", + "IS_NULL": "IS NULL", + "IS_NOT_NULL": "IS NOT NULL", + "NE": "<>", + } + for operator in ["LIKE", "IN"] + list(alias.keys()): + operator = "__" + operator + if field.endswith(operator): + op = field[-len(operator) + 2:] + field = field[:-len(operator)] + op = alias.get(op, op) + field = _to_dot_format(field) + return (field, op, value) diff --git a/gramps/gen/db/generic.py b/gramps/gen/db/generic.py index bd2ffe634..0cbc42b04 100644 --- a/gramps/gen/db/generic.py +++ b/gramps/gen/db/generic.py @@ -46,20 +46,15 @@ from gramps.gen.const import GRAMPS_LOCALE as glocale _ = glocale.translation.gettext from gramps.gen.db import (DbReadBase, DbWriteBase, DbTxn, DbUndo, KEY_TO_NAME_MAP, KEY_TO_CLASS_MAP, - CLASS_TO_KEY_MAP, TXNADD, TXNUPD, TXNDEL) + CLASS_TO_KEY_MAP, TXNADD, TXNUPD, TXNDEL, + PERSON_KEY, FAMILY_KEY, CITATION_KEY, + SOURCE_KEY, EVENT_KEY, MEDIA_KEY, + PLACE_KEY, REPOSITORY_KEY, NOTE_KEY, + TAG_KEY, eval_order_by) +from gramps.gen.db.base import QuerySet from gramps.gen.utils.callback import Callback from gramps.gen.updatecallback import UpdateCallback from gramps.gen.db.dbconst import * -from gramps.gen.db import (PERSON_KEY, - FAMILY_KEY, - CITATION_KEY, - SOURCE_KEY, - EVENT_KEY, - MEDIA_KEY, - PLACE_KEY, - REPOSITORY_KEY, - NOTE_KEY, - TAG_KEY) from gramps.gen.utils.id import create_id from gramps.gen.lib.researcher import Researcher @@ -227,7 +222,7 @@ class Table(object): if funcs: self.funcs = funcs else: - self.funcs = db._tables[table_name] + self.funcs = db.get_table_func(table_name) def cursor(self): """ @@ -439,7 +434,8 @@ class DbGeneric(DbWriteBase, DbReadBase, UpdateCallback, Callback): DbReadBase.__init__(self) DbWriteBase.__init__(self) Callback.__init__(self) - self._tables['Person'].update( + self.__tables = { + 'Person': { "handle_func": self.get_person_from_handle, "gramps_id_func": self.get_person_from_gramps_id, @@ -456,8 +452,8 @@ class DbGeneric(DbWriteBase, DbReadBase, UpdateCallback, Callback): "raw_func": self._get_raw_person_data, "raw_id_func": self._get_raw_person_from_id_data, "del_func": self.remove_person, - }) - self._tables['Family'].update( + }, + 'Family': { "handle_func": self.get_family_from_handle, "gramps_id_func": self.get_family_from_gramps_id, @@ -474,8 +470,8 @@ class DbGeneric(DbWriteBase, DbReadBase, UpdateCallback, Callback): "raw_func": self._get_raw_family_data, "raw_id_func": self._get_raw_family_from_id_data, "del_func": self.remove_family, - }) - self._tables['Source'].update( + }, + 'Source': { "handle_func": self.get_source_from_handle, "gramps_id_func": self.get_source_from_gramps_id, @@ -492,8 +488,8 @@ class DbGeneric(DbWriteBase, DbReadBase, UpdateCallback, Callback): "raw_func": self._get_raw_source_data, "raw_id_func": self._get_raw_source_from_id_data, "del_func": self.remove_source, - }) - self._tables['Citation'].update( + }, + 'Citation': { "handle_func": self.get_citation_from_handle, "gramps_id_func": self.get_citation_from_gramps_id, @@ -510,8 +506,8 @@ class DbGeneric(DbWriteBase, DbReadBase, UpdateCallback, Callback): "raw_func": self._get_raw_citation_data, "raw_id_func": self._get_raw_citation_from_id_data, "del_func": self.remove_citation, - }) - self._tables['Event'].update( + }, + 'Event': { "handle_func": self.get_event_from_handle, "gramps_id_func": self.get_event_from_gramps_id, @@ -528,8 +524,8 @@ class DbGeneric(DbWriteBase, DbReadBase, UpdateCallback, Callback): "raw_func": self._get_raw_event_data, "raw_id_func": self._get_raw_event_from_id_data, "del_func": self.remove_event, - }) - self._tables['Media'].update( + }, + 'Media': { "handle_func": self.get_media_from_handle, "gramps_id_func": self.get_media_from_gramps_id, @@ -546,8 +542,8 @@ class DbGeneric(DbWriteBase, DbReadBase, UpdateCallback, Callback): "raw_func": self._get_raw_media_data, "raw_id_func": self._get_raw_media_from_id_data, "del_func": self.remove_media, - }) - self._tables['Place'].update( + }, + 'Place': { "handle_func": self.get_place_from_handle, "gramps_id_func": self.get_place_from_gramps_id, @@ -564,8 +560,8 @@ class DbGeneric(DbWriteBase, DbReadBase, UpdateCallback, Callback): "raw_func": self._get_raw_place_data, "raw_id_func": self._get_raw_place_from_id_data, "del_func": self.remove_place, - }) - self._tables['Repository'].update( + }, + 'Repository': { "handle_func": self.get_repository_from_handle, "gramps_id_func": self.get_repository_from_gramps_id, @@ -582,8 +578,8 @@ class DbGeneric(DbWriteBase, DbReadBase, UpdateCallback, Callback): "raw_func": self._get_raw_repository_data, "raw_id_func": self._get_raw_repository_from_id_data, "del_func": self.remove_repository, - }) - self._tables['Note'].update( + }, + 'Note': { "handle_func": self.get_note_from_handle, "gramps_id_func": self.get_note_from_gramps_id, @@ -600,8 +596,8 @@ class DbGeneric(DbWriteBase, DbReadBase, UpdateCallback, Callback): "raw_func": self._get_raw_note_data, "raw_id_func": self._get_raw_note_from_id_data, "del_func": self.remove_note, - }) - self._tables['Tag'].update( + }, + 'Tag': { "handle_func": self.get_tag_from_handle, "gramps_id_func": None, @@ -615,7 +611,8 @@ class DbGeneric(DbWriteBase, DbReadBase, UpdateCallback, Callback): "count_func": self.get_number_of_tags, "raw_func": self._get_raw_tag_data, "del_func": self.remove_tag, - }) + } + } self.set_save_path(directory) # skip GEDCOM cross-ref check for now: self.set_feature("skip-check-xref", True) @@ -725,6 +722,19 @@ class DbGeneric(DbWriteBase, DbReadBase, UpdateCallback, Callback): if directory: self.load(directory) + def get_table_func(self, table=None, func=None): + """ + Private implementation of get_table_func. + """ + if table is None: + return self.__tables.keys() + elif func is None: + return self.__tables[table] # dict of functions + elif func in self.__tables[table].keys(): + return self.__tables[table][func] + else: + return super().get_table_func(table, func) + def load(self, directory, callback=None, mode=None, force_schema_upgrade=False, force_bsddb_upgrade=False, @@ -801,12 +811,12 @@ class DbGeneric(DbWriteBase, DbReadBase, UpdateCallback, Callback): def get_table_names(self): """Return a list of valid table names.""" - return list(self._tables.keys()) + return list(self.get_table_func()) def get_table_metadata(self, table_name): """Return the metadata for a valid table name.""" - if table_name in self._tables: - return self._tables[table_name] + if table_name in self.get_table_func(): + return self.get_table_func(table_name) return None def transaction_begin(self, transaction): @@ -1154,7 +1164,7 @@ class DbGeneric(DbWriteBase, DbReadBase, UpdateCallback, Callback): Iterate over items in a class, possibly ordered by a list of field names and direction ("ASC" or "DESC"). """ - cursor = self._tables[class_.__name__]["cursor_func"] + cursor = self.get_table_func(class_.__name__,"cursor_func") if order_by is None: for data in cursor(): yield class_.create(data[1]) @@ -1164,7 +1174,7 @@ class DbGeneric(DbWriteBase, DbReadBase, UpdateCallback, Callback): for data in cursor(): obj = class_.create(data[1]) # just use values and handle to keep small: - sorted_items.append((self.eval_order_by(order_by, obj), obj.handle)) + sorted_items.append((eval_order_by(order_by, obj, self), obj.handle)) # next we sort by fields and direction def getitem(item, pos): sort_items = item[0] @@ -1173,17 +1183,17 @@ class DbGeneric(DbWriteBase, DbReadBase, UpdateCallback, Callback): elif sort_items[pos] is None: return "" else: - # FIXME: should do something clever/recurive to + # FIXME: should do something clever/recurive to # sort these meaningfully, and return a string: return str(sort_items[pos]) pos = len(order_by) - 1 for (field, order) in reversed(order_by): # sort the lasts parts first - sorted_items.sort(key=lambda item: getitem(item, pos), + sorted_items.sort(key=lambda item: getitem(item, pos), reverse=(order=="DESC")) pos -= 1 # now we will look them up again: for (order_by_values, handle) in sorted_items: - yield self._tables[class_.__name__]["handle_func"](handle) + yield self.get_table_func(class_.__name__,"handle_func")(handle) def iter_people(self, order_by=None): return self.iter_items(order_by, Person) @@ -1553,9 +1563,9 @@ class DbGeneric(DbWriteBase, DbReadBase, UpdateCallback, Callback): A (possibily) implementation-specific method to get data from db into this database. """ - for key in db._tables.keys(): - cursor = db._tables[key]["cursor_func"] - class_ = db._tables[key]["class_func"] + for key in db.get_table_func(): + cursor = db.get_table_func(key,"cursor_func") + class_ = db.get_table_func(key,"class_func") for (handle, data) in cursor(): map = getattr(self, "%s_map" % key.lower()) if isinstance(handle, bytes): @@ -1578,8 +1588,8 @@ class DbGeneric(DbWriteBase, DbReadBase, UpdateCallback, Callback): >>> self.get_from_name_and_handle("Person", "a7ad62365bc652387008") >>> self.get_from_name_and_handle("Media", "c3434653675bcd736f23") """ - if table_name in self._tables: - return self._tables[table_name]["handle_func"](handle) + if table_name in self.get_table_func(): + return self.get_table_func(table_name,"handle_func")(handle) return None def get_from_name_and_gramps_id(self, table_name, gramps_id): @@ -1593,8 +1603,8 @@ class DbGeneric(DbWriteBase, DbReadBase, UpdateCallback, Callback): >>> self.get_from_name_and_gramps_id("Family", "F056") >>> self.get_from_name_and_gramps_id("Media", "M00012") """ - if table_name in self._tables: - return self._tables[table_name]["gramps_id_func"](gramps_id) + if table_name in self.get_table_func(): + return self.get_table_func(table_name,"gramps_id_func")(gramps_id) return None def remove_source(self, handle, transaction): @@ -1673,8 +1683,8 @@ class DbGeneric(DbWriteBase, DbReadBase, UpdateCallback, Callback): """ Return true if there are no [primary] records in the database """ - for table in self._tables: - if len(self._tables[table]["handles_func"]()) > 0: + for table in self.get_table_func(): + if len(self.get_table_func(table,"handles_func")()) > 0: return False return True @@ -1719,8 +1729,9 @@ class DbGeneric(DbWriteBase, DbReadBase, UpdateCallback, Callback): self.set_metadata('place_types', self.place_types) # Save misc items: - self.save_surname_list() - self.save_gender_stats(self.genderStats) + if self.has_changed: + self.save_surname_list() + self.save_gender_stats(self.genderStats) # Indexes: self.set_metadata('cmap_index', self.cmap_index) @@ -1735,6 +1746,7 @@ class DbGeneric(DbWriteBase, DbReadBase, UpdateCallback, Callback): self.close_backend() self.db_is_open = False + self._directory = None def get_bookmarks(self): return self.bookmarks @@ -2074,3 +2086,10 @@ class DbGeneric(DbWriteBase, DbReadBase, UpdateCallback, Callback): def set_default_person_handle(self, handle): self.set_metadata("default-person-handle", handle) self.emit('home-person-changed') + + def add_table_funcs(self, table, funcs): + """ + Add a new table and funcs to the database. + """ + self.__tables[table] = funcs + setattr(DbGeneric, table, property(lambda self: QuerySet(self, table))) diff --git a/gramps/gen/lib/handle.py b/gramps/gen/lib/handle.py index 8cbaeb5f1..565c47008 100644 --- a/gramps/gen/lib/handle.py +++ b/gramps/gen/lib/handle.py @@ -23,7 +23,7 @@ class HandleClass(str): super(HandleClass, self).__init__() def join(self, database, handle): - return database._tables[self.classname]["handle_func"](handle) + return database.get_table_func(self.classname,"handle_func")(handle) @classmethod def get_schema(cls): diff --git a/gramps/gen/lib/struct.py b/gramps/gen/lib/struct.py index 250dbda5b..13478f583 100644 --- a/gramps/gen/lib/struct.py +++ b/gramps/gen/lib/struct.py @@ -297,20 +297,20 @@ class Struct(object): name, handle = struct["_class"], struct["handle"] old_obj = self.db.get_from_name_and_handle(name, handle) if old_obj: - commit_func = self.db._tables[name]["commit_func"] + commit_func = self.db.get_table_func(name,"commit_func") commit_func(new_obj, trans) else: - add_func = self.db._tables[name]["add_func"] + add_func = self.db.get_table_func(name,"add_func") add_func(new_obj, trans) else: new_obj = Struct.instance_from_struct(struct) name, handle = struct["_class"], struct["handle"] old_obj = self.db.get_from_name_and_handle(name, handle) if old_obj: - commit_func = self.db._tables[name]["commit_func"] + commit_func = self.db.get_table_func(name,"commit_func") commit_func(new_obj, trans) else: - add_func = self.db._tables[name]["add_func"] + add_func = self.db.get_table_func(name,"add_func") add_func(new_obj, trans) def from_struct(self): diff --git a/gramps/gen/lib/test/struct_test.py b/gramps/gen/lib/test/struct_test.py index 258e947b0..debeeb0c0 100644 --- a/gramps/gen/lib/test/struct_test.py +++ b/gramps/gen/lib/test/struct_test.py @@ -109,9 +109,9 @@ def generate_case(obj): #setattr(DatabaseCheck, name, test2) db = import_as_dict("example/gramps/example.gramps", User()) -for table in db._tables.keys(): - for handle in db._tables[table]["handles_func"](): - obj = db._tables[table]["handle_func"](handle) +for table in db.get_table_func(): + for handle in db.get_table_func(table,"handles_func")(): + obj = db.get_table_func(table,"handle_func")(handle) generate_case(obj) class StructTest(unittest.TestCase): diff --git a/gramps/gen/merge/diff.py b/gramps/gen/merge/diff.py index 3eb0d95cd..a53f72c27 100644 --- a/gramps/gen/merge/diff.py +++ b/gramps/gen/merge/diff.py @@ -190,14 +190,14 @@ def diff_dbs(db1, db2, user=None): for item in ['Person', 'Family', 'Source', 'Citation', 'Event', 'Media', 'Place', 'Repository', 'Note', 'Tag']: step() - handles1 = sorted([handle for handle in db1._tables[item]["handles_func"]()]) - handles2 = sorted([handle for handle in db2._tables[item]["handles_func"]()]) + handles1 = sorted([handle for handle in db1.get_table_func(item,"handles_func")()]) + handles2 = sorted([handle for handle in db2.get_table_func(item,"handles_func")()]) p1 = 0 p2 = 0 while p1 < len(handles1) and p2 < len(handles2): if handles1[p1] == handles2[p2]: # in both - item1 = db1._tables[item]["handle_func"](handles1[p1]) - item2 = db2._tables[item]["handle_func"](handles2[p2]) + item1 = db1.get_table_func(item,"handle_func")(handles1[p1]) + item2 = db2.get_tables_func(item,"handle_func")(handles2[p2]) diff = diff_items(item, item1.to_struct(), item2.to_struct()) if diff: diffs += [(item, item1, item2)] @@ -205,19 +205,19 @@ def diff_dbs(db1, db2, user=None): p1 += 1 p2 += 1 elif handles1[p1] < handles2[p2]: # p1 is mssing in p2 - item1 = db1._tables[item]["handle_func"](handles1[p1]) + item1 = db1.get_table_func(item,"handle_func")(handles1[p1]) missing_from_new += [(item, item1)] p1 += 1 elif handles1[p1] > handles2[p2]: # p2 is mssing in p1 - item2 = db2._tables[item]["handle_func"](handles2[p2]) + item2 = db2.get_table_func(item,"handle_func")(handles2[p2]) missing_from_old += [(item, item2)] p2 += 1 while p1 < len(handles1): - item1 = db1._tables[item]["handle_func"](handles1[p1]) + item1 = db1.get_table_func(item,"handle_func")(handles1[p1]) missing_from_new += [(item, item1)] p1 += 1 while p2 < len(handles2): - item2 = db2._tables[item]["handle_func"](handles2[p2]) + item2 = db2.get_table_func(item,"handle_func")(handles2[p2]) missing_from_old += [(item, item2)] p2 += 1 return diffs, missing_from_old, missing_from_new diff --git a/gramps/gen/plug/menu/_enumeratedlist.py b/gramps/gen/plug/menu/_enumeratedlist.py index 7798a84c8..cb95ce4bb 100644 --- a/gramps/gen/plug/menu/_enumeratedlist.py +++ b/gramps/gen/plug/menu/_enumeratedlist.py @@ -123,3 +123,4 @@ class EnumeratedListOption(Option): else: logging.warning(_("Value '%(val)s' not found for option '%(opt)s'") % {'val' : str(value), 'opt' : self.get_label()}) + logging.warning(_("Valid values: ") + str(self.__items)) diff --git a/gramps/gen/proxy/filter.py b/gramps/gen/proxy/filter.py index bf277ad21..f62f24fdc 100644 --- a/gramps/gen/proxy/filter.py +++ b/gramps/gen/proxy/filter.py @@ -30,7 +30,10 @@ Proxy class for the Gramps databases. Apply filter # Gramps libraries # #------------------------------------------------------------------------- +from gramps.gen.db.base import sort_objects from .proxybase import ProxyDbBase +from ..lib import (Date, Person, Name, Surname, NameOriginType, Family, Source, + Citation, Event, Media, Place, Repository, Note, Tag) class FilterProxyDb(ProxyDbBase): """ @@ -70,6 +73,121 @@ class FilterProxyDb(ProxyDbBase): if person: self.flist.update(person.get_family_handle_list()) self.flist.update(person.get_parent_family_handle_list()) + self.__tables = { + 'Person': + { + "handle_func": self.get_person_from_handle, + "gramps_id_func": self.get_person_from_gramps_id, + "class_func": Person, + "cursor_func": self.get_person_cursor, + "handles_func": self.get_person_handles, + "iter_func": self.iter_people, + "count_func": self.get_number_of_people, + }, + 'Family': + { + "handle_func": self.get_family_from_handle, + "gramps_id_func": self.get_family_from_gramps_id, + "class_func": Family, + "cursor_func": self.get_family_cursor, + "handles_func": self.get_family_handles, + "iter_func": self.iter_families, + "count_func": self.get_number_of_families, + }, + 'Source': + { + "handle_func": self.get_source_from_handle, + "gramps_id_func": self.get_source_from_gramps_id, + "class_func": Source, + "cursor_func": self.get_source_cursor, + "handles_func": self.get_source_handles, + "iter_func": self.iter_sources, + "count_func": self.get_number_of_sources, + }, + 'Citation': + { + "handle_func": self.get_citation_from_handle, + "gramps_id_func": self.get_citation_from_gramps_id, + "class_func": Citation, + "cursor_func": self.get_citation_cursor, + "handles_func": self.get_citation_handles, + "iter_func": self.iter_citations, + "count_func": self.get_number_of_citations, + }, + 'Event': + { + "handle_func": self.get_event_from_handle, + "gramps_id_func": self.get_event_from_gramps_id, + "class_func": Event, + "cursor_func": self.get_event_cursor, + "handles_func": self.get_event_handles, + "iter_func": self.iter_events, + "count_func": self.get_number_of_events, + }, + 'Media': + { + "handle_func": self.get_media_from_handle, + "gramps_id_func": self.get_media_from_gramps_id, + "class_func": Media, + "cursor_func": self.get_media_cursor, + "handles_func": self.get_media_handles, + "iter_func": self.iter_media, + "count_func": self.get_number_of_media, + }, + 'Place': + { + "handle_func": self.get_place_from_handle, + "gramps_id_func": self.get_place_from_gramps_id, + "class_func": Place, + "cursor_func": self.get_place_cursor, + "handles_func": self.get_place_handles, + "iter_func": self.iter_places, + "count_func": self.get_number_of_places, + }, + 'Repository': + { + "handle_func": self.get_repository_from_handle, + "gramps_id_func": self.get_repository_from_gramps_id, + "class_func": Repository, + "cursor_func": self.get_repository_cursor, + "handles_func": self.get_repository_handles, + "iter_func": self.iter_repositories, + "count_func": self.get_number_of_repositories, + }, + 'Note': + { + "handle_func": self.get_note_from_handle, + "gramps_id_func": self.get_note_from_gramps_id, + "class_func": Note, + "cursor_func": self.get_note_cursor, + "handles_func": self.get_note_handles, + "iter_func": self.iter_notes, + "count_func": self.get_number_of_notes, + }, + 'Tag': + { + "handle_func": self.get_tag_from_handle, + "gramps_id_func": None, + "class_func": Tag, + "cursor_func": self.get_tag_cursor, + "handles_func": self.get_tag_handles, + "iter_func": self.iter_tags, + "count_func": self.get_number_of_tags, + } + } + + def get_table_func(self, table=None, func=None): + """ + Private implementation of get_table_func. + """ + if table is None: + return self.__tables.keys() + elif func is None: + return self.__tables[table].keys() + elif func in self.__tables[table].keys(): + return self.__tables[table][func] + else: + return super().get_table_func(table, func) def get_person_from_handle(self, handle): """ @@ -398,11 +516,14 @@ class FilterProxyDb(ProxyDbBase): """ return self.plist - def iter_people(self): + def iter_people(self, order_by=None): """ Return an iterator over objects for Persons in the database """ - return map(self.get_person_from_handle, self.plist) + if order_by: + return sort_objects(map(self.get_person_from_handle, self.plist), order_by, self) + else: + return map(self.get_person_from_handle, self.plist) def get_event_handles(self): """ @@ -418,11 +539,14 @@ class FilterProxyDb(ProxyDbBase): """ return self.elist - def iter_events(self): + def iter_events(self, order_by=None): """ Return an iterator over objects for Events in the database """ - return map(self.get_event_from_handle, self.elist) + if order_by: + return sort_objects(map(self.get_event_from_handle, self.elist), order_by, self) + else: + return map(self.get_event_from_handle, self.elist) def get_family_handles(self): """ @@ -438,11 +562,14 @@ class FilterProxyDb(ProxyDbBase): """ return self.flist - def iter_families(self): + def iter_families(self, order_by=None): """ Return an iterator over objects for Families in the database """ - return map(self.get_family_from_handle, self.flist) + if order_by: + return sort_objects(map(self.get_family_from_handle, self.flist), order_by, self) + else: + return map(self.get_family_from_handle, self.flist) def get_note_handles(self): """ @@ -458,11 +585,14 @@ class FilterProxyDb(ProxyDbBase): """ return self.nlist - def iter_notes(self): + def iter_notes(self, order_by=None): """ Return an iterator over objects for Notes in the database """ - return map(self.get_note_from_handle, self.nlist) + if order_by: + return sort_objects(map(self.get_note_from_handle, self.nlist), order_by, self) + else: + return map(self.get_note_from_handle, self.nlist) def get_default_person(self): """returns the default Person of the database""" diff --git a/gramps/gen/proxy/living.py b/gramps/gen/proxy/living.py index 88481240a..7617400cb 100644 --- a/gramps/gen/proxy/living.py +++ b/gramps/gen/proxy/living.py @@ -32,6 +32,7 @@ from ..lib import (Date, Person, Name, Surname, NameOriginType, Family, Source, Citation, Event, Media, Place, Repository, Note, Tag) from ..utils.alive import probably_alive from ..config import config +from gramps.gen.db.base import sort_objects #------------------------------------------------------------------------- # @@ -77,6 +78,121 @@ class LivingProxyDb(ProxyDbBase): else: self.current_date = None self.years_after_death = years_after_death + self.__tables = { + 'Person': + { + "handle_func": self.get_person_from_handle, + "gramps_id_func": self.get_person_from_gramps_id, + "class_func": Person, + "cursor_func": self.get_person_cursor, + "handles_func": self.get_person_handles, + "iter_func": self.iter_people, + "count_func": self.get_number_of_people, + }, + 'Family': + { + "handle_func": self.get_family_from_handle, + "gramps_id_func": self.get_family_from_gramps_id, + "class_func": Family, + "cursor_func": self.get_family_cursor, + "handles_func": self.get_family_handles, + "iter_func": self.iter_families, + "count_func": self.get_number_of_families, + }, + 'Source': + { + "handle_func": self.get_source_from_handle, + "gramps_id_func": self.get_source_from_gramps_id, + "class_func": Source, + "cursor_func": self.get_source_cursor, + "handles_func": self.get_source_handles, + "iter_func": self.iter_sources, + "count_func": self.get_number_of_sources, + }, + 'Citation': + { + "handle_func": self.get_citation_from_handle, + "gramps_id_func": self.get_citation_from_gramps_id, + "class_func": Citation, + "cursor_func": self.get_citation_cursor, + "handles_func": self.get_citation_handles, + "iter_func": self.iter_citations, + "count_func": self.get_number_of_citations, + }, + 'Event': + { + "handle_func": self.get_event_from_handle, + "gramps_id_func": self.get_event_from_gramps_id, + "class_func": Event, + "cursor_func": self.get_event_cursor, + "handles_func": self.get_event_handles, + "iter_func": self.iter_events, + "count_func": self.get_number_of_events, + }, + 'Media': + { + "handle_func": self.get_media_from_handle, + "gramps_id_func": self.get_media_from_gramps_id, + "class_func": Media, + "cursor_func": self.get_media_cursor, + "handles_func": self.get_media_handles, + "iter_func": self.iter_media, + "count_func": self.get_number_of_media, + }, + 'Place': + { + "handle_func": self.get_place_from_handle, + "gramps_id_func": self.get_place_from_gramps_id, + "class_func": Place, + "cursor_func": self.get_place_cursor, + "handles_func": self.get_place_handles, + "iter_func": self.iter_places, + "count_func": self.get_number_of_places, + }, + 'Repository': + { + "handle_func": self.get_repository_from_handle, + "gramps_id_func": self.get_repository_from_gramps_id, + "class_func": Repository, + "cursor_func": self.get_repository_cursor, + "handles_func": self.get_repository_handles, + "iter_func": self.iter_repositories, + "count_func": self.get_number_of_repositories, + }, + 'Note': + { + "handle_func": self.get_note_from_handle, + "gramps_id_func": self.get_note_from_gramps_id, + "class_func": Note, + "cursor_func": self.get_note_cursor, + "handles_func": self.get_note_handles, + "iter_func": self.iter_notes, + "count_func": self.get_number_of_notes, + }, + 'Tag': + { + "handle_func": self.get_tag_from_handle, + "gramps_id_func": None, + "class_func": Tag, + "cursor_func": self.get_tag_cursor, + "handles_func": self.get_tag_handles, + "iter_func": self.iter_tags, + "count_func": self.get_number_of_tags, + } + } + + def get_table_func(self, table=None, func=None): + """ + Private implementation of get_table_func. + """ + if table is None: + return self.__tables.keys() + elif func is None: + return self.__tables[table].keys() + elif func in self.__tables[table].keys(): + return self.__tables[table][func] + else: + return super().get_table_func(table, func) def get_person_from_handle(self, handle): """ @@ -100,18 +216,32 @@ class LivingProxyDb(ProxyDbBase): family = self.__remove_living_from_family(family) return family - def iter_people(self): + def iter_people(self, order_by=None): """ Protected version of iter_people """ - for person in filter(None, self.db.iter_people()): - if self.__is_living(person): - if self.mode == self.MODE_EXCLUDE_ALL: - continue + if order_by: + retval = [] + for person in filter(None, self.db.iter_people()): + if self.__is_living(person): + if self.mode == self.MODE_EXCLUDE_ALL: + continue + else: + retval.append(self.__restrict_person(person)) else: - yield self.__restrict_person(person) - else: - yield person + retval.append(person) + retval = sort_objects(retval, order_by, self) + for item in retval: + yield item + else: + for person in filter(None, self.db.iter_people()): + if self.__is_living(person): + if self.mode == self.MODE_EXCLUDE_ALL: + continue + else: + yield self.__restrict_person(person) + else: + yield person def get_person_from_gramps_id(self, val): """ diff --git a/gramps/gen/proxy/private.py b/gramps/gen/proxy/private.py index bb7e90556..76e6da228 100644 --- a/gramps/gen/proxy/private.py +++ b/gramps/gen/proxy/private.py @@ -56,6 +56,121 @@ class PrivateProxyDb(ProxyDbBase): Create a new PrivateProxyDb instance. """ ProxyDbBase.__init__(self, db) + self.__tables = { + 'Person': + { + "handle_func": self.get_person_from_handle, + "gramps_id_func": self.get_person_from_gramps_id, + "class_func": Person, + "cursor_func": self.get_person_cursor, + "handles_func": self.get_person_handles, + "iter_func": self.iter_people, + "count_func": self.get_number_of_people, + }, + 'Family': + { + "handle_func": self.get_family_from_handle, + "gramps_id_func": self.get_family_from_gramps_id, + "class_func": Family, + "cursor_func": self.get_family_cursor, + "handles_func": self.get_family_handles, + "iter_func": self.iter_families, + "count_func": self.get_number_of_families, + }, + 'Source': + { + "handle_func": self.get_source_from_handle, + "gramps_id_func": self.get_source_from_gramps_id, + "class_func": Source, + "cursor_func": self.get_source_cursor, + "handles_func": self.get_source_handles, + "iter_func": self.iter_sources, + "count_func": self.get_number_of_sources, + }, + 'Citation': + { + "handle_func": self.get_citation_from_handle, + "gramps_id_func": self.get_citation_from_gramps_id, + "class_func": Citation, + "cursor_func": self.get_citation_cursor, + "handles_func": self.get_citation_handles, + "iter_func": self.iter_citations, + "count_func": self.get_number_of_citations, + }, + 'Event': + { + "handle_func": self.get_event_from_handle, + "gramps_id_func": self.get_event_from_gramps_id, + "class_func": Event, + "cursor_func": self.get_event_cursor, + "handles_func": self.get_event_handles, + "iter_func": self.iter_events, + "count_func": self.get_number_of_events, + }, + 'Media': + { + "handle_func": self.get_media_from_handle, + "gramps_id_func": self.get_media_from_gramps_id, + "class_func": Media, + "cursor_func": self.get_media_cursor, + "handles_func": self.get_media_handles, + "iter_func": self.iter_media, + "count_func": self.get_number_of_media, + }, + 'Place': + { + "handle_func": self.get_place_from_handle, + "gramps_id_func": self.get_place_from_gramps_id, + "class_func": Place, + "cursor_func": self.get_place_cursor, + "handles_func": self.get_place_handles, + "iter_func": self.iter_places, + "count_func": self.get_number_of_places, + }, + 'Repository': + { + "handle_func": self.get_repository_from_handle, + "gramps_id_func": self.get_repository_from_gramps_id, + "class_func": Repository, + "cursor_func": self.get_repository_cursor, + "handles_func": self.get_repository_handles, + "iter_func": self.iter_repositories, + "count_func": self.get_number_of_repositories, + }, + 'Note': + { + "handle_func": self.get_note_from_handle, + "gramps_id_func": self.get_note_from_gramps_id, + "class_func": Note, + "cursor_func": self.get_note_cursor, + "handles_func": self.get_note_handles, + "iter_func": self.iter_notes, + "count_func": self.get_number_of_notes, + }, + 'Tag': + { + "handle_func": self.get_tag_from_handle, + "gramps_id_func": None, + "class_func": Tag, + "cursor_func": self.get_tag_cursor, + "handles_func": self.get_tag_handles, + "iter_func": self.iter_tags, + "count_func": self.get_number_of_tags, + } + } + + def get_table_func(self, table=None, func=None): + """ + Private implementation of get_table_func. + """ + if table is None: + return self.__tables.keys() + elif func is None: + return self.__tables[table].keys() + elif func in self.__tables[table].keys(): + return self.__tables[table][func] + else: + return super().get_table_func(table, func) def get_person_from_handle(self, handle): """ @@ -285,7 +400,7 @@ class PrivateProxyDb(ProxyDbBase): """ Predicate returning True if object is to be included, else False """ - obj = self.get_unfiltered_object(handle) + obj = self.get_unfiltered_media(handle) return obj and not obj.get_privacy() def include_repository(self, handle): diff --git a/gramps/gen/proxy/proxybase.py b/gramps/gen/proxy/proxybase.py index 37ca063c8..ab588fd6b 100644 --- a/gramps/gen/proxy/proxybase.py +++ b/gramps/gen/proxy/proxybase.py @@ -35,7 +35,11 @@ import types # Gramps libraries # #------------------------------------------------------------------------- -from ..db.base import DbReadBase, DbWriteBase +from ..db.base import DbReadBase, DbWriteBase, sort_objects +from ..lib import (MediaRef, Attribute, Address, EventRef, + Person, Name, Source, RepoRef, Media, Place, Event, + Family, ChildRef, Repository, LdsOrd, Surname, Citation, + SrcAttribute, Note, Tag) class ProxyCursor(object): """ @@ -120,6 +124,122 @@ class ProxyDbBase(DbReadBase): self.note_map = ProxyMap(self, self.get_raw_note_data, self.get_note_handles) + self.__tables = { + 'Person': + { + "handle_func": self.get_person_from_handle, + "gramps_id_func": self.get_person_from_gramps_id, + "class_func": Person, + "cursor_func": self.get_person_cursor, + "handles_func": self.get_person_handles, + "iter_func": self.iter_people, + "count_func": self.get_number_of_people, + }, + 'Family': + { + "handle_func": self.get_family_from_handle, + "gramps_id_func": self.get_family_from_gramps_id, + "class_func": Family, + "cursor_func": self.get_family_cursor, + "handles_func": self.get_family_handles, + "iter_func": self.iter_families, + "count_func": self.get_number_of_families, + }, + 'Source': + { + "handle_func": self.get_source_from_handle, + "gramps_id_func": self.get_source_from_gramps_id, + "class_func": Source, + "cursor_func": self.get_source_cursor, + "handles_func": self.get_source_handles, + "iter_func": self.iter_sources, + "count_func": self.get_number_of_sources, + }, + 'Citation': + { + "handle_func": self.get_citation_from_handle, + "gramps_id_func": self.get_citation_from_gramps_id, + "class_func": Citation, + "cursor_func": self.get_citation_cursor, + "handles_func": self.get_citation_handles, + "iter_func": self.iter_citations, + "count_func": self.get_number_of_citations, + }, + 'Event': + { + "handle_func": self.get_event_from_handle, + "gramps_id_func": self.get_event_from_gramps_id, + "class_func": Event, + "cursor_func": self.get_event_cursor, + "handles_func": self.get_event_handles, + "iter_func": self.iter_events, + "count_func": self.get_number_of_events, + }, + 'Media': + { + "handle_func": self.get_media_from_handle, + "gramps_id_func": self.get_media_from_gramps_id, + "class_func": Media, + "cursor_func": self.get_media_cursor, + "handles_func": self.get_media_handles, + "iter_func": self.iter_media, + "count_func": self.get_number_of_media, + }, + 'Place': + { + "handle_func": self.get_place_from_handle, + "gramps_id_func": self.get_place_from_gramps_id, + "class_func": Place, + "cursor_func": self.get_place_cursor, + "handles_func": self.get_place_handles, + "iter_func": self.iter_places, + "count_func": self.get_number_of_places, + }, + 'Repository': + { + "handle_func": self.get_repository_from_handle, + "gramps_id_func": self.get_repository_from_gramps_id, + "class_func": Repository, + "cursor_func": self.get_repository_cursor, + "handles_func": self.get_repository_handles, + "iter_func": self.iter_repositories, + "count_func": self.get_number_of_repositories, + }, + 'Note': + { + "handle_func": self.get_note_from_handle, + "gramps_id_func": self.get_note_from_gramps_id, + "class_func": Note, + "cursor_func": self.get_note_cursor, + "handles_func": self.get_note_handles, + "iter_func": self.iter_notes, + "count_func": self.get_number_of_notes, + }, + 'Tag': + { + "handle_func": self.get_tag_from_handle, + "gramps_id_func": None, + "class_func": Tag, + "cursor_func": self.get_tag_cursor, + "handles_func": self.get_tag_handles, + "iter_func": self.iter_tags, + "count_func": self.get_number_of_tags, + } + } + + def get_table_func(self, table=None, func=None): + """ + Private implementation of get_table_func. + """ + if table is None: + return self.__tables.keys() + elif func is None: + return self.__tables[table].keys() + elif func in self.__tables[table].keys(): + return self.__tables[table][func] + else: + return super().get_table_func(table, func) + def is_open(self): """ Return 1 if the database has been opened. @@ -374,73 +494,76 @@ class ProxyDbBase(DbReadBase): """ return filter(self.include_tag, self.db.iter_tag_handles()) - @staticmethod - def __iter_object(selector, method): + def __iter_object(self, selector, method, order_by=None): """ Helper function to return an iterator over an object class """ - return filter(lambda obj: ((selector is None) or selector(obj.handle)), - method()) + retval = filter(lambda obj: ((selector is None) or selector(obj.handle)), + method()) + if order_by: + return sort_objects([item for item in retval], order_by, self) + else: + return retval - def iter_people(self): + def iter_people(self, order_by=None): """ Return an iterator over Person objects in the database """ - return self.__iter_object(self.include_person, self.db.iter_people) + return self.__iter_object(self.include_person, self.db.iter_people, order_by) - def iter_families(self): + def iter_families(self, order_by=None): """ Return an iterator over Family objects in the database """ - return self.__iter_object(self.include_family, self.db.iter_families) + return self.__iter_object(self.include_family, self.db.iter_families, order_by) - def iter_events(self): + def iter_events(self, order_by=None): """ Return an iterator over Event objects in the database """ - return self.__iter_object(self.include_event, self.db.iter_events) + return self.__iter_object(self.include_event, self.db.iter_events, order_by) - def iter_places(self): + def iter_places(self, order_by=None): """ Return an iterator over Place objects in the database """ - return self.__iter_object(self.include_place, self.db.iter_places) + return self.__iter_object(self.include_place, self.db.iter_places, order_by) - def iter_sources(self): + def iter_sources(self, order_by=None): """ Return an iterator over Source objects in the database """ - return self.__iter_object(self.include_source, self.db.iter_sources) + return self.__iter_object(self.include_source, self.db.iter_sources, order_by) - def iter_citations(self): + def iter_citations(self, order_by=None): """ Return an iterator over Citation objects in the database """ - return self.__iter_object(self.include_citation, self.db.iter_citations) + return self.__iter_object(self.include_citation, self.db.iter_citations, order_by) - def iter_media(self): + def iter_media(self, order_by=None): """ Return an iterator over Media objects in the database """ return self.__iter_object(self.include_media, - self.db.iter_media) + self.db.iter_media, order_by) - def iter_repositories(self): + def iter_repositories(self, order_by=None): """ Return an iterator over Repositories objects in the database """ return self.__iter_object(self.include_repository, - self.db.iter_repositories) + self.db.iter_repositories, order_by) - def iter_notes(self): + def iter_notes(self, order_by=None): """ Return an iterator over Note objects in the database """ - return self.__iter_object(self.include_note, self.db.iter_notes) + return self.__iter_object(self.include_note, self.db.iter_notes, order_by) - def iter_tags(self): + def iter_tags(self, order_by=None): """ Return an iterator over Tag objects in the database """ - return self.__iter_object(self.include_tag, self.db.iter_tags) + return self.__iter_object(self.include_tag, self.db.iter_tags, order_by) @staticmethod def gfilter(predicate, obj): @@ -468,9 +591,12 @@ class ProxyDbBase(DbReadBase): return attr # if a write-method: - if (name in DbWriteBase.__dict__ and - not name.startswith("__") and - type(DbWriteBase.__dict__[name]) is types.FunctionType): + if ((name in DbWriteBase.__dict__ and + not name.startswith("__") and + type(DbWriteBase.__dict__[name]) is types.FunctionType) or + (name in DbWriteBase.__dict__ and + not name.startswith("__") and + type(DbWriteBase.__dict__[name]) is types.FunctionType)): raise AttributeError # Default behaviour: lookup attribute in parent object return getattr(self.db, name) diff --git a/gramps/gen/proxy/referencedbyselection.py b/gramps/gen/proxy/referencedbyselection.py index 8e522cb75..d488ec778 100644 --- a/gramps/gen/proxy/referencedbyselection.py +++ b/gramps/gen/proxy/referencedbyselection.py @@ -71,7 +71,7 @@ class ReferencedBySelectionProxyDb(ProxyDbBase): # get rid of orphaned people: # first, get all of the links from people: for person in self.db.iter_people(): - self.queue_object("Person", person, False) + self.queue_object("Person", person.handle, False) # save those people: self.restricted_to["Person"] = self.referenced["Person"] # reset, and just follow those people @@ -83,6 +83,152 @@ class ReferencedBySelectionProxyDb(ProxyDbBase): obj_type, handle, reference = self.queue.pop() self.process_object(obj_type, handle, reference) + self.__tables = { + 'Person': + { + "handle_func": self.get_person_from_handle, + "gramps_id_func": self.get_person_from_gramps_id, + "class_func": Person, + "cursor_func": self.get_person_cursor, + "handles_func": self.get_person_handles, + "add_func": self.add_person, + "commit_func": self.commit_person, + "iter_func": self.iter_people, + "count_func": self.get_number_of_people, + "del_func": self.remove_person, + }, + 'Family': + { + "handle_func": self.get_family_from_handle, + "gramps_id_func": self.get_family_from_gramps_id, + "class_func": Family, + "cursor_func": self.get_family_cursor, + "handles_func": self.get_family_handles, + "add_func": self.add_family, + "commit_func": self.commit_family, + "iter_func": self.iter_families, + "count_func": self.get_number_of_families, + "del_func": self.remove_family, + }, + 'Source': + { + "handle_func": self.get_source_from_handle, + "gramps_id_func": self.get_source_from_gramps_id, + "class_func": Source, + "cursor_func": self.get_source_cursor, + "handles_func": self.get_source_handles, + "add_func": self.add_source, + "commit_func": self.commit_source, + "iter_func": self.iter_sources, + "count_func": self.get_number_of_sources, + "del_func": self.remove_source, + }, + 'Citation': + { + "handle_func": self.get_citation_from_handle, + "gramps_id_func": self.get_citation_from_gramps_id, + "class_func": Citation, + "cursor_func": self.get_citation_cursor, + "handles_func": self.get_citation_handles, + "add_func": self.add_citation, + "commit_func": self.commit_citation, + "iter_func": self.iter_citations, + "count_func": self.get_number_of_citations, + "del_func": self.remove_citation, + }, + 'Event': + { + "handle_func": self.get_event_from_handle, + "gramps_id_func": self.get_event_from_gramps_id, + "class_func": Event, + "cursor_func": self.get_event_cursor, + "handles_func": self.get_event_handles, + "add_func": self.add_event, + "commit_func": self.commit_event, + "iter_func": self.iter_events, + "count_func": self.get_number_of_events, + "del_func": self.remove_event, + }, + 'Media': + { + "handle_func": self.get_media_from_handle, + "gramps_id_func": self.get_media_from_gramps_id, + "class_func": Media, + "cursor_func": self.get_media_cursor, + "handles_func": self.get_media_handles, + "add_func": self.add_media, + "commit_func": self.commit_media, + "iter_func": self.iter_media, + "count_func": self.get_number_of_media, + "del_func": self.remove_media, + }, + 'Place': + { + "handle_func": self.get_place_from_handle, + "gramps_id_func": self.get_place_from_gramps_id, + "class_func": Place, + "cursor_func": self.get_place_cursor, + "handles_func": self.get_place_handles, + "add_func": self.add_place, + "commit_func": self.commit_place, + "iter_func": self.iter_places, + "count_func": self.get_number_of_places, + "del_func": self.remove_place, + }, + 'Repository': + { + "handle_func": self.get_repository_from_handle, + "gramps_id_func": self.get_repository_from_gramps_id, + "class_func": Repository, + "cursor_func": self.get_repository_cursor, + "handles_func": self.get_repository_handles, + "add_func": self.add_repository, + "commit_func": self.commit_repository, + "iter_func": self.iter_repositories, + "count_func": self.get_number_of_repositories, + "del_func": self.remove_repository, + }, + 'Note': + { + "handle_func": self.get_note_from_handle, + "gramps_id_func": self.get_note_from_gramps_id, + "class_func": Note, + "cursor_func": self.get_note_cursor, + "handles_func": self.get_note_handles, + "add_func": self.add_note, + "commit_func": self.commit_note, + "iter_func": self.iter_notes, + "count_func": self.get_number_of_notes, + "del_func": self.remove_note, + }, + 'Tag': + { + "handle_func": self.get_tag_from_handle, + "gramps_id_func": None, + "class_func": Tag, + "cursor_func": self.get_tag_cursor, + "handles_func": self.get_tag_handles, + "add_func": self.add_tag, + "commit_func": self.commit_tag, + "iter_func": self.iter_tags, + "count_func": self.get_number_of_tags, + "del_func": self.remove_tag, + } + } + + def get_table_func(self, table=None, func=None): + """ + Private implementation of get_table_func. + """ + if table is None: + return self.__tables.keys() + elif func is None: + return self.__tables[table].keys() + elif func in self.__tables[table].keys(): + return self.__tables[table][func] + else: + return super().get_table_func(table, func) + def queue_object(self, obj_type, handle, reference=True): self.queue.append((obj_type, handle, reference)) diff --git a/gramps/plugins/database/bsddb_support/read.py b/gramps/plugins/database/bsddb_support/read.py index 3d7b049a4..00a4ad90e 100644 --- a/gramps/plugins/database/bsddb_support/read.py +++ b/gramps/plugins/database/bsddb_support/read.py @@ -70,7 +70,7 @@ from gramps.gen.lib.nameorigintype import NameOriginType from gramps.gen.utils.callback import Callback from . import BsddbBaseCursor -from gramps.gen.db.base import DbReadBase +from gramps.gen.db.base import DbReadBase, eval_order_by from gramps.gen.utils.id import create_id from gramps.gen.errors import DbError, HandleError from gramps.gen.constfunc import get_env_var @@ -290,96 +290,6 @@ class DbBsddbRead(DbReadBase, Callback): """ DbReadBase.__init__(self) Callback.__init__(self) - self._tables['Person'].update( - { - "handle_func": self.get_person_from_handle, - "gramps_id_func": self.get_person_from_gramps_id, - "class_func": Person, - "cursor_func": self.get_person_cursor, - "handles_func": self.get_person_handles, - "iter_func": self.iter_people, - }) - self._tables['Family'].update( - { - "handle_func": self.get_family_from_handle, - "gramps_id_func": self.get_family_from_gramps_id, - "class_func": Family, - "cursor_func": self.get_family_cursor, - "handles_func": self.get_family_handles, - "iter_func": self.iter_families, - }) - self._tables['Source'].update( - { - "handle_func": self.get_source_from_handle, - "gramps_id_func": self.get_source_from_gramps_id, - "class_func": Source, - "cursor_func": self.get_source_cursor, - "handles_func": self.get_source_handles, - "iter_func": self.iter_sources, - }) - self._tables['Citation'].update( - { - "handle_func": self.get_citation_from_handle, - "gramps_id_func": self.get_citation_from_gramps_id, - "class_func": Citation, - "cursor_func": self.get_citation_cursor, - "handles_func": self.get_citation_handles, - "iter_func": self.iter_citations, - }) - self._tables['Event'].update( - { - "handle_func": self.get_event_from_handle, - "gramps_id_func": self.get_event_from_gramps_id, - "class_func": Event, - "cursor_func": self.get_event_cursor, - "handles_func": self.get_event_handles, - "iter_func": self.iter_events, - }) - self._tables['Media'].update( - { - "handle_func": self.get_media_from_handle, - "gramps_id_func": self.get_media_from_gramps_id, - "class_func": Media, - "cursor_func": self.get_media_cursor, - "handles_func": self.get_media_handles, - "iter_func": self.iter_media, - }) - self._tables['Place'].update( - { - "handle_func": self.get_place_from_handle, - "gramps_id_func": self.get_place_from_gramps_id, - "class_func": Place, - "cursor_func": self.get_place_cursor, - "handles_func": self.get_place_handles, - "iter_func": self.iter_places, - }) - self._tables['Repository'].update( - { - "handle_func": self.get_repository_from_handle, - "gramps_id_func": self.get_repository_from_gramps_id, - "class_func": Repository, - "cursor_func": self.get_repository_cursor, - "handles_func": self.get_repository_handles, - "iter_func": self.iter_repositories, - }) - self._tables['Note'].update( - { - "handle_func": self.get_note_from_handle, - "gramps_id_func": self.get_note_from_gramps_id, - "class_func": Note, - "cursor_func": self.get_note_cursor, - "handles_func": self.get_note_handles, - "iter_func": self.iter_notes, - }) - self._tables['Tag'].update( - { - "handle_func": self.get_tag_from_handle, - "gramps_id_func": None, - "class_func": Tag, - "cursor_func": self.get_tag_cursor, - "handles_func": self.get_tag_handles, - "iter_func": self.iter_tags, - }) self.set_person_id_prefix('I%04d') self.set_media_id_prefix('O%04d') @@ -474,6 +384,112 @@ class DbBsddbRead(DbReadBase, Callback): self.txn = None self.has_changed = False + self.__tables = { + 'Person': + { + "handle_func": self.get_person_from_handle, + "gramps_id_func": self.get_person_from_gramps_id, + "class_func": Person, + "cursor_func": self.get_person_cursor, + "handles_func": self.get_person_handles, + "iter_func": self.iter_people, + }, + 'Family': + { + "handle_func": self.get_family_from_handle, + "gramps_id_func": self.get_family_from_gramps_id, + "class_func": Family, + "cursor_func": self.get_family_cursor, + "handles_func": self.get_family_handles, + "iter_func": self.iter_families, + }, + 'Source': + { + "handle_func": self.get_source_from_handle, + "gramps_id_func": self.get_source_from_gramps_id, + "class_func": Source, + "cursor_func": self.get_source_cursor, + "handles_func": self.get_source_handles, + "iter_func": self.iter_sources, + }, + 'Citation': + { + "handle_func": self.get_citation_from_handle, + "gramps_id_func": self.get_citation_from_gramps_id, + "class_func": Citation, + "cursor_func": self.get_citation_cursor, + "handles_func": self.get_citation_handles, + "iter_func": self.iter_citations, + }, + 'Event': + { + "handle_func": self.get_event_from_handle, + "gramps_id_func": self.get_event_from_gramps_id, + "class_func": Event, + "cursor_func": self.get_event_cursor, + "handles_func": self.get_event_handles, + "iter_func": self.iter_events, + }, + 'Media': + { + "handle_func": self.get_media_from_handle, + "gramps_id_func": self.get_media_from_gramps_id, + "class_func": Media, + "cursor_func": self.get_media_cursor, + "handles_func": self.get_media_handles, + "iter_func": self.iter_media, + }, + 'Place': + { + "handle_func": self.get_place_from_handle, + "gramps_id_func": self.get_place_from_gramps_id, + "class_func": Place, + "cursor_func": self.get_place_cursor, + "handles_func": self.get_place_handles, + "iter_func": self.iter_places, + }, + 'Repository': + { + "handle_func": self.get_repository_from_handle, + "gramps_id_func": self.get_repository_from_gramps_id, + "class_func": Repository, + "cursor_func": self.get_repository_cursor, + "handles_func": self.get_repository_handles, + "iter_func": self.iter_repositories, + }, + 'Note': + { + "handle_func": self.get_note_from_handle, + "gramps_id_func": self.get_note_from_gramps_id, + "class_func": Note, + "cursor_func": self.get_note_cursor, + "handles_func": self.get_note_handles, + "iter_func": self.iter_notes, + }, + 'Tag': + { + "handle_func": self.get_tag_from_handle, + "gramps_id_func": None, + "class_func": Tag, + "cursor_func": self.get_tag_cursor, + "handles_func": self.get_tag_handles, + "iter_func": self.iter_tags, + } + } + + def get_table_func(self, table=None, func=None): + """ + Private implementation of get_table_func. + """ + if table is None: + return self.__tables.keys() + elif func is None: + return self.__tables[table].keys() + elif func in self.__tables[table].keys(): + return self.__tables[table][func] + else: + return super().get_table_func(table, func) + def set_prefixes(self, person, media, family, source, citation, place, event, repository, note): self.set_person_id_prefix(person) @@ -493,12 +509,12 @@ class DbBsddbRead(DbReadBase, Callback): def get_table_names(self): """Return a list of valid table names.""" - return list(self._tables.keys()) + return list(self.get_table_func()) def get_table_metadata(self, table_name): """Return the metadata for a valid table name.""" - if table_name in self._tables: - return self._tables[table_name] + if table_name in self.get_table_func(): + return self.get_table_func(table_name) return None def get_cursor(self, table, *args, **kwargs): @@ -552,12 +568,6 @@ class DbBsddbRead(DbReadBase, Callback): self.basedb = None #remove links to functions self.disconnect_all() - for key in self._tables: - for subkey in self._tables[key]: - self._tables[key][subkey] = None - del self._tables[key][subkey] - self._tables[key] = None - del self._tables ## self.bookmarks = None ## self.family_bookmarks = None ## self.event_bookmarks = None @@ -706,8 +716,8 @@ class DbBsddbRead(DbReadBase, Callback): >>> self.get_from_name_and_handle("Person", "a7ad62365bc652387008") >>> self.get_from_name_and_handle("Media", "c3434653675bcd736f23") """ - if table_name in self._tables: - return self._tables[table_name]["handle_func"](handle) + if table_name in self.get_table_func(): + return self.get_table_func(table_name,"handle_func")(handle) return None def get_from_name_and_gramps_id(self, table_name, gramps_id): @@ -721,8 +731,8 @@ class DbBsddbRead(DbReadBase, Callback): >>> self.get_from_name_and_gramps_id("Family", "F056") >>> self.get_from_name_and_gramps_id("Media", "M00012") """ - if table_name in self._tables: - return self._tables[table_name]["gramps_id_func"](gramps_id) + if table_name in self.get_table_func(): + return self.get_table_func(table_name,"gramps_id_func")(gramps_id) return None def get_person_from_handle(self, handle): @@ -1233,7 +1243,7 @@ class DbBsddbRead(DbReadBase, Callback): obj = obj_() obj.unserialize(data) # just use values and handle to keep small: - sorted_items.append((self.eval_order_by(order_by, obj), obj.handle)) + sorted_items.append((eval_order_by(order_by, obj, self), obj.handle)) # next we sort by fields and direction pos = len(order_by) - 1 for (field, order) in reversed(order_by): # sort the lasts parts first @@ -1241,7 +1251,7 @@ class DbBsddbRead(DbReadBase, Callback): pos -= 1 # now we will look them up again: for (order_by_values, handle) in sorted_items: - yield self._tables[obj_.__name__]["handle_func"](handle) + yield self.get_table_func(obj_.__name__,"handle_func")(handle) return g # Use closure to define iterators for each primary object type diff --git a/gramps/plugins/database/bsddb_support/write.py b/gramps/plugins/database/bsddb_support/write.py index 114b8fe18..84ce1446e 100644 --- a/gramps/plugins/database/bsddb_support/write.py +++ b/gramps/plugins/database/bsddb_support/write.py @@ -240,7 +240,8 @@ class DbBsddb(DbBsddbRead, DbWriteBase, UpdateCallback): DbBsddbRead.__init__(self) DbWriteBase.__init__(self) #UpdateCallback.__init__(self) - self._tables['Person'].update( + self.__tables = { + 'Person': { "handle_func": self.get_person_from_handle, "gramps_id_func": self.get_person_from_gramps_id, @@ -252,8 +253,8 @@ class DbBsddb(DbBsddbRead, DbWriteBase, UpdateCallback): "count_func": self.get_number_of_people, "del_func": self.remove_person, "iter_func": self.iter_people, - }) - self._tables['Family'].update( + }, + 'Family': { "handle_func": self.get_family_from_handle, "gramps_id_func": self.get_family_from_gramps_id, @@ -265,8 +266,8 @@ class DbBsddb(DbBsddbRead, DbWriteBase, UpdateCallback): "count_func": self.get_number_of_families, "del_func": self.remove_family, "iter_func": self.iter_families, - }) - self._tables['Source'].update( + }, + 'Source': { "handle_func": self.get_source_from_handle, "gramps_id_func": self.get_source_from_gramps_id, @@ -278,8 +279,8 @@ class DbBsddb(DbBsddbRead, DbWriteBase, UpdateCallback): "count_func": self.get_number_of_sources, "del_func": self.remove_source, "iter_func": self.iter_sources, - }) - self._tables['Citation'].update( + }, + 'Citation': { "handle_func": self.get_citation_from_handle, "gramps_id_func": self.get_citation_from_gramps_id, @@ -291,8 +292,8 @@ class DbBsddb(DbBsddbRead, DbWriteBase, UpdateCallback): "count_func": self.get_number_of_citations, "del_func": self.remove_citation, "iter_func": self.iter_citations, - }) - self._tables['Event'].update( + }, + 'Event': { "handle_func": self.get_event_from_handle, "gramps_id_func": self.get_event_from_gramps_id, @@ -304,8 +305,8 @@ class DbBsddb(DbBsddbRead, DbWriteBase, UpdateCallback): "count_func": self.get_number_of_events, "del_func": self.remove_event, "iter_func": self.iter_events, - }) - self._tables['Media'].update( + }, + 'Media': { "handle_func": self.get_media_from_handle, "gramps_id_func": self.get_media_from_gramps_id, @@ -317,8 +318,8 @@ class DbBsddb(DbBsddbRead, DbWriteBase, UpdateCallback): "count_func": self.get_number_of_media, "del_func": self.remove_media, "iter_func": self.iter_media, - }) - self._tables['Place'].update( + }, + 'Place': { "handle_func": self.get_place_from_handle, "gramps_id_func": self.get_place_from_gramps_id, @@ -330,8 +331,8 @@ class DbBsddb(DbBsddbRead, DbWriteBase, UpdateCallback): "count_func": self.get_number_of_places, "del_func": self.remove_place, "iter_func": self.iter_places, - }) - self._tables['Repository'].update( + }, + 'Repository': { "handle_func": self.get_repository_from_handle, "gramps_id_func": self.get_repository_from_gramps_id, @@ -343,8 +344,8 @@ class DbBsddb(DbBsddbRead, DbWriteBase, UpdateCallback): "count_func": self.get_number_of_repositories, "del_func": self.remove_repository, "iter_func": self.iter_repositories, - }) - self._tables['Note'].update( + }, + 'Note': { "handle_func": self.get_note_from_handle, "gramps_id_func": self.get_note_from_gramps_id, @@ -356,8 +357,8 @@ class DbBsddb(DbBsddbRead, DbWriteBase, UpdateCallback): "count_func": self.get_number_of_notes, "del_func": self.remove_note, "iter_func": self.iter_notes, - }) - self._tables['Tag'].update( + }, + 'Tag': { "handle_func": self.get_tag_from_handle, "gramps_id_func": None, @@ -369,7 +370,8 @@ class DbBsddb(DbBsddbRead, DbWriteBase, UpdateCallback): "count_func": self.get_number_of_tags, "del_func": self.remove_tag, "iter_func": self.iter_tags, - }) + } + } self.secondary_connected = False self.has_changed = False @@ -378,6 +380,19 @@ class DbBsddb(DbBsddbRead, DbWriteBase, UpdateCallback): self.update_python_version = False self.update_pickle_version = False + def get_table_func(self, table=None, func=None): + """ + Private implementation of get_table_func. + """ + if table is None: + return self.__tables.keys() + elif func is None: + return self.__tables[table].keys() + elif func in self.__tables[table].keys(): + return self.__tables[table][func] + else: + return super().get_table_func(table, func) + def catch_db_error(func): """ Decorator function for catching database errors. If *func* throws diff --git a/gramps/plugins/database/dbapi.py b/gramps/plugins/database/dbapi.py index b36304e29..693cde4a5 100644 --- a/gramps/plugins/database/dbapi.py +++ b/gramps/plugins/database/dbapi.py @@ -31,6 +31,7 @@ import dbapi_support import time import pickle +from operator import itemgetter import logging LOG = logging.getLogger(".dbapi") @@ -1048,10 +1049,9 @@ class DBAPI(DbGeneric): self.dbapi.execute(query) rows = self.dbapi.fetchall() for row in rows: - obj = obj_() - obj.unserialize(row[0]) + obj = self.get_table_func(class_.__name__,"class_func").create(pickle.loads(row[0])) # just use values and handle to keep small: - sorted_items.append((self.eval_order_by(order_by, obj), obj.handle)) + sorted_items.append((eval_order_by(order_by, obj, self), obj.handle)) # next we sort by fields and direction pos = len(order_by) - 1 for (field, order) in reversed(order_by): # sort the lasts parts first @@ -1059,7 +1059,7 @@ class DBAPI(DbGeneric): pos -= 1 # now we will look them up again: for (order_by_values, handle) in sorted_items: - yield self._tables[obj_.__name__]["handle_func"](handle) + yield self.get_table_func(class_.__name__,"handle_func")(handle) def iter_items(self, order_by, class_): # check if order_by fields are secondary @@ -1068,7 +1068,7 @@ class DBAPI(DbGeneric): if order_by: secondary_fields = class_.get_secondary_fields() if not self.check_order_by_fields(class_.__name__, order_by, secondary_fields): - for item in super().iter_items(order_by, class_): + for item in self.iter_items_order_by_python(order_by, class_): yield item return ## Continue with dbapi select @@ -1578,13 +1578,13 @@ class DBAPI(DbGeneric): Add secondary fields, update, and create indexes. """ LOG.info("Rebuilding secondary fields...") - for table in self._tables.keys(): - if not hasattr(self._tables[table]["class_func"], "get_secondary_fields"): + for table in self.get_table_func(): + if not hasattr(self.get_table_func(table,"class_func"), "get_secondary_fields"): continue # do a select on all; if it works, then it is ok; else, check them all try: fields = [self._hash_name(table, field) for (field, ptype) in - self._tables[table]["class_func"].get_secondary_fields()] + self.get_table_func(table,"class_func").get_secondary_fields()] if fields: self.dbapi.execute("select %s from %s limit 1;" % (", ".join(fields), table)) # if no error, continue @@ -1594,7 +1594,7 @@ class DBAPI(DbGeneric): pass # got to add missing ones, so continue LOG.info("Table %s needs rebuilding..." % table) altered = False - for field_pair in self._tables[table]["class_func"].get_secondary_fields(): + for field_pair in self.get_table_func(table,"class_func").get_secondary_fields(): field, python_type = field_pair field = self._hash_name(table, field) sql_type = self._sql_type(python_type) @@ -1617,8 +1617,8 @@ class DBAPI(DbGeneric): """ Create the indexes for the secondary fields. """ - for table in self._tables.keys(): - if not hasattr(self._tables[table]["class_func"], "get_index_fields"): + for table in self.get_table_func(): + if not hasattr(self.get_table_func(table,"class_func"), "get_index_fields"): continue self.create_secondary_indexes_table(table) @@ -1626,7 +1626,7 @@ class DBAPI(DbGeneric): """ Create secondary indexes for just this table. """ - for fields in self._tables[table]["class_func"].get_index_fields(): + for fields in self.get_table_func(table,"class_func").get_index_fields(): for field in fields: field = self._hash_name(table, field) self.dbapi.try_execute("CREATE INDEX %s_%s ON %s(%s);" % (table, field, table, field)) @@ -1636,7 +1636,7 @@ class DBAPI(DbGeneric): Go through all items in all tables, and update their secondary field values. """ - for table in self._tables.keys(): + for table in self.get_table_func(): self.update_secondary_values_table(table) def update_secondary_values_table(self, table): @@ -1646,9 +1646,9 @@ class DBAPI(DbGeneric): table - "Person", "Place", "Media", etc. Commits changes. """ - if not hasattr(self._tables[table]["class_func"], "get_secondary_fields"): + if not hasattr(self.get_table_func(table,"class_func"), "get_secondary_fields"): return - for item in self._tables[table]["iter_func"](): + for item in self.get_table_func(table,"iter_func")(): self.update_secondary_values(item) self.dbapi.commit() @@ -1658,7 +1658,7 @@ class DBAPI(DbGeneric): Does not commit. """ table = item.__class__.__name__ - fields = self._tables[table]["class_func"].get_secondary_fields() + fields = self.get_table_func(table,"class_func").get_secondary_fields() fields = [field for (field, direction) in fields] sets = [] values = [] @@ -1671,7 +1671,7 @@ class DBAPI(DbGeneric): self.dbapi.execute("UPDATE %s SET %s where handle = ?;" % (table, ", ".join(sets)), values + [item.handle]) - def sql_repr(self, value): + def _sql_repr(self, value): """ Given a Python value, turn it into a SQL value. """ @@ -1684,7 +1684,7 @@ class DBAPI(DbGeneric): else: return repr(value) - def build_where_clause_recursive(self, table, where): + def _build_where_clause_recursive(self, table, where): """ where - (field, op, value) - ["NOT", where] @@ -1695,26 +1695,26 @@ class DBAPI(DbGeneric): return "" elif len(where) == 3: field, op, value = where - return "(%s %s %s)" % (self._hash_name(table, field), op, self.sql_repr(value)) + return "(%s %s %s)" % (self._hash_name(table, field), op, self._sql_repr(value)) elif where[0] in ["AND", "OR"]: - parts = [self.build_where_clause_recursive(table, part) + parts = [self._build_where_clause_recursive(table, part) for part in where[1]] return "(%s)" % ((" %s " % where[0]).join(parts)) else: - return "(NOT %s)" % self.build_where_clause_recursive(table, where[1]) + return "(NOT %s)" % self._build_where_clause_recursive(table, where[1]) - def build_where_clause(self, table, where): + def _build_where_clause(self, table, where): """ where - a list in where format return - "WHERE conditions..." """ - parts = self.build_where_clause_recursive(table, where) + parts = self._build_where_clause_recursive(table, where) if parts: return ("WHERE " + parts) else: return "" - def build_order_clause(self, table, order_by): + def _build_order_clause(self, table, order_by): """ order_by - [(field, "ASC" | "DESC"), ...] """ @@ -1725,7 +1725,7 @@ class DBAPI(DbGeneric): else: return "" - def build_select_fields(self, table, select_fields, secondary_fields): + def _build_select_fields(self, table, select_fields, secondary_fields): """ fields - [field, ...] return: "field, field, field" @@ -1736,7 +1736,7 @@ class DBAPI(DbGeneric): else: return ["blob_data"] # nope, we'll have to expand blob to get all fields - def check_order_by_fields(self, table, order_by, secondary_fields): + def _check_order_by_fields(self, table, order_by, secondary_fields): """ Check to make sure all order_by fields are defined. If not, then we need to do the Python-based order. @@ -1749,7 +1749,7 @@ class DBAPI(DbGeneric): return False return True - def check_where_fields(self, table, where, secondary_fields): + def _check_where_fields(self, table, where, secondary_fields): """ Check to make sure all where fields are defined. If not, then we need to do the Python-based select. @@ -1762,19 +1762,19 @@ class DBAPI(DbGeneric): connector, exprs = where if connector in ["AND", "OR"]: for expr in exprs: - value = self.check_where_fields(table, expr, secondary_fields) + value = self._check_where_fields(table, expr, secondary_fields) if value == False: return False return True else: # "NOT" - return self.check_where_fields(table, exprs, secondary_fields) + return self._check_where_fields(table, exprs, secondary_fields) elif len(where) == 3: # (name, op, value) (name, op, value) = where # just the ones we need for where return (self._hash_name(table, name) in secondary_fields) - def select(self, table, fields=None, start=0, limit=-1, - where=None, order_by=None): + def _select(self, table, fields=None, start=0, limit=-1, + where=None, order_by=None): """ Default implementation of a select for those databases that don't support SQL. Returns a list of dicts, total, @@ -1790,65 +1790,65 @@ class DBAPI(DbGeneric): ["NOT", where] order_by - [[fieldname, "ASC" | "DESC"], ...] """ - hashed_fields = [self._hash_name(table, field) for field in fields] secondary_fields = ([self._hash_name(table, field) for (field, ptype) in - self._tables[table]["class_func"].get_secondary_fields()] + + self.get_table_func(table,"class_func").get_secondary_fields()] + ["handle"]) # handle is a sql field, but not listed in secondaries - if ((not self.check_where_fields(table, where, secondary_fields)) or - (not self.check_order_by_fields(table, order_by, secondary_fields))): - return super().select(table, fields, start, limit, where, order_by) - fields = hashed_fields - start_time = time.time() - where_clause = self.build_where_clause(table, where) - order_clause = self.build_order_clause(table, order_by) - select_fields = self.build_select_fields(table, fields, secondary_fields) - # Get the total count: - if limit != -1 or start != 0: # get the total that would match: - self.dbapi.execute("select count(1) from %s %s;" % (table, where_clause)) - total = self.dbapi.fetchone()[0] + # If no fields, then we need objects: + # Check to see if where matches SQL fields: + if ((not self._check_where_fields(table, where, secondary_fields)) or + (not self._check_order_by_fields(table, order_by, secondary_fields))): + # If not, then need to do select via Python: + generator = super()._select(table, fields, start, limit, where, order_by) + for item in generator: + yield item + return + # Otherwise, we are SQL + if fields is None: + fields = ["blob_data"] + get_count_only = False + if fields[0] == "count(1)": + hashed_fields = ["count(1)"] + fields = ["count(1)"] + select_fields = ["count(1)"] + get_count_only = True else: - total = None # need to get later - class Result(list): - """ - A list rows of just matching for this page, with total = all, - time = time to select, query = the SQL query, and expanded - if unpickled. - """ - total = 0 - time = 0.0 - query = None - expanded = False - result = Result() + hashed_fields = [self._hash_name(table, field) for field in fields] + fields = hashed_fields + select_fields = self._build_select_fields(table, fields, secondary_fields) + where_clause = self._build_where_clause(table, where) + order_clause = self._build_order_clause(table, order_by) + if get_count_only: + select_fields = ["1"] if start: - query = "SELECT %s FROM %s %s %s LIMIT %s, %s;" % ( + query = "SELECT %s FROM %s %s %s LIMIT %s, %s" % ( ", ".join(select_fields), table, where_clause, order_clause, start, limit ) else: - query = "SELECT %s FROM %s %s %s LIMIT %s;" % ( + query = "SELECT %s FROM %s %s %s LIMIT %s" % ( ", ".join(select_fields), table, where_clause, order_clause, limit ) + if get_count_only: + self.dbapi.execute("SELECT count(1) from (%s);" % query) + rows = self.dbapi.fetchall() + yield rows[0][0] + return self.dbapi.execute(query) rows = self.dbapi.fetchall() - if total is None: - total = len(rows) - expanded = False for row in rows: - obj = None # don't build it if you don't need it - data = {} - for field in fields: - if field in select_fields: - data[field.replace("__", ".")] = row[select_fields.index(field)] - else: - if obj is None: # we need it! create it and cache it: - obj = self._tables[table]["class_func"].create(pickle.loads(row[0])) - expanded = True - # get the field, even if we need to do a join: - # FIXME: possible optimize: do a join in select for this if needed: - field = field.replace("__", ".") - data[field] = obj.get_field(field, self, ignore_errors=True) - result.append(data) - result.total = total - result.time = time.time() - start_time - result.query = query - result.expanded = expanded - return result + if fields[0] != "blob_data": + obj = None # don't build it if you don't need it + data = {} + for field in fields: + if field in select_fields: + data[field.replace("__", ".")] = row[select_fields.index(field)] + else: + if obj is None: # we need it! create it and cache it: + obj = self.get_table_func(table,"class_func").create(pickle.loads(row[0])) + # get the field, even if we need to do a join: + # FIXME: possible optimize: do a join in select for this if needed: + field = field.replace("__", ".") + data[field] = obj.get_field(field, self, ignore_errors=True) + yield data + else: + obj = self.get_table_func(table,"class_func").create(pickle.loads(row[0])) + yield obj diff --git a/gramps/plugins/database/dbapi_support/sqlite.py b/gramps/plugins/database/dbapi_support/sqlite.py index b3262fed9..0d527a3e3 100644 --- a/gramps/plugins/database/dbapi_support/sqlite.py +++ b/gramps/plugins/database/dbapi_support/sqlite.py @@ -1,14 +1,17 @@ import os import sqlite3 +import logging sqlite3.paramstyle = 'qmark' class Sqlite(object): def __init__(self, *args, **kwargs): + self.log = logging.getLogger(".sqlite") self.connection = sqlite3.connect(*args, **kwargs) self.queries = {} def execute(self, *args, **kwargs): + self.log.debug(args) self.cursor = self.connection.execute(*args, **kwargs) def fetchone(self): diff --git a/gramps/plugins/database/test/db_test.py b/gramps/plugins/database/test/db_test.py index 017279972..4b8e2202d 100644 --- a/gramps/plugins/database/test/db_test.py +++ b/gramps/plugins/database/test/db_test.py @@ -79,42 +79,170 @@ class BSDDBTest(unittest.TestCase): self.assertTrue(all([isinstance(r, EventRef) for r in result]), result) def test_select_1(self): - result = self.db.select("Person", ["gramps_id"]) + result = list(self.db._select("Person", ["gramps_id"])) self.assertTrue(len(result) == 60, len(result)) def test_select_2(self): - result = self.db.select("Person", ["gramps_id"], - where=("gramps_id", "LIKE", "I000%")) + result = list(self.db._select("Person", ["gramps_id"], + where=("gramps_id", "LIKE", "I000%"))) self.assertTrue(len(result) == 10, len(result)) def test_select_3(self): - result = self.db.select("Family", ["mother_handle.gramps_id"], - where=("mother_handle.gramps_id", "LIKE", "I003%")) + result = list(self.db._select("Family", ["mother_handle.gramps_id"], + where=("mother_handle.gramps_id", "LIKE", "I003%"))) self.assertTrue(len(result) == 6, result) def test_select_4(self): - result = self.db.select("Family", ["mother_handle.event_ref_list.ref.gramps_id"]) + result = list(self.db._select("Family", + ["mother_handle.event_ref_list.ref.gramps_id"])) self.assertTrue(len(result) == 23, len(result)) - def test_select_4(self): - result = self.db.select("Family", ["mother_handle.event_ref_list.ref.gramps_id"], - where=("mother_handle.event_ref_list.ref.gramps_id", "=", 'E0156')) - self.assertTrue(len(result) == 1, len(result)) - def test_select_5(self): - result = self.db.select("Family", ["mother_handle.event_ref_list.ref.self.gramps_id"]) + result = list(self.db._select("Family", + ["mother_handle.event_ref_list.ref.self.gramps_id"])) self.assertTrue(len(result) == 23, len(result)) def test_select_6(self): - result = self.db.select("Family", ["mother_handle.event_ref_list.0"]) - self.assertTrue(all([isinstance(r["mother_handle.event_ref_list.0"], (EventRef, type(None))) for r in result]), + result = list(self.db._select("Family", ["mother_handle.event_ref_list.0"])) + self.assertTrue(all([isinstance(r["mother_handle.event_ref_list.0"], + (EventRef, type(None))) for r in result]), [r["mother_handle.event_ref_list.0"] for r in result]) def test_select_7(self): - result = self.db.select("Family", ["mother_handle.event_ref_list.0"], - where=("mother_handle.event_ref_list.0", "!=", None)) + result = list(self.db._select("Family", ["mother_handle.event_ref_list.0"], + where=("mother_handle.event_ref_list.0", "!=", None))) self.assertTrue(len(result) == 21, len(result)) + def test_select_8(self): + result = list(self.db._select("Family", ["mother_handle.event_ref_list.ref.gramps_id"], + where=("mother_handle.event_ref_list.ref.gramps_id", "=", 'E0156'))) + self.assertTrue(len(result) == 1, len(result)) + + def test_queryset_1(self): + result = list(self.db.Person.select()) + self.assertTrue(len(result) == 60, len(result)) + + def test_queryset_2(self): + result = list(self.db.Person.filter(gramps_id__LIKE="I000%").select()) + self.assertTrue(len(result) == 10, len(result)) + + def test_queryset_3(self): + result = list(self.db.Family + .filter(mother_handle__gramps_id__LIKE="I003%") + .select()) + self.assertTrue(len(result) == 6, result) + + def test_queryset_4(self): + result = list(self.db.Family.select()) + self.assertTrue(len(result) == 23, len(result)) + + def test_queryset_4(self): + result = list(self.db.Family + .filter(mother_handle__event_ref_list__ref__gramps_id='E0156') + .select()) + self.assertTrue(len(result) == 1, len(result)) + + def test_queryset_5(self): + result = list(self.db.Family + .select("mother_handle.event_ref_list.ref.self.gramps_id")) + self.assertTrue(len(result) == 23, len(result)) + + def test_queryset_6(self): + result = list(self.db.Family.select("mother_handle.event_ref_list.0")) + self.assertTrue(all([isinstance(r["mother_handle.event_ref_list.0"], + (EventRef, type(None))) for r in result]), + [r["mother_handle.event_ref_list.0"] for r in result]) + + def test_queryset_7(self): + from gramps.gen.db import NOT + result = list(self.db.Family + .filter(NOT(mother_handle__event_ref_list__0=None)) + .select()) + self.assertTrue(len(result) == 21, len(result)) + + def test_order_1(self): + result = list(self.db.Person.order("gramps_id").select()) + self.assertTrue(len(result) == 60, len(result)) + + def test_order_2(self): + result = list(self.db.Person.order("-gramps_id").select()) + self.assertTrue(len(result) == 60, len(result)) + + def test_proxy_1(self): + result = list(self.db.Person.proxy("living", False).select()) + self.assertTrue(len(result) == 31, len(result)) + + def test_proxy_2(self): + result = list(self.db.Person.proxy("living", True).select()) + self.assertTrue(len(result) == 60, len(result)) + + def test_proxy_3(self): + result = len(list(self.db.Person + .proxy("private") + .order("-gramps_id") + .select("gramps_id"))) + self.assertTrue(result == 59, result) + + def test_map_1(self): + result = sum(list(self.db.Person.map(lambda p: 1).select())) + self.assertTrue(result == 60, result) + + def test_tag_1(self): + self.db.Person.filter(gramps_id="I0001").tag("Test") + result = self.db.Person.filter(tag_list__name="Test").count() + self.assertTrue(result == 1, result) + + # def test_filter_1(self): + # from gramps.gen.filters.rules.person import (IsDescendantOf, + # IsAncestorOf) + # from gramps.gen.filters import GenericFilter + # filter = GenericFilter() + # filter.set_logical_op("or") + # filter.add_rule(IsDescendantOf([self.db.get_default_person().gramps_id, + # True])) + # filter.add_rule(IsAncestorOf([self.db.get_default_person().gramps_id, + # True])) + # result = self.db.Person.filter(filter).count() + # self.assertTrue(result == 15, result) + + def test_filter_2(self): + result = self.db.Person.filter(lambda p: p.private).count() + self.assertTrue(result == 1, result) + + def test_filter_3(self): + result = self.db.Person.filter(lambda p: not p.private).count() + self.assertTrue(result == 59, result) + + def test_limit_1(self): + result = self.db.Person.limit(count=50).count() + self.assertTrue(result == 50, result) + + def test_limit_2(self): + result = self.db.Person.limit(start=50, count=50).count() + self.assertTrue(result == 10, result) + + def test_ordering_1(self): + worked = None + try: + result = list(self.db.Person + .filter(lambda p: p.private) + .order("private") + .select()) + worked = True + except: + worked = False + self.assertTrue(not worked, "should have failed") + + def test_ordering_2(self): + worked = None + try: + result = list(self.db.Person.order("private") + .filter(lambda p: p.private) + .select()) + worked = True + except: + worked = False + self.assertTrue(worked, "should have worked") class DBAPITest(BSDDBTest): dbwrap = DBAPI() diff --git a/gramps/test/test/test_util_test.py b/gramps/test/test/test_util_test.py index 760df928c..791cba01d 100644 --- a/gramps/test/test/test_util_test.py +++ b/gramps/test/test/test_util_test.py @@ -202,8 +202,9 @@ class Test4(U.TestCase): logging.error(emsg) ll = tl.logfile_getlines() nl = len(ll) - self.assertEquals(nl,3, - tu.msg(nl,3, "pass %d: expected line count" % i)) + print(repr(ll)) + self.assertEquals(nl,2, + tu.msg(nl,2, "pass %d: expected line count" % i)) #del tl