diff --git a/README.md b/README.md index a1c326498..c2fc42348 100644 --- a/README.md +++ b/README.md @@ -98,13 +98,6 @@ The following packages are optional: More font support in the reports -* **Meta** - - Required for experimental "where" clause creation. This functionality - is not yet in main-line code so it is not needed by users. If the package - will be used by gramps developers, to support further development, then - it may be included. Install with pypi: https://pypi.python.org/pypi/meta. - Optional packages required by Third-party Addons ------------------------------------------------ diff --git a/gramps/gen/db/base.py b/gramps/gen/db/base.py index e321cb46b..35c184963 100644 --- a/gramps/gen/db/base.py +++ b/gramps/gen/db/base.py @@ -1214,173 +1214,6 @@ class DbReadBase: """ raise NotImplementedError - def _select(self, table, fields=None, start=0, limit=-1, - where=None, order_by=None): - """ - Default implementation of a select for those databases - that don't support SQL. Returns a list of dicts, total, - and time. - - table - Person, Family, etc. - fields - used by object.get_field() - start - position to start - limit - count to get; -1 for all - where - (field, SQL string_operator, value) | - ["AND", [where, where, ...]] | - ["OR", [where, where, ...]] | - ["NOT", where] - order_by - [[fieldname, "ASC" | "DESC"], ...] - """ - def compare(v, op, value): - """ - Compare values in a SQL-like way - """ - if isinstance(v, (list, tuple)) and len(v) > 0: # join, or multi-values - # If any is true: - for item in v: - if compare(item, op, value): - return True - return False - if op in ["=", "=="]: - matched = v == value - elif op == ">": - matched = v > value - elif op == ">=": - matched = v >= value - elif op == "<": - matched = v < value - elif op == "<=": - matched = v <= value - elif op == "IN": - matched = v in value - elif op == "IS": - matched = v is value - elif op == "IS NOT": - matched = v is not value - elif op == "IS NULL": - matched = v is None - elif op == "IS NOT NULL": - matched = v is not None - elif op == "BETWEEN": - matched = value[0] <= v <= value[1] - elif op in ["<>", "!="]: - matched = v != value - elif op == "LIKE": - if value and v: - value = value.replace("%", "(.*)").replace("_", ".") - ## FIXME: allow a case-insensitive version - matched = re.match("^" + value + "$", v, re.MULTILINE) - else: - matched = False - elif op == "REGEXP": - if value and v: - matched = re.search(value, v, re.MULTILINE) is not None - else: - matched = False - else: - raise Exception("invalid select operator: '%s'" % op) - return True if matched else False - - def evaluate_values(condition, item, db, table, env): - """ - Evaluates the names in all conditions. - """ - if len(condition) == 2: # ["AND" [...]] | ["OR" [...]] | ["NOT" expr] - connector, exprs = condition - if connector in ["AND", "OR"]: - for expr in exprs: - evaluate_values(expr, item, db, table, env) - else: # "NOT" - evaluate_values(exprs, item, db, table, env) - elif len(condition) == 3: # (name, op, value) - (name, op, value) = condition - # just the ones we need for where - hname = self._hash_name(table, name) - if hname not in env: - value = item.get_field(name, db, ignore_errors=True) - env[hname] = value - - def evaluate_truth(condition, item, db, table, env): - if len(condition) == 2: # ["AND"|"OR" [...]] - connector, exprs = condition - if connector == "AND": # all must be true - for expr in exprs: - if not evaluate_truth(expr, item, db, table, env): - return False - return True - elif connector == "OR": # any will return true - for expr in exprs: - if evaluate_truth(expr, item, db, table, env): - return True - return False - elif connector == "NOT": # return not of single value - return not evaluate_truth(exprs, item, db, table, env) - else: - raise Exception("No such connector: '%s'" % connector) - elif len(condition) == 3: # (name, op, value) - (name, op, value) = condition - v = env.get(self._hash_name(table, name)) - return compare(v, op, value) - - # Fields is None or list, maybe containing "*": - if fields is None: - pass # ok - elif not isinstance(fields, (list, tuple)): - raise Exception("fields must be a list/tuple of field names") - elif "*" in fields: - fields.remove("*") - fields.extend(self.get_table_func(table,"class_func").get_schema().keys()) - get_count_only = (fields is not None and fields[0] == "count(1)") - position = 0 - selected = 0 - if get_count_only: - if where or limit != -1 or start != 0: - # no need to order for a count - data = self.get_table_func(table,"iter_func")() - else: - yield self.get_table_func(table,"count_func")() - else: - data = self.get_table_func(table, "iter_func")(order_by=order_by) - if where: - for item in data: - # Go through all fliters and evaluate the fields: - env = {} - evaluate_values(where, item, self, table, env) - matched = evaluate_truth(where, item, self, table, env) - if matched: - if ((selected < limit) or (limit == -1)) and start <= position: - selected += 1 - if not get_count_only: - if fields: - row = {} - for field in fields: - value = item.get_field(field, self, ignore_errors=True) - row[field.replace("__", ".")] = value - yield row - else: - yield item - position += 1 - if get_count_only: - yield selected - else: # no where - for item in data: - if position >= start: - if ((selected >= limit) and (limit != -1)): - break - selected += 1 - if not get_count_only: - if fields: - row = {} - for field in fields: - value = item.get_field(field, self, ignore_errors=True) - row[field.replace("__", ".")] = value - yield row - else: - yield item - position += 1 - if get_count_only: - yield selected - def _hash_name(self, table, name): """ Used in SQL functions to eval expressions involving selected @@ -1389,17 +1222,6 @@ class DbReadBase: name = self.get_table_func(table,"class_func").get_field_alias(name) return name.replace(".", "__") - Person = property(lambda self: QuerySet(self, "Person")) - Family = property(lambda self: QuerySet(self, "Family")) - Note = property(lambda self: QuerySet(self, "Note")) - Citation = property(lambda self: QuerySet(self, "Citation")) - Source = property(lambda self: QuerySet(self, "Source")) - Repository = property(lambda self: QuerySet(self, "Repository")) - Place = property(lambda self: QuerySet(self, "Place")) - Event = property(lambda self: QuerySet(self, "Event")) - Tag = property(lambda self: QuerySet(self, "Tag")) - Media = property(lambda self: QuerySet(self, "Media")) - class DbWriteBase(DbReadBase): """ Gramps database object. This object is a base class for all @@ -1976,12 +1798,6 @@ class DbWriteBase(DbReadBase): else: raise ValueError("invalid instance type: %s" % instance.__class__.__name__) - def get_queryset_by_table_name(self, table_name): - """ - Get Person, Family queryset by name. - """ - return getattr(self, table_name) - def autobackup(self, user=None): """ Backup the current file as a backup file. @@ -2001,222 +1817,3 @@ class DbWriteBase(DbReadBase): if user.uistate: user.uistate.set_busy_cursor(False) user.uistate.progress.hide() - -class QuerySet: - """ - A container for selection criteria before being actually - applied to a database. - """ - def __init__(self, database, table): - self.database = database - self.table = table - self.generator = None - self.where_by = None - self.order_by = None - self.limit_by = -1 - self.start = 0 - self.needs_to_run = False - - def limit(self, start=None, count=None): - """ - Put limits on the selection. - """ - if start is not None: - self.start = start - if count is not None: - self.limit_by = count - self.needs_to_run = True - return self - - def order(self, *args): - """ - Put an ordering on the selection. - """ - for arg in args: - if self.order_by is None: - self.order_by = [] - if arg.startswith("-"): - self.order_by.append((arg[1:], "DESC")) - else: - self.order_by.append((arg, "ASC")) - self.needs_to_run = True - return self - - def _add_where_clause(self, *args): - """ - Add a condition to the where clause. - """ - # First, handle AND, OR, NOT args: - and_expr = [] - for expr in args: - and_expr.append(expr) - # Next, handle kwargs: - if and_expr: - if self.where_by: - self.where_by = ["AND", [self.where_by] + and_expr] - elif len(and_expr) == 1: - self.where_by = and_expr[0] - else: - self.where_by = ["AND", and_expr] - self.needs_to_run = True - return self - - def count(self): - """ - Run query with just where, start, limit to get count. - """ - if self.generator and self.needs_to_run: - raise Exception("Queries in invalid order") - elif self.generator: - return len(list(self.generator)) - else: - generator = self.database._select(self.table, - ["count(1)"], - where=self.where_by, - start=self.start, - limit=self.limit_by) - return next(generator) - - def _generate(self, args=None): - """ - Create a generator from current options. - """ - generator = self.database._select(self.table, - args, - order_by=self.order_by, - where=self.where_by, - start=self.start, - limit=self.limit_by) - # Reset all criteria - self.where_by = None - self.order_by = None - self.limit_by = -1 - self.start = 0 - self.needs_to_run = False - return generator - - def select(self, *args): - """ - Actually touch the database. - """ - if len(args) == 0: - args = None - if self.generator and self.needs_to_run: - ## problem - raise Exception("Queries in invalid order") - elif self.generator: - if args: # there is a generator, with args - for i in self.generator: - yield [i.get_field(arg) for arg in args] - else: # generator, no args - for i in self.generator: - yield i - else: # need to run or not - self.generator = self._generate(args) - for i in self.generator: - yield i - - def proxy(self, proxy_name, *args, **kwargs): - """ - Apply a named proxy to the db. - """ - from gramps.gen.proxy import (LivingProxyDb, PrivateProxyDb, - ReferencedBySelectionProxyDb) - if proxy_name == "living": - proxy_class = LivingProxyDb - elif proxy_name == "private": - proxy_class = PrivateProxyDb - elif proxy_name == "referenced": - proxy_class = ReferencedBySelectionProxyDb - else: - raise Exception("No such proxy name: '%s'" % proxy_name) - self.database = proxy_class(self.database, *args, **kwargs) - return self - - def where(self, where_clause): - """ - Apply a where_clause (closure) to the selection process. - """ - from gramps.gen.db.where import eval_where - # if there is already a generator, then error: - if self.generator: - raise Exception("Queries in invalid order") - where_by = eval_where(where_clause) - self._add_where_clause(where_by) - return self - - def filter(self, *args): - """ - Apply a filter to the database. - """ - from gramps.gen.proxy import FilterProxyDb - from gramps.gen.filters import GenericFilter - from gramps.gen.db.where import eval_where - for i in range(len(args)): - arg = args[i] - if isinstance(arg, GenericFilter): - self.database = FilterProxyDb(self.database, arg, *args[i+1:]) - if hasattr(arg, "where"): - where_by = eval_where(arg.where) - self._add_where_clause(where_by) - elif callable(arg): - if self.generator and self.needs_to_run: - ## error - raise Exception("Queries in invalid order") - elif self.generator: - pass # ok - else: - self.generator = self._generate() - self.generator = filter(arg, self.generator) - else: - pass # ignore, may have been arg from previous Filter - return self - - def map(self, f): - """ - Apply the function f to the selected items and return results. - """ - if self.generator and self.needs_to_run: - raise Exception("Queries in invalid order") - elif self.generator: - pass # ok - else: - self.generator = self._generate() - previous_generator = self.generator - def generator(): - for item in previous_generator: - yield f(item) - self.generator = generator() - return self - - def tag(self, tag_text, remove=False): - """ - Tag or untag the selected items with the tag name. - """ - if self.generator and self.needs_to_run: - raise Exception("Queries in invalid order") - elif self.generator: - pass # ok - else: - self.generator = self._generate() - tag = self.database.get_tag_from_name(tag_text) - if (not tag and remove): - # no tag by this name, and want to remove it - # nothing to do - return - trans_class = self.database.get_transaction_class() - with trans_class("Tag Selected Items", self.database, batch=False) as trans: - if tag is None: - tag = self.database.get_table_func("Tag","class_func")() - tag.set_name(tag_text) - self.database.add_tag(tag, trans) - commit_func = self.database.get_table_func(self.table,"commit_func") - for item in self.generator: - if remove and (tag.handle in item.tag_list): - item.remove_tag(tag.handle) - elif (not remove) and (tag.handle not in item.tag_list): - item.add_tag(tag.handle) - else: - continue - commit_func(item, trans) - diff --git a/gramps/gen/db/generic.py b/gramps/gen/db/generic.py index b301abd86..831b31c0e 100644 --- a/gramps/gen/db/generic.py +++ b/gramps/gen/db/generic.py @@ -53,7 +53,6 @@ from gramps.gen.db import (DbReadBase, DbWriteBase, DbTxn, DbUndo, PLACE_KEY, REPOSITORY_KEY, NOTE_KEY, TAG_KEY, eval_order_by) from gramps.gen.errors import HandleError -from gramps.gen.db.base import QuerySet from gramps.gen.utils.callback import Callback from gramps.gen.updatecallback import UpdateCallback from gramps.gen.db.dbconst import * @@ -2260,7 +2259,6 @@ class DbGeneric(DbWriteBase, DbReadBase, UpdateCallback, Callback): Add a new table and funcs to the database. """ self.__tables[table] = funcs - setattr(DbGeneric, table, property(lambda self: QuerySet(self, table))) def get_version(self): """ diff --git a/gramps/gen/db/test/test_where.py b/gramps/gen/db/test/test_where.py deleted file mode 100644 index 5dc84e870..000000000 --- a/gramps/gen/db/test/test_where.py +++ /dev/null @@ -1,110 +0,0 @@ -# -# Gramps - a GTK+/GNOME based genealogy program -# -# Copyright (C) 2016 Gramps Development Team -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -# - -from gramps.gen.db.where import eval_where -from gramps.gen.lib import Person -import unittest - -########## -# Tests: - -def make_closure(surname): - """ - Test closure. - """ - from gramps.gen.lib import Person - return (lambda person: - (person.primary_name.surname_list[0].surname == surname and - person.gender == Person.MALE)) - -class Thing: - def __init__(self): - self.list = ["I0", "I1", "I2"] - - def where(self): - return lambda person: person.gramps_id == self.list[1] - - def apply(self, db, person): - return person.gender == Person.MALE - -class ClosureTest(unittest.TestCase): - def check(self, test): - result = eval_where(test[0]) - self.assertTrue(result == test[1], "%s is not %s" % (result, test[1])) - - def test_01(self): - self.check( - (lambda family: (family.private and - family.mother_handle.gramps_id != "I0001"), - ['AND', [['private', '==', True], - ['mother_handle.gramps_id', '!=', 'I0001']]])) - - def test_02(self): - self.check( - (lambda person: LIKE(person.gramps_id, "I0001"), - ['gramps_id', 'LIKE', 'I0001'])) - - def test_03(self): - self.check( - (lambda note: note.gramps_id == "N0001", - ['gramps_id', '==', 'N0001'])) - - def test_04(self): - self.check( - (lambda person: person.event_ref_list.ref.gramps_id == "E0001", - ['event_ref_list.ref.gramps_id', '==', 'E0001'])) - - def test_05(self): - self.check( - (lambda person: LIKE(person.gramps_id, "I0001") or person.private, - ["OR", [['gramps_id', 'LIKE', 'I0001'], - ["private", "==", True]]])) - - def test_06(self): - self.check( - (lambda person: person.event_ref_list <= 0, - ["event_ref_list", "<=", 0])) - - def test_07(self): - self.check( - (lambda person: person.primary_name.surname_list[0].surname == "Smith", - ["primary_name.surname_list.0.surname", "==", "Smith"])) - - def test_08(self): - self.check( - (make_closure("Smith"), - ["AND", [["primary_name.surname_list.0.surname", "==", "Smith"], - ["gender", "==", 1]]])) - - def test_09(self): - self.check( - [Thing().where(), ["gramps_id", "==", "I1"]]) - - def test_10(self): - self.check( - (lambda person: LIKE(person.gramps_id, "I000%"), - ["gramps_id", "LIKE", "I000%"])) - - def test_11(self): - self.check( - [Thing().apply, ["gender", "==", 1]]) - -if __name__ == "__main__": - unittest.main() diff --git a/gramps/gen/db/where.py b/gramps/gen/db/where.py deleted file mode 100644 index e63b2a718..000000000 --- a/gramps/gen/db/where.py +++ /dev/null @@ -1,222 +0,0 @@ -# -# Gramps - a GTK+/GNOME based genealogy program -# -# Copyright (C) 2016 Gramps Development Team -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -# - -from meta.asttools import Visitor -from meta.decompiler import _ast, decompile_func - -import copy - -class ParseFilter(Visitor): - """ - This class is used to turn Python lambda expressions into AST - which is used as a SELECT statements in databases via the .where() - method. This is used by both BSDDB and SQL-based databases. - - Not all Python is allowed as a where-clause, and some functions - used here are not real Python functions. - - Examples: - - db.Person.where( - lambda person: person.gramps_id == "I0001" - ).select() - - Some uses look (and evaluate) like regular Python. - - db.Person.where( - lambda person: LIKE(person.gramps_id, "I000%") - ).select() - - LIKE is not a real Python function, but the syntax is used to - indicate a fuzzy match. - - db.Family.where( - lambda family: LIKE(family.mother_handle.gramps_id, "I003%") - ).select() - - LIKE uses % as a wildcard matching character, like ".*" in re. - - db.Family.where( - lambda family: family.mother_handle.event_ref_list.ref.gramps_id == 'E0156' - ).select() - - Here, property chaining is shown without having to check to see if - values actually exist. The checking for valid/existing properties - is done by the select system. - - db.Family.where( - lambda family: family.mother_handle.event_ref_list[0] != None - ).select() - - Indexing and use of None is allowed. - - db.Person.where( - lambda person: person.private == True - ).select() - - One current limitiation is that it cannot detect a boolean value, - so we must use the "== True" to make sure the proper code is - generated. - - The following method names are dictated by meta's Visitor. Additional - methods can be added if an error is received such as: - - AttributeError: visitXXX does not exist - - The method must be added, return the proper value for that - syntax. May require recursive calls to process_ITEM(). - - Please see meta for more information: - http://srossross.github.io/Meta/html/index.html - """ - - def visitName(self, node): - return node.id - - def visitNum(self, node): - return node.n - - def visitlong(self, node): - return node - - def process_expression(self, expr): - if isinstance(expr, str): - # boolean - return [self.process_field(expr), "==", True] - elif len(expr) == 3: - # (field, op, value) - return [self.process_field(expr[0]), - expr[1], - self.process_value(expr[2])] - else: - # list of exprs - return [self.process_expression(exp) for - exp in expr] - - def process_value(self, value): - try: - return eval(value, self.env) - except: - return value - - def process_field(self, field): - field = field.replace("[", ".").replace("]", "") - if field.startswith(self.parameter + "."): - return field[len(self.parameter) + 1:] - else: - return field - - def visitCall(self, node): - """ - Handle LIKE() - """ - return [self.process_field(self.visit(node.args[0])), - self.visit(node.func), - self.process_value(self.visit(node.args[1]))] - - def visitStr(self, node): - return node.s - - def visitlist(self, list): - return [self.visit(node) for node in list] - - def visitCompare(self, node): - return [self.process_field(self.visit(node.left)), - " ".join(self.visit(node.ops)), - self.process_value(self.visit(node.comparators[0]))] - - def visitAttribute(self, node): - return "%s.%s" % (self.visit(node.value), node.attr) - - def get_boolean_op(self, node): - if isinstance(node, _ast.And): - return "AND" - elif isinstance(node, _ast.Or): - return "OR" - else: - raise Exception("invalid boolean") - - def visitNotEq(self, node): - return "!=" - - def visitLtE(self, node): - return "<=" - - def visitGtE(self, node): - return ">=" - - def visitEq(self, node): - return "==" - - def visitBoolOp(self, node): - """ - BoolOp: boolean operator - """ - op = self.get_boolean_op(node.op) - values = list(node.values) - return [op, self.process_expression( - [self.visit(value) for value in values])] - - def visitLambda(self, node): - self.parameter = self.visit(node.args)[0] - return self.visit(node.body) - - def visitFunctionDef(self, node): - self.parameter = self.visit(node.args)[2] # ['self', 'db', 'person'] - return self.visit(node.body)[0] - - def visitReturn(self, node): - return self.visit(node.value) - - def visitarguments(self, node): - return [self.visit(arg) for arg in node.args] - - def visitarg(self, node): - return node.arg - - def visitSubscript(self, node): - return "%s[%s]" % (self.visit(node.value), - self.visit(node.slice)) - - def visitIndex(self, node): - return self.visit(node.value) - -def make_env(closure): - """ - Create an environment from the closure. - """ - env = copy.copy(closure.__globals__) - if closure.__closure__: - for i in range(len(closure.__closure__)): - env[closure.__code__.co_freevars[i]] = closure.__closure__[i].cell_contents - return env - -def eval_where(closure): - """ - Given a closure, parse and evaluate it. - Return a WHERE expression. - - See ParseFilter.__doc__ for more information and examples. - """ - parser = ParseFilter() - parser.env = make_env(closure) - ast_top = decompile_func(closure) - result = parser.visit(ast_top) - return result diff --git a/gramps/gen/lib/struct.py b/gramps/gen/lib/struct.py deleted file mode 100644 index 0e8d0b3fb..000000000 --- a/gramps/gen/lib/struct.py +++ /dev/null @@ -1,359 +0,0 @@ -# -# Gramps - a GTK+/GNOME based genealogy program -# -# Copyright (C) 2015 Gramps Development Team -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# - -from gramps.gen.lib.handle import HandleClass - -def from_struct(struct): - return Struct.instance_from_struct(struct) - -class Struct: - """ - Class for getting and setting parts of a struct by dotted path. - - >>> s = Struct({"gramps_id": "I0001", ...}, database) - >>> s.primary_name.surname_list[0].surname - Jones - >>> s.primary_name.surname_list[0].surname = "Smith" - >>> s.primary_name.surname_list[0]surname - Smith - """ - def __init__(self, struct, db=None): - self.struct = struct - self.db = db - if self.db: - self.transaction = db.get_transaction_class() - else: - self.transaction = None - - @classmethod - def wrap(cls, instance, db=None): - return Struct(instance.to_struct(), db) - - def __setitem__(self, item, value): - self.struct[item] = value - - def __eq__(self, other): - if isinstance(other, Struct): - return self.struct == other.struct - elif isinstance(self.struct, list): - ## FIXME: self.struct can be a dict, list, etc - for item in self.struct: - if item == other: - return True - return False - else: - return self.struct == other - - def __lt__(self, other): - if isinstance(other, Struct): - return self.struct < other.struct - else: - return self.struct < other - - def __gt__(self, other): - if isinstance(other, Struct): - return self.struct > other.struct - else: - return self.struct > other - - def __le__(self, other): - if isinstance(other, Struct): - return self.struct <= other.struct - else: - return self.struct <= other - - def __ge__(self, other): - if isinstance(other, Struct): - return self.struct >= other.struct - else: - return self.struct >= other - - def __ne__(self, other): - if isinstance(other, Struct): - return self.struct != other.struct - else: - return self.struct != other - - def __len__(self): - return len(self.struct) - - def __contains__(self, item): - return item in self.struct - - def __call__(self, *args, **kwargs): - """ - You can use this to select and filter a list of structs. - - args are dotted strings of what componets of the structs to - select, and kwargs is the selection criteria, double-under - scores represent dots. - - If no args are given, all are provided. - """ - selected = self.struct # better be dicts - # First, find elements of the list that match any given - # selection criteria: - selected = self.struct # assume dicts - # assume True - to_delete = [] - for key in kwargs: # value="Social Security Number" - parts = self.getitem_from_path(key.split("__")) # returns all - # This will return a list; we keep the ones that match - for p in range(len(parts)): - # if it matches, keep it: - if parts[p] != kwargs[key]: - to_delete.append(p) - # delete from highest to lowest, to use pop: - for p in reversed(to_delete): - selected.pop(p) - # now select which parts to show: - if args: # just some of the parts, ["type.string", ...] - results = [] - for select in selected: # dict in dicts - parts = [] - for item in args: # ["type.string"] - items = item.split(".") # "type.string" - values = Struct(select, self.db).getitem_from_path(items) - if values: - parts.append((item, values)) - results.append(parts) # return [["type.string", "Social Security Number"], ...] - else: # return all - results = selected - # return them - return results - - def select(self, thing1, thing2): - if thing2 == "*": - return thing1 - elif thing2 in thing1: - return thing2 - elif thing1 == thing2: - return thing1 - else: - return None - - def __getattr__(self, attr): - """ - Called when getattr fails. Lookup attr in struct; returns Struct - if more struct. - - >>> Struct({}, db).primary_name - returns: Struct([], db) or value - - struct can be list/tuple, dict with _class, or value (including dict). - - self.setitem_from_path(path, v) should be used to set value of - item. - """ - if isinstance(self.struct, dict) and "_class" in self.struct.keys(): - # this is representing an object - if attr in self.struct.keys(): - return self.handle_join(self.struct[attr]) - else: - raise AttributeError("attempt to access a property of an object: '%s', '%s'" % (self.struct, attr)) - elif isinstance(self.struct, HandleClass): - struct = self.handle_join(self.struct) - return getattr(struct, attr) - elif isinstance(self.struct, (list, tuple)): - # get first item in list that matches: - sublist = [getattr(Struct(item, self.db), attr) for item in self.struct] - return Struct(sublist, self.db) - elif hasattr(self.struct, attr): - # better be a property of the list/tuple/dict/value: - return getattr(self.struct, attr) - else: - return Struct({}, self.db) # dummy, extending a previous dummy - - def __getitem__(self, item): - """ - Called when getitem fails. Lookup item in struct; returns Struct - if more struct. - - >>> Struct({}, db)[12] - returns: Struct([], db) or value - - struct can be list/tuple, dict with _class, or value (including dict). - """ - if isinstance(item, str) and isinstance(self.struct, (list, tuple)): - fields = [field.strip() for field in item.split(",")] - results = [] - for item in self.struct: - sublist = [getattr(Struct(item, self.db), field) for field in fields] - if any(sublist): - results.append(tuple(sublist)) - return results if results else None - else: - return self.handle_join(self.struct[item]) - - def getitem_from_path(self, items): - """ - path is a list - """ - current = self - for item in items: - current = getattr(current, item) - return current - - def handle_join(self, item): - """ - If the item is a handle, look up reference object. - """ - if isinstance(item, HandleClass) and self.db: - obj = self.db.get_from_name_and_handle(item.classname, str(item)) - if obj: - return Struct(obj.to_struct(), self.db) - else: - return Struct({}, self.db) # dummy, a db error - elif isinstance(item, (list, tuple)): - return Struct(item, self.db) - elif isinstance(item, dict) and "_class" in item.keys(): - return Struct(item, self.db) - else: - return item - - def setitem(self, path, value, trans=None): - """ - Given a path to a struct part, set the last part to value. - - >>> Struct(struct).setitem("primary_name.surname_list.0.surname", "Smith") - """ - return self.setitem_from_path(parse(path), value, trans) - - def primary_object_q(self, _class): - return _class in ["Person", "Family", "Event", "Source", "Citation", - "Tag", "Repository", "Note", "Media"] - - def setitem_from_path(self, path, value, trans=None): - """ - Given a path to a struct part, set the last part to value. - - >>> Struct(struct).setitem_from_path(["primary_name", "surname_list", "[0]", "surname"], "Smith", transaction) - """ - path, item = path[:-1], path[-1] - if item.startswith("["): - item = item[1:-1] - struct = self.struct - primary_obj = struct - for p in range(len(path)): - part = path[p] - if part.startswith("["): # getitem - struct = struct[eval(part[1:-1])] # for int or string use - else: # getattr - struct = struct[part] - if struct is None: # invalid part to set, skip - return - if isinstance(struct, HandleClass): - obj = self.db.get_from_name_and_handle(struct.classname, str(struct)) - struct = obj.to_struct() - # keep track of primary object for update, below - if isinstance(struct, dict) and "_class" in struct and self.primary_object_q(struct["_class"]): - primary_obj = struct - # struct is now set - if item in struct and isinstance(struct[item], list): # assigning to a list - if value is not None: - struct[item].append(value) # we append the value - else: - struct[item] = [] - elif isinstance(struct, (list, tuple)): - pos = int(item) - if pos < len(struct): - if value is not None: - struct[int(item)] = value - else: - struct.pop(int(item)) - elif isinstance(struct, dict): - if item in struct.keys(): - struct[item] = value - elif hasattr(struct, item): - setattr(struct, item, value) - else: - return - self.update_db(primary_obj, trans) - - def update_db(self, struct, trans=None): - if self.db: - if trans is None: - with self.transaction("Struct Update", self.db, batch=True) as trans: - new_obj = Struct.instance_from_struct(struct) - name, handle = struct["_class"], struct["handle"] - old_obj = self.db.get_from_name_and_handle(name, handle) - if old_obj: - commit_func = self.db.get_table_func(name,"commit_func") - commit_func(new_obj, trans) - else: - add_func = self.db.get_table_func(name,"add_func") - add_func(new_obj, trans) - else: - new_obj = Struct.instance_from_struct(struct) - name, handle = struct["_class"], struct["handle"] - old_obj = self.db.get_from_name_and_handle(name, handle) - if old_obj: - commit_func = self.db.get_table_func(name,"commit_func") - commit_func(new_obj, trans) - else: - add_func = self.db.get_table_func(name,"add_func") - add_func(new_obj, trans) - - def from_struct(self): - return Struct.instance_from_struct(self.struct) - - @classmethod - def instance_from_struct(cls, struct): - """ - Given a struct with metadata, create a Gramps object. - - self is class when called as a classmethod. - """ - from gramps.gen.lib import (Person, Family, Event, Source, Place, Citation, - Repository, Media, Note, Tag, Date) - if isinstance(struct, dict): - if "_class" in struct.keys(): - if struct["_class"] == "Person": - return Person.create(Person.from_struct(struct)) - elif struct["_class"] == "Family": - return Family.create(Family.from_struct(struct)) - elif struct["_class"] == "Event": - return Event.create(Event.from_struct(struct)) - elif struct["_class"] == "Source": - return Source.create(Source.from_struct(struct)) - elif struct["_class"] == "Place": - return Place.create(Place.from_struct(struct)) - elif struct["_class"] == "Citation": - return Citation.create(Citation.from_struct(struct)) - elif struct["_class"] == "Repository": - return Repository.create(Repository.from_struct(struct)) - elif struct["_class"] == "Media": - return Media.create(Media.from_struct(struct)) - elif struct["_class"] == "Note": - return Note.create(Note.from_struct(struct)) - elif struct["_class"] == "Tag": - return Tag.create(Tag.from_struct(struct)) - elif struct["_class"] == "Date": - return Date().unserialize(Date.from_struct(struct, full=True)) - raise AttributeError("invalid struct: %s" % struct) - - def __str__(self): - return str(self.struct) - - def __repr__(self): - if "_class" in self.struct: - return "<%s struct instance>" % self._class - else: - return repr(self.struct) diff --git a/gramps/gen/lib/test/struct_test.py b/gramps/gen/lib/test/struct_test.py index bb38c450d..63bccf2d7 100644 --- a/gramps/gen/lib/test/struct_test.py +++ b/gramps/gen/lib/test/struct_test.py @@ -25,7 +25,6 @@ import os from .. import (Person, Family, Event, Source, Place, Citation, Repository, Media, Note, Tag) -from gramps.gen.lib.struct import Struct from gramps.gen.merge.diff import import_as_dict from gramps.cli.user import User from gramps.gen.merge.diff import * @@ -119,18 +118,5 @@ for table in db.get_table_func(): obj = db.get_table_func(table,"handle_func")(handle) generate_case(obj) -class StructTest(unittest.TestCase): - def test(self): - family = db.get_family_from_gramps_id("F0001") - s = Struct(family.to_struct(), db) - self.assertEqual(s["gramps_id"], "F0001") - s["gramps_id"] = "TEST" - self.assertEqual(s["gramps_id"], "TEST") - self.assertEqual(s.father_handle.primary_name.first_name, - "Allen Carl") - s["father_handle.primary_name.first_name"] = "Edward" - self.assertEqual(s["father_handle.primary_name.first_name"], - "Edward") - if __name__ == "__main__": unittest.main() diff --git a/gramps/plugins/db/dbapi/dbapi.py b/gramps/plugins/db/dbapi/dbapi.py index e6b71324a..f522e4a2a 100644 --- a/gramps/plugins/db/dbapi/dbapi.py +++ b/gramps/plugins/db/dbapi/dbapi.py @@ -2080,63 +2080,6 @@ class DBAPI(DbGeneric): else: return repr(value) - def _build_where_clause_recursive(self, table, where): - """ - where - (field, op, value) - - ["NOT", where] - - ["AND", (where, ...)] - - ["OR", (where, ...)] - """ - if where is None: - return "" - elif len(where) == 3: - field, db_op, value = where - return "(%s %s %s)" % (self._hash_name(table, field), - db_op, self._sql_repr(value)) - elif where[0] in ["AND", "OR"]: - parts = [self._build_where_clause_recursive(table, part) - for part in where[1]] - return "(%s)" % ((" %s " % where[0]).join(parts)) - else: - return "(NOT %s)" % self._build_where_clause_recursive(table, - where[1]) - - def _build_where_clause(self, table, where): - """ - where - a list in where format - return - "WHERE conditions..." - """ - parts = self._build_where_clause_recursive(table, where) - if parts: - return "WHERE " + parts - else: - return "" - - def _build_order_clause(self, table, order_by): - """ - order_by - [(field, "ASC" | "DESC"), ...] - """ - if order_by: - order_clause = ", ".join(["%s %s" % (self._hash_name(table, field), - dir) - for (field, dir) in order_by]) - return "ORDER BY " + order_clause - else: - return "" - - def _build_select_fields(self, table, select_fields, secondary_fields): - """ - fields - [field, ...] - return: "field, field, field" - """ - all_available = all([(field in secondary_fields) - for field in select_fields]) - if all_available: # we can get them without expanding - return select_fields - else: - # nope, we'll have to expand blob to get all fields - return ["blob_data"] - def _check_order_by_fields(self, table, order_by, secondary_fields): """ Check to make sure all order_by fields are defined. If not, then @@ -2150,128 +2093,6 @@ class DBAPI(DbGeneric): return False return True - def _check_where_fields(self, table, where, secondary_fields): - """ - Check to make sure all where fields are defined. If not, then - we need to do the Python-based select. - - secondary_fields are hashed. - """ - if where is None: - return True - elif len(where) == 2: # ["AND" [...]] | ["OR" [...]] | ["NOT" expr] - connector, exprs = where - if connector in ["AND", "OR"]: - for expr in exprs: - value = self._check_where_fields(table, expr, - secondary_fields) - if value == False: - return False - return True - else: # "NOT" - return self._check_where_fields(table, exprs, secondary_fields) - elif len(where) == 3: # (name, db_op, value) - (name, db_op, value) = where - # just the ones we need for where - return self._hash_name(table, name) in secondary_fields - - def _select(self, table, fields=None, start=0, limit=-1, - where=None, order_by=None): - """ - Default implementation of a select for those databases - that don't support SQL. Returns a list of dicts, total, - and time. - - table - Person, Family, etc. - fields - used by object.get_field() - start - position to start - limit - count to get; -1 for all - where - (field, SQL string_operator, value) | - ["AND", [where, where, ...]] | - ["OR", [where, where, ...]] | - ["NOT", where] - order_by - [[fieldname, "ASC" | "DESC"], ...] - """ - secondary_fields = ([self._hash_name(table, field) - for (field, ptype) - in self.get_table_func( - table, "class_func").get_secondary_fields()] - + ["handle"]) - # handle is a sql field, but not listed in secondaries - # If no fields, then we need objects: - # Check to see if where matches SQL fields: - table_name = table.lower() - if ((not self._check_where_fields(table, where, secondary_fields)) - or (not self._check_order_by_fields(table, order_by, - secondary_fields))): - # If not, then need to do select via Python: - generator = super()._select(table, fields, start, - limit, where, order_by) - for item in generator: - yield item - return - # Otherwise, we are SQL - if fields is None: - fields = ["blob_data"] - get_count_only = False - if fields[0] == "count(1)": - hashed_fields = ["count(1)"] - fields = ["count(1)"] - select_fields = ["count(1)"] - get_count_only = True - else: - hashed_fields = [self._hash_name(table, field) for field in fields] - fields = hashed_fields - select_fields = self._build_select_fields(table, fields, - secondary_fields) - where_clause = self._build_where_clause(table, where) - order_clause = self._build_order_clause(table, order_by) - if get_count_only: - select_fields = ["1"] - if start: - query = "SELECT %s FROM %s %s %s LIMIT %s, %s " % ( - ", ".join(select_fields), - table_name, where_clause, order_clause, start, limit - ) - else: - query = "SELECT %s FROM %s %s %s LIMIT %s" % ( - ", ".join(select_fields), - table_name, where_clause, order_clause, limit - ) - if get_count_only: - self.dbapi.execute("SELECT count(1) from (%s) AS temp_select;" - % query) - rows = self.dbapi.fetchall() - yield rows[0][0] - return - self.dbapi.execute(query) - rows = self.dbapi.fetchall() - for row in rows: - if fields[0] != "blob_data": - obj = None # don't build it if you don't need it - data = {} - for field in fields: - if field in select_fields: - data[field.replace("__", ".") - ] = row[select_fields.index(field)] - else: - if obj is None: # we need it! create it and cache it: - obj = self.get_table_func(table, - "class_func").create( - pickle.loads(row[0])) - # get the field, even if we need to do a join: - # FIXME: possible optimize: - # do a join in select for this if needed: - field = field.replace("__", ".") - data[field] = obj.get_field(field, self, - ignore_errors=True) - yield data - else: - obj = self.get_table_func(table, - "class_func").create( - pickle.loads(row[0])) - yield obj - def get_summary(self): """ Returns dictionary of summary item.