From 2dae5c3a08692151fa7670f32456e6f90a84d833 Mon Sep 17 00:00:00 2001 From: Doug Blank Date: Wed, 19 Aug 2015 22:15:14 -0400 Subject: [PATCH] 8377: Slow scrolling in Gramps 4.X, on all platforms. Additions: 1. cache database access for column values 2. cache do_get_path lookups 3. sped up load access on treeviews with no filters 4. use cache in do_get_path from siblings 5. new LRU size of 1k (was 250) --- gramps/gen/config.py | 1 + gramps/gui/views/treemodels/basemodel.py | 101 ++++++ .../gui/views/treemodels/citationbasemodel.py | 154 +++++--- gramps/gui/views/treemodels/eventmodel.py | 41 ++- gramps/gui/views/treemodels/familymodel.py | 128 ++++--- gramps/gui/views/treemodels/flatbasemodel.py | 21 +- gramps/gui/views/treemodels/lru.py | 7 +- gramps/gui/views/treemodels/mediamodel.py | 26 +- gramps/gui/views/treemodels/notemodel.py | 31 +- gramps/gui/views/treemodels/peoplemodel.py | 339 ++++++++++-------- gramps/gui/views/treemodels/placemodel.py | 42 ++- gramps/gui/views/treemodels/repomodel.py | 20 +- gramps/gui/views/treemodels/sourcemodel.py | 31 +- gramps/gui/views/treemodels/treebasemodel.py | 136 ++++--- 14 files changed, 679 insertions(+), 399 deletions(-) create mode 100644 gramps/gui/views/treemodels/basemodel.py diff --git a/gramps/gen/config.py b/gramps/gen/config.py index ce982484b..2733e4999 100644 --- a/gramps/gen/config.py +++ b/gramps/gen/config.py @@ -269,6 +269,7 @@ register('interface.url-width', 600) register('interface.view', True) register('interface.width', 775) register('interface.surname-box-height', 150) +register('interface.treemodel-cache-size', 1000) register('paths.recent-export-dir', '') register('paths.recent-file', '') diff --git a/gramps/gui/views/treemodels/basemodel.py b/gramps/gui/views/treemodels/basemodel.py new file mode 100644 index 000000000..598178206 --- /dev/null +++ b/gramps/gui/views/treemodels/basemodel.py @@ -0,0 +1,101 @@ +# +# Gramps - a GTK+/GNOME based genealogy program +# +# Copyright (C) 2000-2006 Donald N. Allingham +# Copyright (C) 2009 Benny Malengier +# Copyright (C) 2010 Nick Hall +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# + +#------------------------------------------------------------------------- +# +# GRAMPS modules +# +#------------------------------------------------------------------------- +from .lru import LRU +from gramps.gen.config import config + +class BaseModel(object): + + # LRU cache size + _CACHE_SIZE = config.get('interface.treemodel-cache-size') + + def __init__(self): + self.lru_data = LRU(BaseModel._CACHE_SIZE) + self.lru_path = LRU(BaseModel._CACHE_SIZE) + + def destroy(self): + """ + Destroy the items in memory. + """ + self.lru_data = None + self.lru_path = None + + def clear_cache(self, handle=None): + """ + Clear the LRU cache. Always clear lru_path, because paths may have + changed. + """ + if handle: + if handle in self.lru_data: + del self.lru_data[handle] + else: + self.lru_data.clear() + # Invalidates all paths + self.lru_path.clear() + + def get_cached_value(self, handle, col): + """ + Get the value of a "col". col may be a number (position in a model) + or a name (special value used by view). + """ + if handle in self.lru_data and col in self.lru_data[handle]: + #print("hit", handle, col) + return (True, self.lru_data[handle][col]) + #print("MISS", handle, col) + return (False, None) + + def set_cached_value(self, handle, col, data): + """ + Set the data associated with handle + col. + """ + if not self._in_build: + if self.lru_data.count > 0: + if handle not in self.lru_data: + self.lru_data[handle] = {} + self.lru_data[handle][col] = data + + ## Cached Path's for TreeView: + def get_cached_path(self, handle): + """ + Saves the Gtk iter path. + """ + if handle in self.lru_path: + return (True, self.lru_path[handle]) + return (False, None) + + def set_cached_path(self, handle, path): + """ + Set the Gtk iter path value. + """ + if not self._in_build: + self.lru_path[handle] = path + + def clear_path_cache(self): + """ + Clear path cache for all. + """ + self.lru_path.clear() diff --git a/gramps/gui/views/treemodels/citationbasemodel.py b/gramps/gui/views/treemodels/citationbasemodel.py index 1c47100ed..372f7347b 100644 --- a/gramps/gui/views/treemodels/citationbasemodel.py +++ b/gramps/gui/views/treemodels/citationbasemodel.py @@ -135,15 +135,18 @@ class CitationBaseModel(object): """ Return the tag color. """ - tag_color = "#000000000000" - tag_priority = None - for handle in data[COLUMN_TAGS]: - tag = self.db.get_tag_from_handle(handle) - if tag: + tag_handle = data[0] + cached, tag_color = self.get_cached_value(tag_handle, "TAG_COLOR") + if not cached: + tag_color = "#000000000000" + tag_priority = None + for handle in data[COLUMN_TAGS]: + tag = self.db.get_tag_from_handle(handle) this_priority = tag.get_priority() if tag_priority is None or this_priority < tag_priority: tag_color = tag.get_color() tag_priority = this_priority + self.set_cached_value(tag_handle, "TAG_COLOR", tag_color) return tag_color def citation_change(self, data): @@ -157,72 +160,104 @@ class CitationBaseModel(object): def citation_src_title(self, data): source_handle = data[COLUMN_SOURCE] - try: - source = self.db.get_source_from_handle(source_handle) - return str(source.get_title()) - except: - return '' + cached, value = self.get_cached_value(source_handle, "SRC_TITLE") + if not cached: + try: + source = self.db.get_source_from_handle(source_handle) + value = str(source.get_title()) + except: + value = '' + self.set_cached_value(source_handle, "SRC_TITLE", value) + return value def citation_src_id(self, data): source_handle = data[COLUMN_SOURCE] - try: - source = self.db.get_source_from_handle(source_handle) - return str(source.gramps_id) - except: - return '' + cached, value = self.get_cached_value(source_handle, "SRC_ID") + if not cached: + try: + source = self.db.get_source_from_handle(source_handle) + value = str(source.gramps_id) + except: + value = '' + self.set_cached_value(source_handle, "SRC_ID", value) + return value def citation_src_auth(self, data): source_handle = data[COLUMN_SOURCE] - try: - source = self.db.get_source_from_handle(source_handle) - return str(source.get_author()) - except: - return '' + cached, value = self.get_cached_value(source_handle, "SRC_AUTH") + if not cached: + try: + source = self.db.get_source_from_handle(source_handle) + value = str(source.get_author()) + except: + value = '' + self.set_cached_value(source_handle, "SRC_AUTH", value) + return value def citation_src_abbr(self, data): source_handle = data[COLUMN_SOURCE] - try: - source = self.db.get_source_from_handle(source_handle) - return str(source.get_abbreviation()) - except: - return '' + cached, value = self.get_cached_value(source_handle, "SRC_ABBR") + if not cached: + try: + source = self.db.get_source_from_handle(source_handle) + value = str(source.get_abbreviation()) + except: + value = '' + self.set_cached_value(source_handle, "SRC_ABBR", value) + return value def citation_src_pinfo(self, data): source_handle = data[COLUMN_SOURCE] - try: - source = self.db.get_source_from_handle(source_handle) - return str(source.get_publication_info()) - except: - return '' + cached, value = self.get_cached_value(source_handle, "SRC_PINFO") + if not cached: + try: + source = self.db.get_source_from_handle(source_handle) + value = str(source.get_publication_info()) + except: + value = '' + self.set_cached_value(source_handle, "SRC_PINFO", value) + return value def citation_src_private(self, data): source_handle = data[COLUMN_SOURCE] - try: - source = self.db.get_source_from_handle(source_handle) - if source.get_privacy(): - return 'gramps-lock' - else: - # There is a problem returning None here. - return '' - except: - return '' + cached, value = self.get_cached_value(source_handle, "SRC_PRIVATE") + if not cached: + try: + source = self.db.get_source_from_handle(source_handle) + if source.get_privacy(): + value = 'gramps-lock' + else: + # There is a problem returning None here. + value = '' + except: + value = '' + self.set_cached_value(source_handle, "SRC_PRIVATE", value) + return value def citation_src_tags(self, data): source_handle = data[COLUMN_SOURCE] - try: - source = self.db.get_source_from_handle(source_handle) - tag_list = list(map(self.get_tag_name, source.get_tag_list())) - return ', '.join(sorted(tag_list, key=glocale.sort_key)) - except: - return '' + cached, value = self.get_cached_value(source_handle, "SRC_TAGS") + if not cached: + try: + source = self.db.get_source_from_handle(source_handle) + tag_list = list(map(self.get_tag_name, source.get_tag_list())) + value = ', '.join(sorted(tag_list, key=glocale.sort_key)) + except: + value = '' + self.set_cached_value(source_handle, "SRC_TAGS", value) + return value def citation_src_chan(self, data): source_handle = data[COLUMN_SOURCE] - try: - source = self.db.get_source_from_handle(source_handle) - return format_time(source.change) - except: - return '' + cached, value = self.get_cached_value(source_handle, "SRC_CHAN") + if not cached: + try: + source = self.db.get_source_from_handle(source_handle) + value = format_time(source.change) + except: + value = '' + self.set_cached_value(source_handle, "SRC_CHAN", value) + return value # Fields access when 'data' is a Source @@ -259,15 +294,18 @@ class CitationBaseModel(object): """ Return the tag color. """ - tag_color = "#000000000000" - tag_priority = None - for handle in data[COLUMN2_TAGS]: - tag = self.db.get_tag_from_handle(handle) - if tag: + tag_handle = data[0] + cached, tag_color = self.get_cached_value(tag_handle, "TAG_COLOR") + if not cached: + tag_color = "#000000000000" + tag_priority = None + for handle in data[COLUMN2_TAGS]: + tag = self.db.get_tag_from_handle(handle) this_priority = tag.get_priority() if tag_priority is None or this_priority < tag_priority: tag_color = tag.get_color() tag_priority = this_priority + self.set_cached_value(tag_handle, "TAG_COLOR", tag_color) return tag_color def source_src_chan(self, data): @@ -284,4 +322,8 @@ class CitationBaseModel(object): """ Return the tag name from the given tag handle. """ - return self.db.get_tag_from_handle(tag_handle).get_name() + cached, value = self.get_cached_value(tag_handle, "TAG_NAME") + if not cached: + value = self.db.get_tag_from_handle(tag_handle).get_name() + self.set_cached_value(tag_handle, "TAG_NAME", value) + return value diff --git a/gramps/gui/views/treemodels/eventmodel.py b/gramps/gui/views/treemodels/eventmodel.py index 6f57dfb66..b3597b274 100644 --- a/gramps/gui/views/treemodels/eventmodel.py +++ b/gramps/gui/views/treemodels/eventmodel.py @@ -49,7 +49,7 @@ from gramps.gen.const import GRAMPS_LOCALE as glocale #------------------------------------------------------------------------- # -# COLUMN constants +# Positions in raw data structure # #------------------------------------------------------------------------- COLUMN_HANDLE = 0 @@ -75,7 +75,7 @@ class EventModel(FlatBaseModel): skip=set(), sort_map=None): self.gen_cursor = db.get_event_cursor self.map = db.get_raw_event_data - + self.fmap = [ self.column_description, self.column_id, @@ -127,13 +127,22 @@ class EventModel(FlatBaseModel): return data[COLUMN_DESCRIPTION] def column_participant(self,data): - return get_participant_from_event(self.db, data[COLUMN_HANDLE]) + handle = data[0] + cached, value = self.get_cached_value(handle, "PARTICIPANT") + if not cached: + value = get_participant_from_event(self.db, data[COLUMN_HANDLE]) + self.set_cached_value(handle, "PARTICIPANT", value) + return value def column_place(self,data): if data[COLUMN_PLACE]: - event = Event() - event.unserialize(data) - return place_displayer.display_event(self.db, event) + cached, value = self.get_cached_value(data[0], "PLACE") + if not cached: + event = Event() + event.unserialize(data) + value = place_displayer.display_event(self.db, event) + self.set_cached_value(data[0], "PLACE", value) + return value else: return '' @@ -185,21 +194,29 @@ class EventModel(FlatBaseModel): """ Return the tag name from the given tag handle. """ - return self.db.get_tag_from_handle(tag_handle).get_name() + # TAG_NAME isn't a column, but we cache it + cached, value = self.get_cached_value(tag_handle, "TAG_NAME") + if not cached: + value = self.db.get_tag_from_handle(tag_handle).get_name() + self.set_cached_value(tag_handle, "TAG_NAME", value) + return value def column_tag_color(self, data): """ Return the tag color. """ - tag_color = "#000000000000" - tag_priority = None - for handle in data[COLUMN_TAGS]: - tag = self.db.get_tag_from_handle(handle) - if tag: + tag_handle = data[0] + cached, tag_color = self.get_cached_value(tag_handle, "TAG_COLOR") + if not cached: + tag_color = "#000000000000" + tag_priority = None + for handle in data[COLUMN_TAGS]: + tag = self.db.get_tag_from_handle(handle) this_priority = tag.get_priority() if tag_priority is None or this_priority < tag_priority: tag_color = tag.get_color() tag_priority = this_priority + self.set_cached_value(tag_handle, "TAG_COLOR", tag_color) return tag_color def column_tags(self, data): diff --git a/gramps/gui/views/treemodels/familymodel.py b/gramps/gui/views/treemodels/familymodel.py index 89ff23451..6d8a24e06 100644 --- a/gramps/gui/views/treemodels/familymodel.py +++ b/gramps/gui/views/treemodels/familymodel.py @@ -106,56 +106,86 @@ class FamilyModel(FlatBaseModel): return len(self.fmap)+1 def column_father(self, data): - if data[2]: - person = self.db.get_person_from_handle(data[2]) - return name_displayer.display_name(person.primary_name) - else: - return "" + handle = data[0] + cached, value = self.get_cached_value(handle, "FATHER") + if not cached: + if data[2]: + person = self.db.get_person_from_handle(data[2]) + value = name_displayer.display_name(person.primary_name) + else: + value = "" + self.set_cached_value(handle, "FATHER", value) + return value def sort_father(self, data): - if data[2]: - person = self.db.get_person_from_handle(data[2]) - return name_displayer.sorted_name(person.primary_name) - else: - return "" + handle = data[0] + cached, value = self.get_cached_value(handle, "SORT_FATHER") + if not cached: + if data[2]: + person = self.db.get_person_from_handle(data[2]) + value = name_displayer.sorted_name(person.primary_name) + else: + value = "" + self.set_cached_value(handle, "SORT_FATHER", value) + return value def column_mother(self, data): - if data[3]: - person = self.db.get_person_from_handle(data[3]) - return name_displayer.display_name(person.primary_name) - else: - return "" + handle = data[0] + cached, value = self.get_cached_value(handle, "MOTHER") + if not cached: + if data[3]: + person = self.db.get_person_from_handle(data[3]) + value = name_displayer.display_name(person.primary_name) + else: + value = "" + self.set_cached_value(handle, "MOTHER", value) + return value def sort_mother(self, data): - if data[3]: - person = self.db.get_person_from_handle(data[3]) - return name_displayer.sorted_name(person.primary_name) - else: - return "" + handle = data[0] + cached, value = self.get_cached_value(handle, "SORT_MOTHER") + if not cached: + if data[3]: + person = self.db.get_person_from_handle(data[3]) + value = name_displayer.sorted_name(person.primary_name) + else: + value = "" + self.set_cached_value(handle, "SORT_MOTHER", value) + return value def column_type(self, data): return str(FamilyRelType(data[5])) def column_marriage(self, data): - family = self.db.get_family_from_handle(data[0]) - event = get_marriage_or_fallback(self.db, family, "%s") - if event: - if event.date.format: - return event.date.format % displayer.display(event.date) - elif not get_date_valid(event): - return invalid_date_format % displayer.display(event.date) + handle = data[0] + cached, value = self.get_cached_value(handle, "MARRIAGE") + if not cached: + family = self.db.get_family_from_handle(data[0]) + event = get_marriage_or_fallback(self.db, family, "%s") + if event: + if event.date.format: + value = event.date.format % displayer.display(event.date) + elif not get_date_valid(event): + value = invalid_date_format % displayer.display(event.date) + else: + value = "%s" % displayer.display(event.date) else: - return "%s" % displayer.display(event.date) - else: - return '' + value = '' + self.set_cached_value(handle, "MARRIAGE", value) + return value def sort_marriage(self, data): - family = self.db.get_family_from_handle(data[0]) - event = get_marriage_or_fallback(self.db, family) - if event: - return "%09d" % event.date.get_sort_value() - else: - return '' + handle = data[0] + cached, value = self.get_cached_value(handle, "SORT_MARRIAGE") + if not cached: + family = self.db.get_family_from_handle(data[0]) + event = get_marriage_or_fallback(self.db, family) + if event: + value = "%09d" % event.date.get_sort_value() + else: + value = '' + self.set_cached_value(handle, "SORT_MARRIAGE", value) + return value def column_id(self, data): return str(data[1]) @@ -177,20 +207,28 @@ class FamilyModel(FlatBaseModel): """ Return the tag name from the given tag handle. """ - return self.db.get_tag_from_handle(tag_handle).get_name() + cached, value = self.get_cached_value(tag_handle, "TAG_NAME") + if not cached: + value = self.db.get_tag_from_handle(tag_handle).get_name() + self.set_cached_value(tag_handle, "TAG_NAME", value) + return value def column_tag_color(self, data): """ Return the tag color. """ - tag_color = "#000000000000" - tag_priority = None - for handle in data[13]: - tag = self.db.get_tag_from_handle(handle) - this_priority = tag.get_priority() - if tag_priority is None or this_priority < tag_priority: - tag_color = tag.get_color() - tag_priority = this_priority + tag_handle = data[0] + cached, tag_color = self.get_cached_value(tag_handle, "TAG_COLOR") + if not cached: + tag_color = "#000000000000" + tag_priority = None + for handle in data[13]: + tag = self.db.get_tag_from_handle(handle) + this_priority = tag.get_priority() + if tag_priority is None or this_priority < tag_priority: + tag_color = tag.get_color() + tag_priority = this_priority + self.set_cached_value(tag_handle, "TAG_COLOR", tag_color) return tag_color def column_tags(self, data): diff --git a/gramps/gui/views/treemodels/flatbasemodel.py b/gramps/gui/views/treemodels/flatbasemodel.py index 02cd5c5f7..7030309a4 100644 --- a/gramps/gui/views/treemodels/flatbasemodel.py +++ b/gramps/gui/views/treemodels/flatbasemodel.py @@ -73,6 +73,7 @@ from gi.repository import Gtk from gramps.gen.filters import SearchFilter, ExactSearchFilter from gramps.gen.constfunc import conv_to_unicode, handle2internal from gramps.gen.const import GRAMPS_LOCALE as glocale +from .basemodel import BaseModel #------------------------------------------------------------------------- # @@ -442,7 +443,7 @@ class FlatNodeMap(object): # FlatBaseModel # #------------------------------------------------------------------------- -class FlatBaseModel(GObject.GObject, Gtk.TreeModel): +class FlatBaseModel(GObject.GObject, Gtk.TreeModel, BaseModel): """ The base class for all flat treeview models. It keeps a FlatNodeMap, and obtains data from database as needed @@ -454,7 +455,8 @@ class FlatBaseModel(GObject.GObject, Gtk.TreeModel): search=None, skip=set(), sort_map=None): cput = time.clock() - super(FlatBaseModel, self).__init__() + GObject.GObject.__init__(self) + BaseModel.__init__(self) #inheriting classes must set self.map to obtain the data self.prev_handle = None self.prev_data = None @@ -491,6 +493,7 @@ class FlatBaseModel(GObject.GObject, Gtk.TreeModel): """ Unset all elements that prevent garbage collection """ + BaseModel.destroy(self) self.db = None self.sort_func = None if self.node_map: @@ -556,15 +559,6 @@ class FlatBaseModel(GObject.GObject, Gtk.TreeModel): """ return None - def clear_cache(self, handle=None): - """ - If you use a cache, overwrite here so it is cleared when this - method is called (on rebuild) - :param handle: if None, clear entire cache, otherwise clear the handle - entry if present - """ - pass - def sort_keys(self): """ Return the (sort_key, handle) list of all data that can maximally @@ -775,7 +769,10 @@ class FlatBaseModel(GObject.GObject, Gtk.TreeModel): We need this to search in the column in the GUI """ if handle != self.prev_handle: - data = self.map(handle) + cached, data = self.get_cached_value(handle, col) + if not cached: + data = self.map(handle) + self.set_cached_value(handle, col, data) if data is None: #object is no longer present return '' diff --git a/gramps/gui/views/treemodels/lru.py b/gramps/gui/views/treemodels/lru.py index 95a84c2d9..1c93c212e 100644 --- a/gramps/gui/views/treemodels/lru.py +++ b/gramps/gui/views/treemodels/lru.py @@ -39,7 +39,10 @@ class LRU(object): Implementation of a length-limited O(1) LRU cache """ def __init__(self, count): - self.count = max(count, 2) + """ + Set count to 0 or 1 to disable. + """ + self.count = count self.data = {} self.first = None self.last = None @@ -60,6 +63,8 @@ class LRU(object): """ Set the item in the LRU, removing an old entry if needed """ + if self.count <= 1: # Disabled + return if obj in self.data: del self[obj] nobj = Node(self.last, (obj, val)) diff --git a/gramps/gui/views/treemodels/mediamodel.py b/gramps/gui/views/treemodels/mediamodel.py index d93a9cc67..56795e8a7 100644 --- a/gramps/gui/views/treemodels/mediamodel.py +++ b/gramps/gui/views/treemodels/mediamodel.py @@ -174,20 +174,28 @@ class MediaModel(FlatBaseModel): """ Return the tag name from the given tag handle. """ - return self.db.get_tag_from_handle(tag_handle).get_name() + cached, value = self.get_cached_value(tag_handle, "TAG_NAME") + if not cached: + value = self.db.get_tag_from_handle(tag_handle).get_name() + self.set_cached_value(tag_handle, "TAG_NAME", value) + return value def column_tag_color(self, data): """ Return the tag color. """ - tag_color = "#000000000000" - tag_priority = None - for handle in data[11]: - tag = self.db.get_tag_from_handle(handle) - this_priority = tag.get_priority() - if tag_priority is None or this_priority < tag_priority: - tag_color = tag.get_color() - tag_priority = this_priority + tag_handle = data[0] + cached, tag_color = self.get_cached_value(tag_handle, "TAG_COLOR") + if not cached: + tag_color = "#000000000000" + tag_priority = None + for handle in data[11]: + tag = self.db.get_tag_from_handle(handle) + this_priority = tag.get_priority() + if tag_priority is None or this_priority < tag_priority: + tag_color = tag.get_color() + tag_priority = this_priority + self.set_cached_value(tag_handle, "TAG_COLOR", tag_color) return tag_color def column_tags(self, data): diff --git a/gramps/gui/views/treemodels/notemodel.py b/gramps/gui/views/treemodels/notemodel.py index 3b803157d..fe2784124 100644 --- a/gramps/gui/views/treemodels/notemodel.py +++ b/gramps/gui/views/treemodels/notemodel.py @@ -137,22 +137,31 @@ class NoteModel(FlatBaseModel): """ Return the tag name from the given tag handle. """ - return self.db.get_tag_from_handle(tag_handle).get_name() + cached, value = self.get_cached_value(tag_handle, "TAG_NAME") + if not cached: + value = self.db.get_tag_from_handle(tag_handle).get_name() + self.set_cached_value(tag_handle, "TAG_NAME", value) + return value def column_tag_color(self, data): """ Return the tag color. """ - tag_color = "#000000000000" - tag_priority = None - for handle in data[Note.POS_TAGS]: - tag = self.db.get_tag_from_handle(handle) - if tag: - this_priority = tag.get_priority() - if tag_priority is None or this_priority < tag_priority: - tag_color = tag.get_color() - tag_priority = this_priority - return tag_color + tag_handle = data[0] + cached, value = self.get_cached_value(tag_handle, "TAG_COLOR") + if not cached: + tag_color = "#000000000000" + tag_priority = None + for handle in data[Note.POS_TAGS]: + tag = self.db.get_tag_from_handle(handle) + if tag: + this_priority = tag.get_priority() + if tag_priority is None or this_priority < tag_priority: + tag_color = tag.get_color() + tag_priority = this_priority + value = tag_color + self.set_cached_value(tag_handle, "TAG_COLOR", value) + return value def column_tags(self, data): """ diff --git a/gramps/gui/views/treemodels/peoplemodel.py b/gramps/gui/views/treemodels/peoplemodel.py index 44a906a3a..fb2168c6f 100644 --- a/gramps/gui/views/treemodels/peoplemodel.py +++ b/gramps/gui/views/treemodels/peoplemodel.py @@ -38,6 +38,7 @@ from html import escape # #------------------------------------------------------------------------- from gi.repository import Gtk +from gi.repository import GObject #------------------------------------------------------------------------- # @@ -59,15 +60,15 @@ from gramps.gen.lib import (Name, EventRef, EventType, EventRoleType, from gramps.gen.display.name import displayer as name_displayer from gramps.gen.display.place import displayer as place_displayer from gramps.gen.datehandler import format_time, get_date, get_date_valid -from .lru import LRU from .flatbasemodel import FlatBaseModel from .treebasemodel import TreeBaseModel +from .basemodel import BaseModel from gramps.gen.config import config from gramps.gen.const import GRAMPS_LOCALE as glocale #------------------------------------------------------------------------- # -# COLUMN constants +# COLUMN constants; positions in raw data structure # #------------------------------------------------------------------------- COLUMN_ID = 1 @@ -90,19 +91,17 @@ invalid_date_format = config.get('preferences.invalid-date-format') # PeopleBaseModel # #------------------------------------------------------------------------- -class PeopleBaseModel(object): +class PeopleBaseModel(BaseModel): """ Basic Model interface to handle the PersonViews """ _GENDER = [ _('female'), _('male'), _('unknown') ] - # LRU cache size - _CACHE_SIZE = 250 - def __init__(self, db): """ Initialize the model building the initial data """ + BaseModel.__init__(self) self.db = db self.gen_cursor = db.get_person_cursor self.map = db.get_raw_person_data @@ -144,24 +143,16 @@ class PeopleBaseModel(object): self.column_tag_color, ] - #columns are accessed on every mouse over, so it is worthwhile to - #cache columns visible in one screen to avoid expensive database - #lookup of derived values - self.lru_name = LRU(PeopleBaseModel._CACHE_SIZE) - self.lru_spouse = LRU(PeopleBaseModel._CACHE_SIZE) - self.lru_bdate = LRU(PeopleBaseModel._CACHE_SIZE) - self.lru_ddate = LRU(PeopleBaseModel._CACHE_SIZE) - def destroy(self): """ Unset all elements that can prevent garbage collection """ + BaseModel.destroy(self) self.db = None self.gen_cursor = None self.map = None self.fmap = None self.smap = None - self.clear_local_cache() def color_column(self): """ @@ -169,63 +160,38 @@ class PeopleBaseModel(object): """ return 15 - def clear_local_cache(self, handle=None): - """ Clear the LRU cache """ - if handle: - try: - del self.lru_name[handle] - except KeyError: - pass - try: - del self.lru_spouse[handle] - except KeyError: - pass - try: - del self.lru_bdate[handle] - except KeyError: - pass - try: - del self.lru_ddate[handle] - except KeyError: - pass - else: - self.lru_name.clear() - self.lru_spouse.clear() - self.lru_bdate.clear() - self.lru_ddate.clear() - def on_get_n_columns(self): """ Return the number of columns in the model """ return len(self.fmap)+1 def sort_name(self, data): - name = name_displayer.raw_sorted_name(data[COLUMN_NAME]) - # internally we work with utf-8 - if not isinstance(name, str): - name = name.decode('utf-8') + handle = data[0] + cached, name = self.get_cached_value(handle, "SORT_NAME") + if not cached: + name = name_displayer.raw_sorted_name(data[COLUMN_NAME]) + # internally we work with utf-8 + if not isinstance(name, str): + name = name.decode('utf-8') + self.set_cached_value(handle, "SORT_NAME", name) return name def column_name(self, data): handle = data[0] - if handle in self.lru_name: - name = self.lru_name[handle] - else: + cached, name = self.get_cached_value(handle, "NAME") + if not cached: name = name_displayer.raw_display_name(data[COLUMN_NAME]) # internally we work with utf-8 for python 2.7 if not isinstance(name, str): name = name.encode('utf-8') - if not self._in_build: - self.lru_name[handle] = name + self.set_cached_value(handle, "NAME", name) return name def column_spouse(self, data): handle = data[0] - if handle in self.lru_spouse: - value = self.lru_spouse[handle] - else: + cached, value = self.get_cached_value(handle, "SPOUSE") + if not cached: value = self._get_spouse_data(data) - if not self._in_build: - self.lru_spouse[handle] = value + self.set_cached_value(handle, "SPOUSE", value) return value def column_private(self, data): @@ -265,17 +231,19 @@ class PeopleBaseModel(object): def column_birth_day(self, data): handle = data[0] - if handle in self.lru_bdate: - value = self.lru_bdate[handle] - else: + cached, value = self.get_cached_value(handle, "BIRTH_DAY") + if not cached: value = self._get_birth_data(data, False) - if not self._in_build: - self.lru_bdate[handle] = value + self.set_cached_value(handle, "BIRTH_DAY", value) return value def sort_birth_day(self, data): handle = data[0] - return self._get_birth_data(data, True) + cached, value = self.get_cached_value(handle, "SORT_BIRTH_DAY") + if not cached: + value = self._get_birth_data(data, True) + self.set_cached_value(handle, "SORT_BIRTH_DAY", value) + return value def _get_birth_data(self, data, sort_mode): index = data[COLUMN_BIRTH] @@ -320,17 +288,19 @@ class PeopleBaseModel(object): def column_death_day(self, data): handle = data[0] - if handle in self.lru_ddate: - value = self.lru_ddate[handle] - else: + cached, value = self.get_cached_value(handle, "DEATH_DAY") + if not cached: value = self._get_death_data(data, False) - if not self._in_build: - self.lru_ddate[handle] = value + self.set_cached_value(handle, "DEATH_DAY", value) return value def sort_death_day(self, data): handle = data[0] - return self._get_death_data(data, True) + cached, value = self.get_cached_value(handle, "SORT_DEATH_DAY") + if not cached: + value = self._get_death_data(data, True) + self.set_cached_value(handle, "SORT_DEATH_DAY", value) + return value def _get_death_data(self, data, sort_mode): index = data[COLUMN_DEATH] @@ -375,61 +345,86 @@ class PeopleBaseModel(object): return "" def column_birth_place(self, data): - index = data[COLUMN_BIRTH] - if index != -1: - try: - local = data[COLUMN_EVENT][index] - br = EventRef() - br.unserialize(local) - event = self.db.get_event_from_handle(br.ref) - if event: - place_title = place_displayer.display_event(self.db, event) - if place_title: - return escape(place_title) - except: - return '' + handle = data[0] + cached, value = self.get_cached_value(handle, "BIRTH_PLACE") + if cached: + return value + else: + index = data[COLUMN_BIRTH] + if index != -1: + try: + local = data[COLUMN_EVENT][index] + br = EventRef() + br.unserialize(local) + event = self.db.get_event_from_handle(br.ref) + if event: + place_title = place_displayer.display_event(self.db, event) + if place_title: + value = escape(place_title) + self.set_cached_value(handle, "BIRTH_PLACE", value) + return value + except: + value = '' + self.set_cached_value(handle, "BIRTH_PLACE", value) + return value - for event_ref in data[COLUMN_EVENT]: - er = EventRef() - er.unserialize(event_ref) - event = self.db.get_event_from_handle(er.ref) - etype = event.get_type() - if (etype in [EventType.BAPTISM, EventType.CHRISTEN] and - er.get_role() == EventRoleType.PRIMARY): - - place_title = place_displayer.display_event(self.db, event) - if place_title: - return "%s" % escape(place_title) - return "" + for event_ref in data[COLUMN_EVENT]: + er = EventRef() + er.unserialize(event_ref) + event = self.db.get_event_from_handle(er.ref) + etype = event.get_type() + if (etype in [EventType.BAPTISM, EventType.CHRISTEN] and + er.get_role() == EventRoleType.PRIMARY): + place_title = place_displayer.display_event(self.db, event) + if place_title: + value = "%s" % escape(place_title) + self.set_cached_value(handle, "BIRTH_PLACE", value) + return value + value = "" + self.set_cached_value(handle, "BIRTH_PLACE", value) + return value def column_death_place(self, data): - index = data[COLUMN_DEATH] - if index != -1: - try: - local = data[COLUMN_EVENT][index] - dr = EventRef() - dr.unserialize(local) - event = self.db.get_event_from_handle(dr.ref) - if event: - place_title = place_displayer.display_event(self.db, event) - if place_title: - return escape(place_title) - except: - return '' - - for event_ref in data[COLUMN_EVENT]: - er = EventRef() - er.unserialize(event_ref) - event = self.db.get_event_from_handle(er.ref) - etype = event.get_type() - if (etype in [EventType.BURIAL, EventType.CREMATION, - EventType.CAUSE_DEATH] - and er.get_role() == EventRoleType.PRIMARY): + handle = data[0] + cached, value = self.get_cached_value(handle, "DEATH_PLACE") + if cached: + return value + else: + index = data[COLUMN_DEATH] + if index != -1: + try: + local = data[COLUMN_EVENT][index] + dr = EventRef() + dr.unserialize(local) + event = self.db.get_event_from_handle(dr.ref) + if event: + place_title = place_displayer.display_event(self.db, event) + if place_title: + value = escape(place_title) + self.set_cached_value(handle, "DEATH_PLACE", value) + return value + except: + value = '' + self.set_cached_value(handle, "DEATH_PLACE", value) + return value - place_title = place_displayer.display_event(self.db, event) - if place_title: - return "%s" % escape(place_title) - return "" + for event_ref in data[COLUMN_EVENT]: + er = EventRef() + er.unserialize(event_ref) + event = self.db.get_event_from_handle(er.ref) + etype = event.get_type() + if (etype in [EventType.BURIAL, EventType.CREMATION, + EventType.CAUSE_DEATH] + and er.get_role() == EventRoleType.PRIMARY): + + place_title = place_displayer.display_event(self.db, event) + if place_title: + value = "%s" % escape(place_title) + self.set_cached_value(handle, "DEATH_PLACE", value) + return value + value = "" + self.set_cached_value(handle, "DEATH_PLACE", value) + return value def _get_parents_data(self, data): parents = 0 @@ -468,56 +463,110 @@ class PeopleBaseModel(object): return todo def column_parents(self, data): - return str(self._get_parents_data(data)) + handle = data[0] + cached, value = self.get_cached_value(handle, "PARENTS") + if not cached: + value = self._get_parents_data(data) + self.set_cached_value(handle, "PARENTS", value) + return str(value) def sort_parents(self, data): - return '%06d' % self._get_parents_data(data) + handle = data[0] + cached, value = self.get_cached_value(handle, "SORT_PARENTS") + if not cached: + value = self._get_parents_data(data) + self.set_cached_value(handle, "SORT_PARENTS", value) + return '%06d' % value def column_marriages(self, data): - return str(self._get_marriages_data(data)) + handle = data[0] + cached, value = self.get_cached_value(handle, "MARRIAGES") + if not cached: + value = self._get_marriages_data(data) + self.set_cached_value(handle, "MARRIAGES", value) + return str(value) def sort_marriages(self, data): - return '%06d' % self._get_marriages_data(data) + handle = data[0] + cached, value = self.get_cached_value(handle, "SORT_MARRIAGES") + if not cached: + value = self._get_marriages_data(data) + self.set_cached_value(handle, "SORT_MARRIAGES", value) + return '%06d' % value def column_children(self, data): - return str(self._get_children_data(data)) + handle = data[0] + cached, value = self.get_cached_value(handle, "CHILDREN") + if not cached: + value = self._get_children_data(data) + self.set_cached_value(handle, "CHILDREN", value) + return str(value) def sort_children(self, data): - return '%06d' % self._get_children_data(data) + handle = data[0] + cached, value = self.get_cached_value(handle, "SORT_CHILDREN") + if not cached: + value = self._get_children_data(data) + self.set_cached_value(handle, "SORT_CHILDREN", value) + return '%06d' % value def column_todo(self, data): - return str(self._get_todo_data(data)) + handle = data[0] + cached, value = self.get_cached_value(handle, "TODO") + if not cached: + value = self._get_todo_data(data) + self.set_cached_value(handle, "TODO", value) + return str(value) def sort_todo(self, data): - return '%06d' % self._get_todo_data(data) + handle = data[0] + cached, value = self.get_cached_value(handle, "SORT_TODO") + if not cached: + value = self._get_todo_data(data) + self.set_cached_value(handle, "SORT_TODO", value) + return '%06d' % value def get_tag_name(self, tag_handle): """ Return the tag name from the given tag handle. """ - return self.db.get_tag_from_handle(tag_handle).get_name() - + cached, value = self.get_cached_value(tag_handle, "TAG_NAME") + if not cached: + value = self.db.get_tag_from_handle(tag_handle).get_name() + self.set_cached_value(tag_handle, "TAG_NAME", value) + return value + def column_tag_color(self, data): """ Return the tag color. """ - tag_color = "#000000000000" - tag_priority = None - for handle in data[COLUMN_TAGS]: - tag = self.db.get_tag_from_handle(handle) - if tag: - this_priority = tag.get_priority() - if tag_priority is None or this_priority < tag_priority: - tag_color = tag.get_color() - tag_priority = this_priority - return tag_color + tag_handle = data[0] + cached, value = self.get_cached_value(tag_handle, "TAG_COLOR") + if not cached: + tag_color = "#000000000000" + tag_priority = None + for handle in data[COLUMN_TAGS]: + tag = self.db.get_tag_from_handle(handle) + if tag: + this_priority = tag.get_priority() + if tag_priority is None or this_priority < tag_priority: + tag_color = tag.get_color() + tag_priority = this_priority + value = tag_color + self.set_cached_value(tag_handle, "TAG_COLOR", value) + return value def column_tags(self, data): """ Return the sorted list of tags. """ - tag_list = list(map(self.get_tag_name, data[COLUMN_TAGS])) - return ', '.join(sorted(tag_list, key=glocale.sort_key)) + handle = data[0] + cached, value = self.get_cached_value(handle, "TAGS") + if not cached: + tag_list = list(map(self.get_tag_name, data[COLUMN_TAGS])) + value = ', '.join(sorted(tag_list, key=glocale.sort_key)) + self.set_cached_value(handle, "TAGS", value) + return value class PersonListModel(PeopleBaseModel, FlatBaseModel): """ @@ -529,10 +578,6 @@ class PersonListModel(PeopleBaseModel, FlatBaseModel): FlatBaseModel.__init__(self, db, search=search, skip=skip, scol=scol, order=order, sort_map=sort_map) - def clear_cache(self, handle=None): - """ Clear the LRU cache """ - PeopleBaseModel.clear_local_cache(self, handle) - def destroy(self): """ Unset all elements that can prevent garbage collection @@ -546,7 +591,6 @@ class PersonTreeModel(PeopleBaseModel, TreeBaseModel): """ def __init__(self, db, scol=0, order=Gtk.SortType.ASCENDING, search=None, skip=set(), sort_map=None): - PeopleBaseModel.__init__(self, db) TreeBaseModel.__init__(self, db, search=search, skip=skip, scol=scol, order=order, sort_map=sort_map) @@ -564,13 +608,6 @@ class PersonTreeModel(PeopleBaseModel, TreeBaseModel): """ self.number_items = self.db.get_number_of_people - def clear_cache(self, handle=None): - """ Clear the LRU cache - overwrite of base methods - """ - TreeBaseModel.clear_cache(self, handle) - PeopleBaseModel.clear_local_cache(self, handle) - def get_tree_levels(self): """ Return the headings of the levels in the hierarchy. diff --git a/gramps/gui/views/treemodels/placemodel.py b/gramps/gui/views/treemodels/placemodel.py index f8c786c43..c7eedd251 100644 --- a/gramps/gui/views/treemodels/placemodel.py +++ b/gramps/gui/views/treemodels/placemodel.py @@ -116,9 +116,14 @@ class PlaceBaseModel(object): return len(self.fmap)+1 def column_title(self, data): - place = Place() - place.unserialize(data) - return place_displayer.display(self.db, place) + handle = data[0] + cached, value = self.get_cached_value(handle, "PLACE") + if not cached: + place = Place() + place.unserialize(data) + value = place_displayer.display(self.db, place) + self.set_cached_value(handle, "PLACE", value) + return value def column_name(self, data): return str(data[6][0]) @@ -181,22 +186,31 @@ class PlaceBaseModel(object): """ Return the tag name from the given tag handle. """ - return self.db.get_tag_from_handle(tag_handle).get_name() + cached, value = self.get_cached_value(tag_handle, "TAG_NAME") + if not cached: + value = self.db.get_tag_from_handle(tag_handle).get_name() + self.set_cached_value(tag_handle, "TAG_NAME", value) + return value def column_tag_color(self, data): """ Return the tag color. """ - tag_color = "#000000000000" - tag_priority = None - for handle in data[16]: - tag = self.db.get_tag_from_handle(handle) - if tag: - this_priority = tag.get_priority() - if tag_priority is None or this_priority < tag_priority: - tag_color = tag.get_color() - tag_priority = this_priority - return tag_color + tag_handle = data[0] + cached, value = self.get_cached_value(tag_handle, "TAG_COLOR") + if not cached: + tag_color = "#000000000000" + tag_priority = None + for handle in data[16]: + tag = self.db.get_tag_from_handle(handle) + if tag: + this_priority = tag.get_priority() + if tag_priority is None or this_priority < tag_priority: + tag_color = tag.get_color() + tag_priority = this_priority + value = tag_color + self.set_cached_value(tag_handle, "TAG_COLOR", value) + return value def column_tags(self, data): """ diff --git a/gramps/gui/views/treemodels/repomodel.py b/gramps/gui/views/treemodels/repomodel.py index bd9e4d5a2..4ebfc05a5 100644 --- a/gramps/gui/views/treemodels/repomodel.py +++ b/gramps/gui/views/treemodels/repomodel.py @@ -239,21 +239,29 @@ class RepositoryModel(FlatBaseModel): """ Return the tag name from the given tag handle. """ - return self.db.get_tag_from_handle(tag_handle).get_name() + # TAG_NAME isn't a column, but we cache it + cached, value = self.get_cached_value(tag_handle, "TAG_NAME") + if not cached: + value = self.db.get_tag_from_handle(tag_handle).get_name() + self.set_cached_value(tag_handle, "TAG_NAME", value) + return value def column_tag_color(self, data): """ Return the tag color. """ - tag_color = "#000000000000" - tag_priority = None - for handle in data[8]: - tag = self.db.get_tag_from_handle(handle) - if tag: + tag_handle = data[0] + cached, tag_color = self.get_cached_value(tag_handle, "TAG_COLOR") + if not cached: + tag_color = "#000000000000" + tag_priority = None + for handle in data[8]: + tag = self.db.get_tag_from_handle(handle) this_priority = tag.get_priority() if tag_priority is None or this_priority < tag_priority: tag_color = tag.get_color() tag_priority = this_priority + self.set_cached_value(tag_handle, "TAG_COLOR", tag_color) return tag_color def column_tags(self, data): diff --git a/gramps/gui/views/treemodels/sourcemodel.py b/gramps/gui/views/treemodels/sourcemodel.py index 07f4a82f6..f445e9e02 100644 --- a/gramps/gui/views/treemodels/sourcemodel.py +++ b/gramps/gui/views/treemodels/sourcemodel.py @@ -130,22 +130,31 @@ class SourceModel(FlatBaseModel): """ Return the tag name from the given tag handle. """ - return self.db.get_tag_from_handle(tag_handle).get_name() + cached, value = self.get_cached_value(tag_handle, "TAG_NAME") + if not cached: + value = self.db.get_tag_from_handle(tag_handle).get_name() + self.set_cached_value(tag_handle, "TAG_NAME", value) + return value def column_tag_color(self, data): """ Return the tag color. """ - tag_color = "#000000000000" - tag_priority = None - for handle in data[11]: - tag = self.db.get_tag_from_handle(handle) - if tag: - this_priority = tag.get_priority() - if tag_priority is None or this_priority < tag_priority: - tag_color = tag.get_color() - tag_priority = this_priority - return tag_color + tag_handle = data[0] + cached, value = self.get_cached_value(tag_handle, "TAG_COLOR") + if not cached: + tag_color = "#000000000000" + tag_priority = None + for handle in data[11]: + tag = self.db.get_tag_from_handle(handle) + if tag: + this_priority = tag.get_priority() + if tag_priority is None or this_priority < tag_priority: + tag_color = tag.get_color() + tag_priority = this_priority + value = tag_color + self.set_cached_value(tag_handle, "TAG_COLOR", value) + return value def column_tags(self, data): """ diff --git a/gramps/gui/views/treemodels/treebasemodel.py b/gramps/gui/views/treemodels/treebasemodel.py index 8abfbae35..4dae0d559 100644 --- a/gramps/gui/views/treemodels/treebasemodel.py +++ b/gramps/gui/views/treemodels/treebasemodel.py @@ -53,9 +53,9 @@ from gi.repository import Gtk from gramps.gen.const import GRAMPS_LOCALE as glocale _ = glocale.translation.gettext import gramps.gui.widgets.progressdialog as progressdlg -from .lru import LRU from bisect import bisect_right from gramps.gen.filters import SearchFilter, ExactSearchFilter +from .basemodel import BaseModel #------------------------------------------------------------------------- # @@ -231,7 +231,7 @@ class NodeMap(object): # TreeBaseModel # #------------------------------------------------------------------------- -class TreeBaseModel(GObject.GObject, Gtk.TreeModel): +class TreeBaseModel(GObject.GObject, Gtk.TreeModel, BaseModel): """ The base class for all hierarchical treeview models. The model defines the mapping between a unique node and a path. Paths are defined by a tuple. @@ -274,9 +274,6 @@ class TreeBaseModel(GObject.GObject, Gtk.TreeModel): secondary object type. """ - # LRU cache size - _CACHE_SIZE = 250 - def __init__(self, db, search=None, skip=set(), scol=0, order=Gtk.SortType.ASCENDING, sort_map=None, @@ -284,7 +281,8 @@ class TreeBaseModel(GObject.GObject, Gtk.TreeModel): group_can_have_handle = False, has_secondary=False): cput = time.clock() - super(TreeBaseModel, self).__init__() + GObject.GObject.__init__(self) + BaseModel.__init__(self) #We create a stamp to recognize invalid iterators. From the docs: #Set the stamp to be equal to your model's stamp, to mark the #iterator as valid. When your model's structure changes, you should @@ -332,8 +330,6 @@ class TreeBaseModel(GObject.GObject, Gtk.TreeModel): self._in_build = False - self.lru_data = LRU(TreeBaseModel._CACHE_SIZE) - self.__total = 0 self.__displayed = 0 @@ -350,6 +346,7 @@ class TreeBaseModel(GObject.GObject, Gtk.TreeModel): """ Unset all elements that prevent garbage collection """ + BaseModel.destroy(self) self.db = None self.sort_func = None if self.has_secondary: @@ -364,8 +361,6 @@ class TreeBaseModel(GObject.GObject, Gtk.TreeModel): self.search2 = None self.current_filter = None self.current_filter2 = None - self.clear_cache() - self.lru_data = None def _set_base_data(self): """ @@ -410,22 +405,11 @@ class TreeBaseModel(GObject.GObject, Gtk.TreeModel): """ return None - def clear_cache(self, handle=None): - """ - Clear the LRU cache. - """ - if handle: - try: - del self.lru_data[handle] - except KeyError: - pass - else: - self.lru_data.clear() - def clear(self): """ Clear the data map. """ + self.clear_cache() self.tree.clear() self.handle2node.clear() self.stamp += 1 @@ -600,49 +584,24 @@ class TreeBaseModel(GObject.GObject, Gtk.TreeModel): status = progressdlg.LongOpStatus(msg=_("Building View"), total_steps=3, interval=1) pmon.add_op(status) - status_ppl = progressdlg.LongOpStatus(msg=_("Obtaining all rows"), + status_ppl = progressdlg.LongOpStatus(msg=_("Loading items..."), total_steps=items, interval=items//10) pmon.add_op(status_ppl) self.__total += items - def beat(key): - status_ppl.heartbeat() - # for python3 this returns a byte object, so conversion needed - if not isinstance(key, str): - key = key.decode('utf-8') - return key - with gen_cursor() as cursor: - handle_list = [beat(key) for key, data in cursor] + for handle, data in cursor: + if not isinstance(handle, str): + handle = handle.decode('utf-8') + status_ppl.heartbeat() + if not handle in skip: + if not dfilter or dfilter.apply(self.db, [handle]): + add_func(handle, data) + self.__displayed += 1 status_ppl.end() - status.heartbeat() - - if dfilter: - _LOG.debug("rebuild filter %s" % dfilter) - _LOG.debug(" list before filter %s" % handle_list) - status_filter = progressdlg.LongOpStatus(msg=_("Applying filter"), - total_steps=items, interval=items//10) - pmon.add_op(status_filter) - handle_list = dfilter.apply(self.db, handle_list, - cb_progress=status_filter.heartbeat) - _LOG.debug(" list after filter %s" % handle_list) - status_filter.end() - status.heartbeat() - - todisplay = len(handle_list) - status_col = progressdlg.LongOpStatus(msg=_("Constructing column data"), - total_steps=todisplay, interval=todisplay//10) - pmon.add_op(status_col) - for handle in handle_list: - status_col.heartbeat() - data = data_map(handle) - if not handle in skip: - add_func(handle, data) - self.__displayed += 1 - status_col.end() status.end() - + def add_node(self, parent, child, sortkey, handle, add_parent=True, secondary=False): """ @@ -660,6 +619,7 @@ class TreeBaseModel(GObject.GObject, Gtk.TreeModel): add_parent Bool, if True, check if parent is present, if not add the parent as a top group with no handle """ + self.clear_path_cache() if add_parent and not (parent in self.tree): #add parent to self.tree as a node with no handle, as the first #group level @@ -710,6 +670,7 @@ class TreeBaseModel(GObject.GObject, Gtk.TreeModel): """ Remove a node from the map. """ + self.clear_path_cache() if node.children: del self.handle2node[node.handle] node.set_handle(None) @@ -736,6 +697,7 @@ class TreeBaseModel(GObject.GObject, Gtk.TreeModel): rows_reordered, so to propagate the change to the view, you need to reattach the model to the view. """ + self.clear_path_cache() self.__reverse = not self.__reverse top_node = self.tree[None] self._reverse_level(top_node) @@ -771,16 +733,17 @@ class TreeBaseModel(GObject.GObject, Gtk.TreeModel): def add_row(self, handle, data): """ - Add a row to the model. In general this will add more then one node by + Add a row to the model. In general this will add more than one node by using the add_node method. """ - raise NotImplementedError + self.clear_path_cache() def add_row_by_handle(self, handle): """ Add a row to the model. """ assert isinstance(handle, str) + self.clear_path_cache() if self._get_node(handle) is not None: return # row already exists cput = time.clock() @@ -805,11 +768,11 @@ class TreeBaseModel(GObject.GObject, Gtk.TreeModel): """ assert isinstance(handle, str) cput = time.clock() + self.clear_cache(handle) node = self._get_node(handle) if node is None: return # row not currently displayed - self.clear_cache(handle) parent = self.nodemap.node(node.parent) self.remove_node(node) @@ -835,6 +798,7 @@ class TreeBaseModel(GObject.GObject, Gtk.TreeModel): Update a row in the model. """ assert isinstance(handle, str) + self.clear_cache(handle) if self._get_node(handle) is None: return # row not currently displayed @@ -947,7 +911,9 @@ class TreeBaseModel(GObject.GObject, Gtk.TreeModel): val = self._get_value(node.handle, col, node.secondary) #GTK 3 should convert unicode objects automatically, but this # gives wrong column values, so convert for python 2.7 - if not isinstance(val, str): + if val is None: + return '' + elif not isinstance(val, str): return val.encode('utf-8') else: return val @@ -958,17 +924,19 @@ class TreeBaseModel(GObject.GObject, Gtk.TreeModel): """ if secondary is None: raise NotImplementedError + + cached, data = self.get_cached_value(handle, col) - if handle in self.lru_data: - data = self.lru_data[handle] - else: + if not cached: if not secondary: data = self.map(handle) else: data = self.map2(handle) - if not self._in_build: - self.lru_data[handle] = data + if store_cache: + self.set_cached_value(handle, col, data) + if data is None: + return '' if not secondary: # None is used to indicate this column has no data if self.fmap[col] is None: @@ -994,7 +962,13 @@ class TreeBaseModel(GObject.GObject, Gtk.TreeModel): pathlist = path.get_indices() for index in pathlist: _index = (-index - 1) if self.__reverse else index - node = self.nodemap.node(node.children[_index][1]) + try: + if len(node.children[_index]) > 0: + node = self.nodemap.node(node.children[_index][1]) + else: + return False, Gtk.TreeIter() + except IndexError: + return False, Gtk.TreeIter() return True, self._get_iter(node) def get_node_from_iter(self, iter): @@ -1002,13 +976,16 @@ class TreeBaseModel(GObject.GObject, Gtk.TreeModel): return self.nodemap.node(iter.user_data) else: print ('Problem', iter, iter.user_data) - raise NotImplementedError return None def do_get_path(self, iter): """ Returns a path from a given node. """ + cached, path = self.get_cached_path(iter.user_data) + if cached: + (treepath, pathtup) = path + return treepath node = self.get_node_from_iter(iter) pathlist = [] while node.parent is not None: @@ -1017,16 +994,33 @@ class TreeBaseModel(GObject.GObject, Gtk.TreeModel): while node is not None: # Step backwards nodeid = node.next if self.__reverse else node.prev + # Let's see if sibling is cached: + cached, sib_path = self.get_cached_path(nodeid) + if cached: + (sib_treepath, sib_pathtup) = sib_path + # Does it have an actual path? + if sib_pathtup: + # Compute path to here from sibling: + # parent_path + sib_path + offset + newtup = (sib_pathtup[:-1] + + (sib_pathtup[-1] + index + 2, ) + + tuple(reversed(pathlist))) + #print("computed path:", iter.user_data, newtup) + retval = Gtk.TreePath(newtup) + self.set_cached_path(iter.user_data, (retval, newtup)) + return retval node = nodeid and self.nodemap.node(nodeid) index += 1 pathlist.append(index) node = parent - if pathlist: pathlist.reverse() - return Gtk.TreePath(tuple(pathlist)) + #print("actual path :", iter.user_data, tuple(pathlist)) + retval = Gtk.TreePath(tuple(pathlist)) else: - return None + retval = None + self.set_cached_path(iter.user_data, (retval, tuple(pathlist) if pathlist else None)) + return retval def do_iter_next(self, iter): """