Further optimization in the flatbasemodel:

* keep list of all possible keys in memory so database is no longer touched 
     for the searches. ==> a search is faster
   * bug fixes to previous patch set: show total and displayed number correctly


svn: r12726

src/DataViews/EventView.py

@@ -109,6 +109,9 @@ class EventView(PageView.ListView):
         Config.client.notify_add("/apps/gramps/interface/filter",
                                  self.filter_toggle)
 
+    def column_ord_setfunc(self, clist):
+        self.dbstate.db.set_event_column_order(clist)
+
     def get_bookmarks(self):
         """
         Return the bookmark object
@@ -209,10 +212,6 @@ class EventView(PageView.ListView):
             EventView.COLUMN_NAMES,
             self.set_column_order)
 
-    def set_column_order(self, clist):
-        self.dbstate.db.set_event_column_order(clist)
-        self.build_columns()
-
     def add(self, obj):
         try:
             EditEvent(self.dbstate, self.uistate, [], gen.lib.Event())

src/DataViews/FamilyList.py

@@ -97,6 +97,9 @@ class FamilyListView(PageView.ListView):
         Config.client.notify_add("/apps/gramps/interface/filter",
                                  self.filter_toggle)
 
+    def column_ord_setfunc(self, clist):
+        self.dbstate.db.self.dbstate.db.set_family_list_column_order(clist)
+
     def column_order(self):
         return self.dbstate.db.get_family_list_column_order()
 
@@ -110,10 +113,6 @@ class FamilyListView(PageView.ListView):
             FamilyListView.COLUMN_NAMES,
             self.set_column_order)
 
-    def set_column_order(self, clist):
-        self.dbstate.db.set_family_list_column_order(clist)
-        self.build_columns()
-
     def get_stock(self):
         return 'gramps-family'
 

src/DataViews/MediaView.py

@@ -118,6 +118,9 @@ class MediaView(PageView.ListView):
         Config.client.notify_add("/apps/gramps/interface/filter", 
                                  self.filter_toggle)
 
+    def column_ord_setfunc(self, clist):
+        self.dbstate.db.set_media_column_order(clist)
+
     def _set_dnd(self):
         """
         Set up drag-n-drop. The source and destionation are set by calling .target()
@@ -242,13 +245,6 @@ class MediaView(PageView.ListView):
             MediaView.COLUMN_NAMES, 
             self.set_column_order)
 
-    def set_column_order(self, clist):
-        """
-        Saves the column order to the database
-        """
-        self.dbstate.db.set_media_column_order(clist)
-        self.build_columns()
-
     def column_order(self):
         """
         Get the column order from the database

src/DataViews/NoteView.py

@@ -100,6 +100,9 @@ class NoteView(PageView.ListView):
         Config.client.notify_add("/apps/gramps/interface/filter",
                                  self.filter_toggle)
 
+    def column_ord_setfunc(self, clist):
+        self.dbstate.db.self.dbstate.db.set_note_column_order(clist)
+
     def get_bookmarks(self):
         """
         Return the bookmark object
@@ -190,10 +193,6 @@ class NoteView(PageView.ListView):
             NoteView.COLUMN_NAMES,
             self.set_column_order)
 
-    def set_column_order(self, clist):
-        self.dbstate.db.set_note_column_order(clist)
-        self.build_columns()
-
     def add(self, obj):
         try:
             EditNote(self.dbstate, self.uistate, [], Note())

src/DataViews/PlaceView.py

@@ -120,6 +120,9 @@ class PlaceView(PageView.ListView):
         Config.client.notify_add("/apps/gramps/interface/filter",
                                  self.filter_toggle)
 
+    def column_ord_setfunc(self, clist):
+        self.dbstate.db.set_place_column_order(clist)
+    
     def get_bookmarks(self):
         return self.dbstate.db.get_place_bookmarks()
 
@@ -261,10 +264,6 @@ class PlaceView(PageView.ListView):
             PlaceView.COLUMN_NAMES,
             self.set_column_order)
 
-    def set_column_order(self, clist):
-        self.dbstate.db.set_place_column_order(clist)
-        self.build_columns()
-
     def column_order(self):
         return self.dbstate.db.get_place_column_order()
 

src/DataViews/RepositoryView.py

@@ -110,6 +110,9 @@ class RepositoryView(PageView.ListView):
         Config.client.notify_add("/apps/gramps/interface/filter",
                                  self.filter_toggle)
 
+    def column_ord_setfunc(self, clist):
+        self.dbstate.db.self.dbstate.db.set_repository_column_order(clist)
+
     def get_bookmarks(self):
         return self.dbstate.db.get_repo_bookmarks()
 
@@ -137,10 +140,6 @@ class RepositoryView(PageView.ListView):
             RepositoryView.COLUMN_NAMES,
             self.set_column_order)
 
-    def set_column_order(self, clist):
-        self.dbstate.db.set_repository_column_order(clist)
-        self.build_columns()
-
     def column_order(self):
         return self.dbstate.db.get_repository_column_order()
 

src/DataViews/SourceView.py

@@ -102,6 +102,9 @@ class SourceView(PageView.ListView):
         Config.client.notify_add("/apps/gramps/interface/filter",
                                  self.filter_toggle)
 
+    def column_ord_setfunc(self, clist):
+        self.dbstate.db.set_source_column_order(clist)
+
     def get_bookmarks(self):
         return self.dbstate.db.get_source_bookmarks()
 
@@ -127,10 +130,6 @@ class SourceView(PageView.ListView):
             SourceView.COLUMN_NAMES,
             self.set_column_order)
 
-    def set_column_order(self, clist):
-        self.dbstate.db.set_source_column_order(clist)
-        self.build_columns()
-
     def column_order(self):
         return self.dbstate.db.get_source_column_order()
 

src/Filters/_GenericFilter.py

@@ -112,7 +112,7 @@ class GenericFilter(object):
     def find_from_handle(self, db, handle):
         return db.get_person_from_handle(handle)
 
-    def check_func(self, db, id_list, task, progress=None):
+    def check_func(self, db, id_list, task, progress=None, tupleind=None):
         final_list = []
         
         if id_list is None:
@@ -125,15 +125,19 @@ class GenericFilter(object):
                     if task(db, person) != self.invert:
                         final_list.append(handle)
         else:
-            for handle in id_list:
+            for data in id_list:
+                if tupleind is None:
+                    handle = data
+                else:
+                    handle = data[tupleind]
                 person = self.find_from_handle(db, handle)
                 if progress:
                     progress.step()
                 if task(db, person) != self.invert:
-                    final_list.append(handle)
+                    final_list.append(data)
         return final_list
 
-    def check_and(self, db, id_list, progress=None):
+    def check_and(self, db, id_list, progress=None, tupleind=None):
         final_list = []
         flist = self.flist
 
@@ -148,23 +152,30 @@ class GenericFilter(object):
                     if val != self.invert:
                         final_list.append(handle)
         else:
-            for handle in id_list:
+            for data in id_list:
+                if tupleind is None:
+                    handle = data
+                else:
+                    handle = data[tupleind]
                 person = self.find_from_handle(db, handle)
                 if progress:
                     progress.step()
                 val = all(rule.apply(db, person) for rule in flist)
                 if val != self.invert:
-                    final_list.append(handle)
+                    final_list.append(data)
         return final_list
 
-    def check_or(self, db, id_list, progress=None):
+    def check_or(self, db, id_list, progress=None, tupleind=None):
-        return self.check_func(db, id_list, self.or_test, progress)
+        return self.check_func(db, id_list, self.or_test, progress,
+                                tupleind)
 
-    def check_one(self, db, id_list, progress=None):
+    def check_one(self, db, id_list, progress=None, tupleind=None):
-        return self.check_func(db, id_list, self.one_test, progress)
+        return self.check_func(db, id_list, self.one_test, progress,
+                                tupleind)
 
-    def check_xor(self, db, id_list, progress=None):
+    def check_xor(self, db, id_list, progress=None, tupleind=None):
-        return self.check_func(db, id_list, self.xor_test, progress)
+        return self.check_func(db, id_list, self.xor_test, progress,
+                                tupleind)
 
     def xor_test(self, db, person):
         test = False
@@ -199,11 +210,28 @@ class GenericFilter(object):
 
     # progress is optional. If present it must be an instance of 
     #  gui.utils.ProgressMeter
-    def apply(self, db, id_list=None, progress=None):
+    def apply(self, db, id_list=None, progress=None, tupleind=None):
+        """
+        Apply the filter using db.
+        If id_list given, the handles in id_list are used. If not given
+        a database cursor will be used over all entries.
+        
+        If progress given, it will be used to indicate progress of the
+        Filtering
+        
+        If typleind is given, id_list is supposed to consist of a list of 
+        tuples, with the handle being index tupleind. So 
+        handle_0 = id_list[0][tupleind]
+        
+        :Returns: if id_list given, it is returned with the items that 
+                do not match the filter, filtered out.
+                if id_list not given, all items in the database that 
+                match the filter are returned as a list of handles
+        """
         m = self.get_check_func()
         for rule in self.flist:
             rule.prepare(db)
-        res = m(db, id_list, progress)
+        res = m(db, id_list, progress, tupleind)
         for rule in self.flist:
             rule.reset()
         return res

src/PageView.py

@@ -640,10 +640,12 @@ class ListView(BookMarkView):
         self.renderer = gtk.CellRendererText()
         self.renderer.set_property('ellipsize', pango.ELLIPSIZE_END)
         self.sort_col = 0
+        self.sort_order = gtk.SORT_ASCENDING
         self.columns = []
         self.colinfo = columns
         self.handle_col = handle_col
         self.make_model = make_model
+        self.model = None
         self.signal_map = signal_map
         self.multiple_selection = multiple
         self.generic_filter = None
@@ -734,7 +736,31 @@ class ListView(BookMarkView):
         return True
 
     def column_order(self):
-        assert False
+        """
+        Must be set by children. The method that obtains the column order
+        to be used. Format: see ColumnOrder.
+        """
+        raise NotImplementedError
+
+    def column_ord_setfunc(self, clist):
+        """
+        Must be set by children. The method that stores the column order
+        given by clist (result of ColumnOrder class).
+        """
+        raise NotImplementedError
+
+    def set_column_order(self, clist):
+        """
+        change the order of the columns to that given in clist
+        """
+        self.column_ord_setfunc(clist)
+        #now we need to rebuild the model so it contains correct column info
+        self.dirty = True
+        #make sure we sort on first column. We have no idea where the 
+        # column that was sorted on before is situated now. 
+        self.sort_col = 0
+        self.sort_order = gtk.SORT_ASCENDING
+        self.build_tree()
  
     def build_widget(self):
         """
@@ -838,6 +864,12 @@ class ListView(BookMarkView):
         # disable the inactive flag
         self.inactive = False
 
+    def __display_column_sort(self):
+        for i in xrange(len(self.columns)):
+            enable_sort_flag = (i==self.sort_col)
+            self.columns[i].set_sort_indicator(enable_sort_flag)
+        self.columns[self.sort_col].set_sort_order(self.sort_order)
+
     def column_clicked(self, obj, data):
         cput = time.clock()
         same_col = False
@@ -852,6 +884,7 @@ class ListView(BookMarkView):
                 order = gtk.SORT_DESCENDING
 
         self.sort_col = data
+        self.sort_order = order
         handle = self.first_selected()
 
         if Config.get(Config.FILTER):
@@ -862,18 +895,16 @@ class ListView(BookMarkView):
         if same_col:
             self.model.reverse_order()
         else:
-            self.model = self.make_model(self.dbstate.db, self.sort_col, order, 
+            self.model = self.make_model(self.dbstate.db, self.sort_col, 
+                                         self.sort_order, 
                                          search=search, 
                                          sort_map=self.column_order())
         
         self.list.set_model(self.model)
+        self.__display_column_sort()
 
         if handle:
             self.goto_handle(handle)
-        for i in xrange(len(self.columns)):
-            enable_sort_flag = (i==self.sort_col)
-            self.columns[i].set_sort_indicator(enable_sort_flag)
-        self.columns[self.sort_col].set_sort_order(order)
 
         # set the search column to be the sorted column
         search_col = self.column_order()[data][1]
@@ -915,18 +946,29 @@ class ListView(BookMarkView):
             else:
                 filter_info = (False, self.search_bar.get_value())
 
+            if self.dirty or self.model is None \
+                    or not self.model.node_map.full_srtkey_hndl_map():
                 self.model = self.make_model(self.dbstate.db, self.sort_col, 
-                                         search=filter_info)
+                                             search=filter_info,
-            self.list.set_model(self.model)
+                                             sort_map=self.column_order())
+            else:
+                #the entire data to show is already in memory.
+                #run only the part that determines what to show
+                self.list.set_model(None)
+                self.model.set_search(filter_info)
+                self.model.rebuild_data()
+            
             self.build_columns()
+            self.list.set_model(self.model)
+            self.__display_column_sort()
 
             if const.USE_TIPS and self.model.tooltip_column is not None:
                 self.tooltips = TreeTips.TreeTips(
                     self.list, self.model.tooltip_column, True)
             self.dirty = False
             self.uistate.show_filter_results(self.dbstate, 
-                                             self.model.displayed, 
+                                             self.model.displayed(), 
-                                             self.model.total)
+                                             self.model.total())
             _LOG.debug(self.__class__.__name__ + ' build_tree ' +
                     str(time.clock() - cput) + ' sec')
             
@@ -936,6 +978,7 @@ class ListView(BookMarkView):
     def object_build(self):
         """callback, for if tree must be rebuilt and bookmarks redrawn
         """
+        self.dirty = True
         if self.active:
             self.bookmarks.redraw()
         self.build_tree()
@@ -966,6 +1009,8 @@ class ListView(BookMarkView):
             db.connect(sig, self.signal_map[sig])
         self.bookmarks.update_bookmarks(self.get_bookmarks())
         if self.active:
+            #force rebuild of the model on build of tree
+            self.dirty = True
             self.build_tree()
             self.bookmarks.redraw()
         else:
@@ -978,6 +1023,9 @@ class ListView(BookMarkView):
                 self.model.add_row_by_handle(handle)
             _LOG.debug('   ' + self.__class__.__name__ + ' row_add ' +
                     str(time.clock() - cput) + ' sec')
+            self.uistate.show_filter_results(self.dbstate, 
+                                             self.model.displayed(), 
+                                             self.model.total())
         else:
             self.dirty = True
 
@@ -1000,6 +1048,9 @@ class ListView(BookMarkView):
                 self.model.delete_row_by_handle(handle)
             _LOG.debug('   '  + self.__class__.__name__ + ' row_delete ' +
                     str(time.clock() - cput) + ' sec')
+            self.uistate.show_filter_results(self.dbstate, 
+                                             self.model.displayed(), 
+                                             self.model.total())
         else:
             self.dirty = True
 
@@ -1086,8 +1137,8 @@ class ListView(BookMarkView):
     def change_page(self):
         if self.model:
             self.uistate.show_filter_results(self.dbstate, 
-                                             self.model.displayed, 
+                                             self.model.displayed(), 
-                                             self.model.total)
+                                             self.model.total())
         self.edit_action.set_sensitive(not self.dbstate.db.readonly)
 
     def key_delete(self):

src/gui/views/treemodels/flatbasemodel.py

@@ -56,6 +56,8 @@ from __future__ import with_statement
 import locale
 import logging
 import bisect
+import time
+import copy
 
 _LOG = logging.getLogger(".gui.basetreemodel")
     
@@ -73,7 +75,6 @@ import gtk
 #-------------------------------------------------------------------------
 from Filters import SearchFilter
 import Config
-import time
 
 #-------------------------------------------------------------------------
 #
@@ -98,7 +99,8 @@ class FlatNodeMap(object):
                     ascending, but will begin at back of list if view shows
                     the entries in reverse.
         * index2hndl : list of (srtkey, hndl) tuples. The index gives the 
-                        (srtkey, hndl) it belongs to
+                        (srtkey, hndl) it belongs to. 
+                       This normally is only a part of all possible data 
         * hndl2index : dictionary of *hndl: index* values
         
     The implementation provides a list of (srtkey, hndl) of which the index is
@@ -114,26 +116,56 @@ class FlatNodeMap(object):
         Create a new instance.
         """
         self._index2hndl = []
+        self._fullhndl = self._index2hndl
+        self._identical = True
         self._hndl2index = {}
         self._reverse = False
         self.__corr = (0, 1)
 
-    def set_path_map(self, index2hndllist, reverse=False):
+    def set_path_map(self, index2hndllist, fullhndllist, identical=True,
+                     reverse=False):
         """
         This is the core method to set up the FlatNodeMap
         Input is a list of (srtkey, handle), of which the index is the path
         Calling this method sets the index2hndllist, and creates the hndl2index
         map. 
+        fullhndllist is the entire list of (srtkey, handle) that is possible, 
+        normally index2hndllist is only part of this list as determined by
+        filtering. To avoid memory, if both lists are the same, pass only one
+        list twice and set identical to True.
+        Reverse sets up how the path is determined from the index. If True the
+        first index is the last path
         
         :param index2hndllist: the ascending sorted (sortkey, handle) values 
-                    as they will appear in the flat treeview. 
+                    as they will appear in the flat treeview. This often is 
+                    a subset of all possible data
         :type index2hndllist: a list of (sortkey, handle) tuples
+        :param fullhndllist: the list of all possilbe ascending sorted 
+                    (sortkey, handle) values as they will appear in the flat 
+                     treeview if all data is shown.
+        :type fullhndllist: a list of (sortkey, handle) tuples
+        :param identical: identify if index2hndllist and fullhndllist are the
+                        same list, so only one is kept in memory.
+        :type identical: bool
         """
         self._index2hndl = index2hndllist
         self._hndl2index = {}
+        self._identical = identical
+        if identical:
+            self._fullhndl = self._index2hndl
+        else:
+            self._fullhndl = fullhndllist
         self._reverse = reverse
         self.reverse_order()
     
+    def full_srtkey_hndl_map(self):
+        """
+        The list of all possible (sortkey, handle) tuples. 
+        This is stored in FlatNodeMap so that it would not be needed to 
+        reiterate over the database to obtain all posibilities.
+        """
+        return self._fullhndl
+
     def reverse_order(self):
         """
         This method keeps the index2hndl map, but sets it up the index in 
@@ -177,6 +209,8 @@ class FlatNodeMap(object):
         """
         self._index2hndl = []
         self._hndl2index = {}
+        self._fullhndl = self._index2hndl
+        self._identical = True
 
     def get_path(self, handle):
         """
@@ -241,7 +275,14 @@ class FlatNodeMap(object):
         """
         return len(self._index2hndl)
     
-    def insert(self, srtkey_hndl):
+    def max_rows(self):
+        """
+        Return maximum number of entries that might be present in the 
+        map
+        """
+        return len(self._fullhndl)
+    
+    def insert(self, srtkey_hndl, allkeyonly=False):
         """
         Insert a node. Given is a tuple (sortkey, handle), and this is added
         in the correct place, while the hndl2index map is updated.
@@ -250,8 +291,13 @@ class FlatNodeMap(object):
         :param srtkey_hndl: the (sortkey, handle) tuple that must be inserted
         
         :Returns: path of the row inserted in the treeview
-        :Returns type: integer
+        :Returns type: integer or None
         """
+        if not self._identical:
+            bisect.insort_left(self._fullhndl, srtkey_hndl)
+            if allkeyonly:
+                #key is not part of the view
+                return None
         insert_pos = bisect.bisect_left(self._index2hndl, srtkey_hndl)
         self._index2hndl.insert(insert_pos, srtkey_hndl)
         #make sure the index map is updated
@@ -264,19 +310,34 @@ class FlatNodeMap(object):
             self.__corr = (len(self._index2hndl) - 1, -1)
         return self.real_path(insert_pos)
     
-    def delete(self, handle):
+    def delete(self, srtkey_hndl):
         """
-        Delete the row with handle.
+        Delete the row with the given (sortkey, handle).
         This then rebuilds the hndl2index, subtracting one from each item
         greater than the deleted index.
+        path of deleted row is returned
+        If handle is not present, None is returned
         
-        :param handle: the handle that must be removed
+        :param srtkey_hndl: the (sortkey, handle) tuple that must be inserted
-        :type handle: an object handle
         
         :Returns: path of the row deleted from the treeview
-        :Returns type: integer
+        :Returns type: integer or None
         """
+        #remove it from the full list first
+        if not self._identical:
+            del_pos = bisect.bisect_left(self._fullhndl, srtkey_hndl)
+            #check that indeed this is correct:
+            if not self._fullhndl[del_pos][1] == srtkey_hndl[1]:
+                raise KeyError, 'Handle %s not in list of all handles' %  \
+                                                srtkey_hndl[1]
+            del self._fullhndl[del_pos]
+        #now remove it from the index maps
+        handle = srtkey_hndl[1]
+        try:
             index = self._hndl2index[handle]
+        except KeyError:
+            # key not present in the treeview
+            return None
         del self._index2hndl[index]
         del self._hndl2index[handle]
         #update self.__corr so it remains correct
@@ -318,29 +379,8 @@ class FlatBaseModel(gtk.GenericTreeModel):
         self.sort_col = scol
         self.skip = skip
 
-        self.total = 0
-        self.displayed = 0
-
         self.node_map = FlatNodeMap()
-
+        self.set_search(search)
-        if search:
-            if search[0]:
-                self.search = search[1]
-                self.rebuild_data = self._rebuild_filter
-            else:
-                if search[1]:
-                    # we have search[1] = (index, text_unicode, inversion)
-                    col = search[1][0]
-                    text = search[1][1]
-                    inv = search[1][2]
-                    func = lambda x: self.on_get_value(x, col) or u""
-                    self.search = SearchFilter(func, text, inv)
-                else:
-                    self.search = None
-                self.rebuild_data = self._rebuild_search
-        else:
-            self.search = None
-            self.rebuild_data = self._rebuild_search
             
         self._reverse = (order == gtk.SORT_DESCENDING)
         self.tooltip_column = tooltip_column
@@ -359,6 +399,35 @@ class FlatBaseModel(gtk.GenericTreeModel):
         _LOG.debug(self.__class__.__name__ + ' __init__ ' +
                     str(time.clock() - cput) + ' sec')
 
+    def set_search(self, search):
+        """
+        Change the search function that filters the data in the model. 
+        When this method is called, make sure:
+        # you call self.rebuild_data() to recalculate what should be seen 
+          in the model
+        # you reattach the model to the treeview so that the treeview updates
+          with the new entries
+        """
+        if search:
+            if search[0]:
+                #following is None if no data given in filter sidebar
+                self.search = search[1]
+                self.rebuild_data = self._rebuild_filter
+            else:
+                if search[1]:
+                    # we have search[1] = (index, text_unicode, inversion)
+                    col = search[1][0]
+                    text = search[1][1]
+                    inv = search[1][2]
+                    func = lambda x: self.on_get_value(x, col) or u""
+                    self.search = SearchFilter(func, text, inv)
+                else:
+                    self.search = None
+                self.rebuild_data = self._rebuild_search
+        else:
+            self.search = None
+            self.rebuild_data = self._rebuild_search
+
     def __update_todo(self, client, cnxn_id, entry, data):
         """
         Callback if preferences todo color changes
@@ -377,11 +446,17 @@ class FlatBaseModel(gtk.GenericTreeModel):
         """
         self.complete_color = Config.get(Config.COMPLETE_COLOR)
 
-    def set_sort_column(self, col):
+    def total(self):
         """
-        set sort column to column with index col
+        Total number of items that maximally can be shown
         """
-        self.sort_func = self.smap[col]
+        return self.node_map.max_rows()
+
+    def displayed(self):
+        """
+        Number of items that are currently displayed
+        """
+        return len(self.node_map)
 
     def reverse_order(self):
         """
@@ -397,7 +472,6 @@ class FlatBaseModel(gtk.GenericTreeModel):
         This list is sorted ascending (via localized string sort)
         """
         sort_data = []
-        self.total = 0
         
         with self.gen_cursor() as cursor:    # use cursor as a context manager
             #loop over database and store the sort field, and the handle
@@ -411,45 +485,60 @@ class FlatBaseModel(gtk.GenericTreeModel):
                 #bisect.insort_left(sort_data,
                 #                   (locale.strxfrm(self.sort_func(data)), key))
         sort_data.sort()
-        self.total = len(sort_data)
         return sort_data
 
     def _rebuild_search(self, ignore=None):
         """ function called when view must be build, given a search text
             in the top search bar
         """
-        self.total = 0
         if self.db.is_open():
+            allkeys = self.node_map.full_srtkey_hndl_map()
+            if not allkeys:
+                allkeys = self.sort_keys()
             if self.search and self.search.text:
-                dlist = [h for h  in self.sort_keys()\
+                dlist = [h for h  in allkeys \
-                             if self.search.match(h[1],self.db) and \
+                             if self.search.match(h[1], self.db) and \
                              h[1] not in self.skip and h[1] != ignore]
+                ident = False
+            elif ignore is None and not self.skip:
+                #nothing to remove from the keys present
+                ident = True
+                dlist = allkeys
             else:
-                dlist = [h for h in self.sort_keys() \
+                ident = False
+                dlist = [h for h in allkeys \
                              if h[1] not in self.skip and h[1] != ignore]
-            self.displayed = len(dlist)
+            self.node_map.set_path_map(dlist, allkeys, identical=ident, 
-            self.node_map.set_path_map(dlist, reverse=self._reverse)
+                                       reverse=self._reverse)
         else:
-            self.displayed = 0
             self.node_map.clear_map()
 
     def _rebuild_filter(self, ignore=None):
         """ function called when view must be build, given filter options
             in the filter sidebar
         """
-        self.total = 0
         if self.db.is_open():
+            allkeys = self.node_map.full_srtkey_hndl_map()
+            if not allkeys:
+                allkeys = self.sort_keys()
             if self.search:
+                ident = False
+                if ignore is None:
+                    tmp = copy.copy(allkeys)
+                    dlist = self.search.apply(self.db, tmp, tupleind=1)
+                else:
                     dlist = self.search.apply(self.db, 
-                    [ k for k in self.sort_keys()\
+                                [ k for k in allkeys if k[1] != ignore],
-                        if k[1] != ignore])
+                                tupleind=1)
+            elif ignore is None :
+                ident = True
+                dlist = allkeys
             else:
-                dlist = [ k for k in self.sort_keys() \
+                ident = False
-                              if k[1] != ignore ]
+                dlist = [ k for k in allkeys if k[1] != ignore ]
-            self.displayed = len(dlist)
+            self.node_map.set_path_map(dlist, allkeys, identical=ident, 
-            self.node_map.set_path_map(dlist, reverse=self._reverse)
+                                       reverse=self._reverse)
         else:
-            self.displayed = 0
             self.node_map.clear_map()
         
     def add_row_by_handle(self, handle):
@@ -457,22 +546,28 @@ class FlatBaseModel(gtk.GenericTreeModel):
         Add a row. This is called after object with handle is created.
         Row is only added if search/filter data is such that it must be shown
         """
+        data = self.map(handle)
+        insert_val = (locale.strxfrm(self.sort_func(data)), handle)
         if not self.search or \
                 (self.search and self.search.match(handle, self.db)):
             #row needs to be added to the model
-            data = self.map(handle)
-            insert_val = (locale.strxfrm(self.sort_func(data)), handle)
             insert_path = self.node_map.insert(insert_val)
 
             if insert_path is not None:
                 node = self.get_iter(insert_path)
                 self.row_inserted(insert_path, node)
+        else:
+            self.node_map.insert(insert_val, allkeyonly=True)
 
     def delete_row_by_handle(self, handle):
         """
         Delete a row, called after the object with handle is deleted
         """
-        delete_path = self.node_map.delete(handle)
+        data = self.map(handle)
+        delete_val = (locale.strxfrm(self.sort_func(data)), handle)
+        delete_path = self.node_map.delete(delete_val)
+        #delete_path is an integer from 0 to n-1
+        if delete_path is not None: 
             self.row_deleted(delete_path)
 
     def update_row_by_handle(self, handle):
@@ -481,6 +576,7 @@ class FlatBaseModel(gtk.GenericTreeModel):
         """
         ## TODO: if sort key changes, this is not updated correctly ....
         path = self.node_map.get_path(handle)
+        if path is not None:
             node = self.get_iter(path)
             self.row_changed(path, node)