From cde65a53a61843beed44ea38a3c44e42b148bce6 Mon Sep 17 00:00:00 2001 From: prculley Date: Sun, 22 Jan 2017 15:34:15 -0600 Subject: [PATCH] Improve Addon management performance --- gramps/{plugins => gen}/db/dummydb.py | 0 gramps/gen/dbstate.py | 6 ++-- gramps/gen/plug/_manager.py | 46 ++++++++++++++++----------- gramps/gen/plug/_pluginreg.py | 33 +++++++++++++------ gramps/plugins/db/dummydb.gpr.py | 36 --------------------- 5 files changed, 54 insertions(+), 67 deletions(-) rename gramps/{plugins => gen}/db/dummydb.py (100%) delete mode 100644 gramps/plugins/db/dummydb.gpr.py diff --git a/gramps/plugins/db/dummydb.py b/gramps/gen/db/dummydb.py similarity index 100% rename from gramps/plugins/db/dummydb.py rename to gramps/gen/db/dummydb.py diff --git a/gramps/gen/dbstate.py b/gramps/gen/dbstate.py index 0a80f1df3..36e8af0bf 100644 --- a/gramps/gen/dbstate.py +++ b/gramps/gen/dbstate.py @@ -43,7 +43,7 @@ from .proxy.proxybase import ProxyDbBase from .utils.callback import Callback from .config import config from gramps.gen.db.dbconst import DBLOGNAME -from gramps.gen.db.utils import make_database +from gramps.gen.db.dummydb import DummyDb #------------------------------------------------------------------------- # @@ -69,7 +69,7 @@ class DbState(Callback): place holder until a real DB is assigned. """ Callback.__init__(self) - self.db = make_database("dummydb") + self.db = DummyDb() self.open = False # Deprecated - use DbState.is_open() self.stack = [] @@ -135,7 +135,7 @@ class DbState(Callback): self.emit('no-database', ()) if self.is_open(): self.db.close() - self.db = make_database("dummydb") + self.db = DummyDb() self.open = False self.emit('database-changed', (self.db, )) diff --git a/gramps/gen/plug/_manager.py b/gramps/gen/plug/_manager.py index 1fe4d3955..152b5862d 100644 --- a/gramps/gen/plug/_manager.py +++ b/gramps/gen/plug/_manager.py @@ -39,7 +39,7 @@ import os import sys import re import logging -LOG = logging.getLogger('.' + __name__) +LOG = logging.getLogger('._manager') LOG.progagate = True from ..const import GRAMPS_LOCALE as glocale _ = glocale.translation.gettext @@ -100,8 +100,8 @@ class BasePluginManager: self.__modules = {} self.__pgr = PluginRegister.get_instance() - self.__registereddir_set = set() self.__loaded_plugins = {} + self.__scanned_dirs = [] def reg_plugins(self, direct, dbstate=None, uistate=None, load_on_reg=False): @@ -112,23 +112,24 @@ class BasePluginManager: If a relationship calculator for env var LANG is present, it is immediately loaded so it is available for all. """ - # if the directory does not exist, do nothing - if not os.path.isdir(direct): - return False # return value is True for error + # if we've already scanned this directory or if the directory does not + # exist, we are done. Should only happen in tests. - for (dirpath, dirnames, filenames) in os.walk(direct): - root, subdir = os.path.split(dirpath) - if subdir.startswith("."): - dirnames[:] = [] - continue - for dirname in dirnames: - # Skip hidden and system directories: - if dirname.startswith(".") or dirname in ["po", "locale"]: - dirnames.remove(dirname) - # if the path has not already been loaded, save it in the - # registereddir_list list for use on reloading. - self.__registereddir_set.add(dirpath) - self.__pgr.scan_dir(dirpath, uistate=uistate) + # LOG.warning("\nPlugin manager registration: %s, load_on_reg=%s," + # " been_here=%s, pahte exists:%s", direct, load_on_reg, + # direct in self.__scanned_dirs, os.path.isdir(direct)) + + if os.path.isdir(direct) and direct not in self.__scanned_dirs: + self.__scanned_dirs.append(direct) + for (dirpath, dirnames, filenames) in os.walk(direct, + topdown=True): + for dirname in dirnames[:]: + # Skip hidden and system directories: + if dirname.startswith(".") or dirname in ["po", "locale", + "__pycache__"]: + dirnames.remove(dirname) + # LOG.warning("Plugin dir scanned: %s", dirpath) + self.__pgr.scan_dir(dirpath, filenames, uistate=uistate) if load_on_reg: # Run plugins that request to be loaded on startup and @@ -136,6 +137,7 @@ class BasePluginManager: # first, remove hidden plugins_to_load = [] for plugin in self.__pgr.filter_load_on_reg(): + # LOG.warning("\nFound %s at registration", plugin.id) if plugin.id in config.get("plugin.hiddenplugins"): continue plugins_to_load.append(plugin) @@ -146,6 +148,8 @@ class BasePluginManager: max_count = len(plugins_to_load) while plugins_to_load: for plugin in plugins_to_load[:]: # copy of list + # LOG.warning("\nDependencies for %s at registration", + # plugin.id) delay = False for depend in plugin.depends_on: if depend not in [p.id for p in plugins_sorted]: @@ -167,8 +171,12 @@ class BasePluginManager: break # now load them: for plugin in plugins_sorted: + # next line shouldn't be necessary, but this gets called a lot + # of times during Travis test; so avoid multiple copies + plugin.data = [] mod = self.load_plugin(plugin) if hasattr(mod, "load_on_reg"): + # LOG.warning("\nRun %s at registration", plugin.id) try: results = mod.load_on_reg(dbstate, uistate, plugin) except: @@ -496,6 +504,8 @@ class BasePluginManager: retval.extend(data) except: retval.append(data) + # LOG.warning("Process plugin data=%s, %s, items=%s", + # process is not None, category, len(retval)) if process: return process(retval) return retval diff --git a/gramps/gen/plug/_pluginreg.py b/gramps/gen/plug/_pluginreg.py index e914e2e10..ec61caab9 100644 --- a/gramps/gen/plug/_pluginreg.py +++ b/gramps/gen/plug/_pluginreg.py @@ -43,6 +43,8 @@ from gramps.version import VERSION as GRAMPSVERSION, VERSION_TUPLE from ..const import IMAGE_DIR from ..const import GRAMPS_LOCALE as glocale _ = glocale.translation.gettext +import logging +LOG = logging.getLogger('._manager') #------------------------------------------------------------------------- # @@ -1104,11 +1106,16 @@ class PluginRegister: if __debug__: self.stable_only = False self.__plugindata = [] + self.__id_to_pdata = {} def add_plugindata(self, plugindata): + """ This is used to add an entry to the registration list. The way it + is used, this entry is not yet filled in, so we cannot use the id to + add to the __id_to_pdata dict at this time. """ self.__plugindata.append(plugindata) - def scan_dir(self, dir, uistate=None): + + def scan_dir(self, dir, filenames, uistate=None): """ The dir name will be scanned for plugin registration code, which will be loaded in :class:`PluginData` objects if they satisfy some checks. @@ -1123,9 +1130,8 @@ class PluginRegister: extlen = -len(ext) pymod = re.compile(r"^(.*)\.py$") - for filename in os.listdir(dir): - name = os.path.split(filename)[1] - if not name[extlen:] == ext: + for filename in filenames: + if not filename[extlen:] == ext: continue lenpd = len(self.__plugindata) full_filename = os.path.join(dir, filename) @@ -1150,9 +1156,14 @@ class PluginRegister: else: local_gettext = glocale.translation.gettext try: - #execfile(full_filename, exec (compile(stream, filename, 'exec'), make_environment(_=local_gettext), {'uistate': uistate}) + for pdata in self.__plugindata[lenpd:]: + # should not be duplicate IDs in different plugins + assert pdata.id not in self.__id_to_pdata + # if pdata.id in self.__id_to_pdata: + # print("Error: %s is duplicated!" % pdata.id) + self.__id_to_pdata[pdata.id] = pdata except ValueError as msg: print(_('ERROR: Failed reading plugin registration %(filename)s') % \ {'filename' : filename}) @@ -1170,6 +1181,7 @@ class PluginRegister: rmlist = [] ind = lenpd-1 for plugin in self.__plugindata[lenpd:]: + #LOG.warning("\nPlugin scanned %s at registration", plugin.id) ind += 1 plugin.directory = dir if not valid_plugin_version(plugin.gramps_target_version): @@ -1211,19 +1223,20 @@ class PluginRegister: module = match.groups()[0] plugin.mod_name = module plugin.fpath = dir + #LOG.warning("\nPlugin added %s at registration", plugin.id) rmlist.reverse() for ind in rmlist: + del self.__id_to_pdata[self.__plugindata[ind].id] del self.__plugindata[ind] def get_plugin(self, id): """ Return the :class:`PluginData` for the plugin with id """ - matches = [x for x in self.__plugindata if x.id == id] - matches.sort(key=lambda x: version(x.version)) - if len(matches) > 0: - return matches[-1] - return None + assert(len(self.__id_to_pdata) == len(self.__plugindata)) + # if len(self.__id_to_pdata) != len(self.__plugindata): + # print(len(self.__id_to_pdata), len(self.__plugindata)) + return self.__id_to_pdata.get(id, None) def type_plugins(self, ptype): """ diff --git a/gramps/plugins/db/dummydb.gpr.py b/gramps/plugins/db/dummydb.gpr.py deleted file mode 100644 index 5d1cbccf7..000000000 --- a/gramps/plugins/db/dummydb.gpr.py +++ /dev/null @@ -1,36 +0,0 @@ -# -# Gramps - a GTK+/GNOME based genealogy program -# -# Copyright (C) 2016 Tim G L Lyons -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# -from gramps.gen.plug._pluginreg import register, STABLE, DATABASE -from gramps.gen.const import GRAMPS_LOCALE as glocale -_ = glocale.translation.gettext - -register(DATABASE, - id = 'dummydb', - name = _("Dummy database"), - name_accell = _("Dummy Database"), - description = _("Dummy Database"), - version = '1.0.0', - gramps_target_version = "5.0", - status = STABLE, - fname = 'dummydb.py', - databaseclass = 'DummyDb', - authors=['Tim Lyons'], - authors_email=[""], -)