# # Gramps - a GTK+/GNOME based genealogy program # # Copyright (C) 2009 Douglas S. Blank # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # # $Id$ "Import from SQL Database" #------------------------------------------------------------------------- # # Standard Python Modules # #------------------------------------------------------------------------- from gettext import gettext as _ from gettext import ngettext import sqlite3 as sqlite import re import time #------------------------------------------------------------------------ # # Set up logging # #------------------------------------------------------------------------ import logging log = logging.getLogger(".ImportSql") #------------------------------------------------------------------------- # # GRAMPS modules # #------------------------------------------------------------------------- import gen.lib from QuestionDialog import ErrorDialog from DateHandler import parser as _dp from gen.plug import PluginManager, ImportPlugin from Utils import gender as gender_map from gui.utils import ProgressMeter from Utils import create_id def lookup(handle, event_ref_list): if handle is None: return -1 else: count = 0 for event_ref in event_ref_list: (private, note_list, attribute_list, ref, role) = event_ref if handle == ref: return count count += 1 return -1 #------------------------------------------------------------------------- # # SQL Reader # #------------------------------------------------------------------------- class Database(object): """ The db connection. """ def __init__(self, database): self.database = database self.db = sqlite.connect(self.database) self.cursor = self.db.cursor() def query(self, q, *args): if q.strip().upper().startswith("DROP"): try: self.cursor.execute(q, args) self.db.commit() except: "WARN: no such table to drop: '%s'" % q else: try: self.cursor.execute(q, args) self.db.commit() except: print "ERROR: query :", q print "ERROR: values:", args raise return self.cursor.fetchall() def close(self): """ Closes and writes out tables """ self.cursor.close() self.db.close() class SQLReader(object): def __init__(self, db, filename, callback): if not callable(callback): callback = lambda (percent): None # dummy self.db = db self.filename = filename self.callback = callback self.debug = 0 def openSQL(self): sql = None try: sql = Database(self.filename) except IOError, msg: errmsg = _("%s could not be opened\n") % self.filename ErrorDialog(errmsg,str(msg)) return None return sql def get_event_refs(self, sql, from_type, from_handle): results = sql.query("select * from event_ref where from_type = ? and from_handle = ?;", from_type, from_handle) return [self.pack_event_ref(sql, result) for result in results] def pack_event_ref(self, sql, data): (from_type, from_handle, ref, role0, role1, private) = data note_list = self.get_links(sql, "event_ref", from_handle, "note") attribute_list = self.get_attribute_list(sql, "event_ref", from_handle) return (private, note_list, attribute_list, ref, (role0, role1)) def get_url_list(self, sql, from_type, from_handle): # FIXME return [] def get_attribute_list(self, sql, from_type, from_handle): # FIXME return [] def get_media_list(self, sql, from_type, from_handle): # FIXME return [] def get_sources(self, sql, handle): results = sql.query("""select * from source where handle = ?;""", handle) return [self.pack_source(sql, result) for result in results] def pack_source(self, sql, data): (handle, gid, title, author, pubinfo, abbrev, change, marker0, marker1, private) = data # FIXME note_list = self.get_links(sql, "source", handle, "note") media_list = self.get_media_list(sql, "source", handle) reporef_list = [] datamap = None return (handle, gid, title, author, pubinfo, note_list, media_list, abbrev, change, datamap, reporef_list, (marker0, marker1), private) def get_links(self, sql, from_type, from_handle, to_type): results = sql.query("""select to_handle from link where from_type = ? and from_handle = ? and to_type = ?;""", from_type, from_handle, to_type) return [str(result) for result in results] def get_names(self, sql, handle, primary): result = sql.query("""select * from name where from_handle = ? and primary_name = ?;""", handle, primary) return result def package_name(self, sql, handle, primary): names = self.get_names(sql, handle, primary) result = [self.pack(sql, name) for name in names] if primary: if len(result) == 1: return result[0] elif len(result) == 0: return None else: raise Exception("too many primary names") else: return result def pack(self, sql, data): # unpack name from SQL table: (from_handle, handle, primary_name, private, first_name, surname, suffix, title, name_type0, name_type1, prefix, patronymic, group_as, sort_as, display_as, call) = data # FIXME source_list = self.get_links(sql, "name", from_handle, "source") note_list = self.get_links(sql, "name", from_handle, "note") date = self.get_date(sql, "name", from_handle) return (private, source_list, note_list, date, first_name, surname, suffix, title, (name_type0, name_type1), prefix, patronymic, group_as, sort_as, display_as, call) def get_date(self, sql, from_type, from_handle): rows = sql.query("select * from date where from_type = ? and from_handle = ?;", from_type, from_handle) if len(rows) == 1: (from_type, from_handle, calendar, modifier, quality, day1, month1, year1, slash1, day2, month2, year2, slash2, text, sortval, newyear) = rows[0] dateval = day1, month1, year1, slash1, day2, month2, year2, slash2 return (calendar, modifier, quality, dateval, text, sortval, newyear) elif len(rows) == 0: return None else: raise Exception("ERROR, wrong number of dates: %s" % rows) def process(self): sql = self.openSQL() total = (sql.query("select count(*) from note;")[0][0] + sql.query("select count(*) from person;")[0][0] + sql.query("select count(*) from event;")[0][0] + sql.query("select count(*) from family;")[0][0] + sql.query("select count(*) from repository;")[0][0] + sql.query("select count(*) from place;")[0][0] + sql.query("select count(*) from media;")[0][0] + sql.query("select count(*) from source;")[0][0]) self.trans = self.db.transaction_begin("",batch=True) self.db.disable_signals() count = 0.0 t = time.time() # --------------------------------- # Process note # --------------------------------- # FIXME: 11 note objects were referenced but not found notes = sql.query("""select * from note;""") for note in notes: (handle, gid, text, format, note_type1, note_type2, change, marker0, marker1, private) = note styled_text = [text, []] markups = sql.query("""select * from markup where handle = ?""", handle) for markup in markups: (mhandle, markup0, markup1, value, start_stop_list) = markup ss_list = eval(start_stop_list) styled_text[1] += [((markup0, markup1), value, ss_list)] self.db.note_map[str(handle)] = (str(handle), gid, styled_text, format, (note_type1, note_type2), change, (marker0, marker1), private) count += 1 self.callback(100 * count/total) # --------------------------------- # Process person # --------------------------------- people = sql.query("""select * from person;""") for person in people: if person is None: continue (handle, # 0 gid, # 1 gender, # 2 death_ref_handle, # 5 birth_ref_handle, # 6 change, # 17 marker0, # 18 marker1, # 18 private, # 19 ) = person # primary_name, # 3 primary_name = self.package_name(sql, handle, True) # alternate_names, # 4 alternate_names = self.package_name(sql, handle, False) # list # event_ref_list, # 7 event_ref_list = self.get_event_refs(sql, "person", handle) # family_list, # 8 family_list = [] # parent_family_list, # 9 parent_family_list = [] # media_list, # 10 media_list = [] # address_list, # 11 address_list = [] # attribute_list, # 12 attribute_list = [] # urls, # 13 urls = self.get_url_list(sql, "person", handle) # lds_ord_list, # 14 lds_ord_list = [] # psource_list, # 15 psource_list = self.get_links(sql, "person", handle, "source") # pnote_list, # 16 pnote_list = [] # person_ref_list, # 20 person_ref_list = [] death_ref_index = lookup(death_ref_handle, event_ref_list) birth_ref_index = lookup(birth_ref_handle, event_ref_list) self.db.person_map[str(handle)] = (str(handle), # 0 gid, # 1 gender, # 2 primary_name, # 3 alternate_names, # 4 death_ref_index, # 5 birth_ref_index, # 6 event_ref_list, # 7 family_list, # 8 parent_family_list, # 9 media_list, # 10 address_list, # 11 attribute_list, # 12 urls, # 13 lds_ord_list, # 14 psource_list, # 15 pnote_list, # 16 change, # 17 (marker0, marker1), # 18 private, # 19 person_ref_list, # 20 ) count += 1 self.callback(100 * count/total) # --------------------------------- # Process event # --------------------------------- events = sql.query("""select * from event;""") for event in events: (handle, gid, the_type0, the_type1, description, change, marker0, marker1, private) = event # FIXME: source_list = self.get_links(sql, "event", handle, "source") note_list = self.get_links(sql, "event", handle, "note") media_list = self.get_links(sql, "event", handle, "media") attribute_list = self.get_links(sql, "event", handle, "attribute") date = self.get_date(sql, "event", handle) place = None # self.get_place() data = (str(handle), gid, (the_type0, the_type1), date, description, place, source_list, note_list, media_list, attribute_list, change, (marker0, marker1), private) self.db.event_map[str(handle)] = data count += 1 self.callback(100 * count/total) # --------------------------------- # Process family # --------------------------------- people = sql.query("""select * from family;""") for person in people: count += 1 self.callback(100 * count/total) # --------------------------------- # Process repository # --------------------------------- people = sql.query("""select * from repository;""") for person in people: count += 1 self.callback(100 * count/total) # --------------------------------- # Process place # --------------------------------- people = sql.query("""select * from place;""") for person in people: count += 1 self.callback(100 * count/total) # --------------------------------- # Process source # --------------------------------- people = sql.query("""select * from source;""") for person in people: count += 1 self.callback(100 * count/total) # --------------------------------- # Process media # --------------------------------- people = sql.query("""select * from media;""") for person in people: count += 1 self.callback(100 * count/total) t = time.time() - t msg = ngettext('Import Complete: %d second','Import Complete: %d seconds', t ) % t self.db.transaction_commit(self.trans,_("SQL import")) self.db.enable_signals() self.db.request_rebuild() print msg return None def importData(db, filename, callback=None): g = SQLReader(db, filename, callback) g.process() #------------------------------------------------------------------------- # # Register the plugin # #------------------------------------------------------------------------- _name = _('SQLite Import') _description = _('SQLite is a common local database format') pmgr = PluginManager.get_instance() plugin = ImportPlugin(name = _('SQLite Database'), description = _("Import data from SQLite database"), import_function = importData, extension = "sql") pmgr.register_plugin(plugin)