# # Gramps - a GTK+/GNOME based genealogy program # # Copyright (C) 2009 Douglas S. Blank # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # # $Id$ "Import from SQL Database" #------------------------------------------------------------------------- # # Standard Python Modules # #------------------------------------------------------------------------- from gettext import gettext as _ from gettext import ngettext import sqlite3 as sqlite import re import time #------------------------------------------------------------------------ # # Set up logging # #------------------------------------------------------------------------ import logging log = logging.getLogger(".ImportSql") #------------------------------------------------------------------------- # # GRAMPS modules # #------------------------------------------------------------------------- import gen.lib from QuestionDialog import ErrorDialog from DateHandler import parser as _dp from gen.plug import PluginManager, ImportPlugin from Utils import gender as gender_map from gui.utils import ProgressMeter from Utils import create_id def lookup(handle, event_ref_list): if handle is None: return -1 else: count = 0 for event_ref in event_ref_list: (private, note_list, attribute_list, ref, role) = event_ref if handle == ref: return count count += 1 return -1 #------------------------------------------------------------------------- # # SQL Reader # #------------------------------------------------------------------------- class Database(object): """ The db connection. """ def __init__(self, database): self.database = database self.db = sqlite.connect(self.database) self.cursor = self.db.cursor() def query(self, q, *args): if q.strip().upper().startswith("DROP"): try: self.cursor.execute(q, args) self.db.commit() except: "WARN: no such table to drop: '%s'" % q else: try: self.cursor.execute(q, args) self.db.commit() except: print "ERROR: query :", q print "ERROR: values:", args raise return self.cursor.fetchall() def close(self): """ Closes and writes out tables """ self.cursor.close() self.db.close() class SQLReader(object): def __init__(self, db, filename, callback): if not callable(callback): callback = lambda (percent): None # dummy self.db = db self.filename = filename self.callback = callback self.debug = 0 def openSQL(self): sql = None try: sql = Database(self.filename) except IOError, msg: errmsg = _("%s could not be opened\n") % self.filename ErrorDialog(errmsg,str(msg)) return None return sql # ----------------------------------------------- # Get methods to retrieve data from the tables # ----------------------------------------------- def get_address_list(self, sql, from_type, from_handle): # FIXME return [] def get_attribute_list(self, sql, from_type, from_handle): rows = sql.query("select * from attribute where from_type = ? and from_handle = ?;", from_type, from_handle) retval = [] for row in rows: (handle, from_type, from_handle, the_type0, the_type1, value, private) = row source_list = self.get_source_list(sql, "attribute", handle) note_list = self.get_note_list(sql, "attribute", handle) retval.append((private, source_list, note_list, (the_type0, the_type1), value)) return retval def get_child_ref_list(self, sql, from_type, from_handle): rows = sql.query("select * from child_ref where from_type = ? and from_handle = ?;", from_type, from_handle) retval = [] for row in rows: (rtype, rhandle, ref, frel0, frel1, mrel0, mrel1, private) = row source_list = self.get_source_list(sql, "child_ref", rhandle) note_list = self.get_note_list(sql, "child_ref", rhandle) retval.append((private, source_list, note_list, ref, (frel0, frel1), (mrel0, mrel1))) return retval def get_datamap(self, sql, from_type, from_handle): # FIXME: return {} def get_event_ref_list(self, sql, from_type, from_handle): results = sql.query("select * from event_ref where from_type = ? and from_handle = ?;", from_type, from_handle) return [self.pack_event_ref(sql, result) for result in results] def get_family_list(self, sql, from_type, from_handle): return self.get_list(sql, from_type, from_handle, "family") def get_parent_family_list(self, sql, from_type, from_handle): return self.get_list(sql, from_type, from_handle, "parent_family") def get_person_ref_list(self, sql, from_type, from_handle): rows = sql.query("select * from person_ref where from_type = ? and from_handle = ?;", from_type, from_handle) retval = [] for row in rows: (from_type, from_handle, handle, description, private) = row source_list = self.get_source_list(sql, "person_ref", handle) note_list = self.get_note_list(sql, "person_ref", handle) retval.append((private, source_list, note_list, handle, description)) return retval def get_location_list(self, sql, from_type, from_handle): # FIXME return [] def get_lds_list(self, sql, from_type, from_handle): # FIXME return [] def get_media_list(self, sql, from_type, from_handle): # FIXME return [] def get_note_list(self, sql, from_type, from_handle): return self.get_list(sql, from_type, from_handle, "note") def get_repository_ref_list(self, sql, from_type, from_handle): # FIXME return [] def get_source_list(self, sql, from_type, from_handle): results = sql.query("""select * from source where from_type = ? and handle = ?;""", from_type, from_handle) return [self.pack_source(sql, result) for result in results] def get_url_list(self, sql, from_type, from_handle): # FIXME return [] # --------------------------------- # Helpers # --------------------------------- def pack_event_ref(self, sql, data): (from_type, from_handle, ref, role0, role1, private) = data note_list = self.get_note_list(sql, "event_ref", from_handle) attribute_list = self.get_attribute_list(sql, "event_ref", from_handle) return (private, note_list, attribute_list, ref, (role0, role1)) def pack_source(self, sql, data): (handle, gid, title, author, pubinfo, abbrev, change, marker0, marker1, private) = data note_list = self.get_note_list(sql, "source", handle) media_list = self.get_media_list(sql, "source", handle) reporef_list = self.get_repository_ref_list(sql, "source", handle) datamap = None return (handle, gid, title, author, pubinfo, note_list, media_list, abbrev, change, datamap, reporef_list, (marker0, marker1), private) def get_list(self, sql, from_type, from_handle, to_type): results = sql.query("""select to_handle from link where from_type = ? and from_handle = ? and to_type = ?;""", from_type, from_handle, to_type) return [str(result) for result in results] def get_names(self, sql, handle, primary): names = sql.query("""select * from name where from_handle = ? and primary_name = ?;""", handle, primary) result = [self.pack_name(sql, name) for name in names] if primary: if len(result) == 1: return result[0] elif len(result) == 0: return None else: print Exception("too many primary names") else: return result def pack_name(self, sql, data): # unpack name from SQL table: (from_handle, handle, primary_name, private, first_name, surname, suffix, title, name_type0, name_type1, prefix, patronymic, group_as, sort_as, display_as, call) = data source_list = self.get_source_list(sql, "name", from_handle) note_list = self.get_note_list(sql, "name", from_handle) date = self.get_date(sql, "name", from_handle) return (private, source_list, note_list, date, first_name, surname, suffix, title, (name_type0, name_type1), prefix, patronymic, group_as, sort_as, display_as, call) def get_place(self, sql, from_type, handle): row = self.get_list(sql, from_type, handle, "place") if len(row) == 1: return row[0] else: print AttributeError("invalid place '%s' '%s'" % (from_type, handle)) def get_date(self, sql, from_type, from_handle): rows = sql.query("select * from date where from_type = ? and from_handle = ?;", from_type, from_handle) if len(rows) == 1: (from_type, from_handle, calendar, modifier, quality, day1, month1, year1, slash1, day2, month2, year2, slash2, text, sortval, newyear) = rows[0] dateval = day1, month1, year1, slash1, day2, month2, year2, slash2 return (calendar, modifier, quality, dateval, text, sortval, newyear) elif len(rows) == 0: return None else: print Exception("ERROR, wrong number of dates: %s" % rows) def process(self): sql = self.openSQL() total = (sql.query("select count(*) from note;")[0][0] + sql.query("select count(*) from person;")[0][0] + sql.query("select count(*) from event;")[0][0] + sql.query("select count(*) from family;")[0][0] + sql.query("select count(*) from repository;")[0][0] + sql.query("select count(*) from place;")[0][0] + sql.query("select count(*) from media;")[0][0] + sql.query("select count(*) from source;")[0][0]) self.trans = self.db.transaction_begin("",batch=True) self.db.disable_signals() count = 0.0 t = time.time() # --------------------------------- # Process note # --------------------------------- notes = sql.query("""select * from note;""") for note in notes: (handle, gid, text, format, note_type1, note_type2, change, marker0, marker1, private) = note styled_text = [text, []] markups = sql.query("""select * from markup where handle = ?""", handle) for markup in markups: (mhandle, markup0, markup1, value, start_stop_list) = markup ss_list = eval(start_stop_list) styled_text[1] += [((markup0, markup1), value, ss_list)] self.db.note_map[str(handle)] = (str(handle), gid, styled_text, format, (note_type1, note_type2), change, (marker0, marker1), private) count += 1 self.callback(100 * count/total) # --------------------------------- # Process person # --------------------------------- people = sql.query("""select * from person;""") for person in people: if person is None: continue (handle, # 0 gid, # 1 gender, # 2 death_ref_handle, # 5 birth_ref_handle, # 6 change, # 17 marker0, # 18 marker1, # 18 private, # 19 ) = person primary_name = self.get_names(sql, handle, True) # one alternate_names = self.get_names(sql, handle, False) # list event_ref_list = self.get_event_ref_list(sql, "person", handle) family_list = self.get_family_list(sql, "person", handle) parent_family_list = self.get_parent_family_list(sql, "person", handle) media_list = self.get_media_list(sql, "person", handle) address_list = self.get_address_list(sql, "person", handle) attribute_list = self.get_attribute_list(sql, "person", handle) urls = self.get_url_list(sql, "person", handle) lds_ord_list = self.get_lds_list(sql, "person", handle) psource_list = self.get_source_list(sql, "person", handle) pnote_list = self.get_note_list(sql, "person", handle) person_ref_list = self.get_person_ref_list(sql, "person", handle) death_ref_index = lookup(death_ref_handle, event_ref_list) birth_ref_index = lookup(birth_ref_handle, event_ref_list) self.db.person_map[str(handle)] = (str(handle), # 0 gid, # 1 gender, # 2 primary_name, # 3 alternate_names, # 4 death_ref_index, # 5 birth_ref_index, # 6 event_ref_list, # 7 family_list, # 8 parent_family_list, # 9 media_list, # 10 address_list, # 11 attribute_list, # 12 urls, # 13 lds_ord_list, # 14 psource_list, # 15 pnote_list, # 16 change, # 17 (marker0, marker1), # 18 private, # 19 person_ref_list, # 20 ) count += 1 self.callback(100 * count/total) # --------------------------------- # Process event # --------------------------------- events = sql.query("""select * from event;""") for event in events: (handle, gid, the_type0, the_type1, description, change, marker0, marker1, private) = event source_list = self.get_source_list(sql, "event", handle) note_list = self.get_note_list(sql, "event", handle) media_list = self.get_media_list(sql, "event", handle) attribute_list = self.get_attribute_list(sql, "event", handle) date = self.get_date(sql, "event", handle) place = self.get_place(sql, "event", handle) data = (str(handle), gid, (the_type0, the_type1), date, description, place, source_list, note_list, media_list, attribute_list, change, (marker0, marker1), private) self.db.event_map[str(handle)] = data count += 1 self.callback(100 * count/total) # --------------------------------- # Process family # --------------------------------- families = sql.query("""select * from family;""") for family in families: (handle, gid, father_handle, mother_handle, the_type0, the_type1, change, marker0, marker1, private) = family child_ref_list = self.get_child_ref_list(sql, "family", handle) event_ref_list = self.get_event_ref_list(sql, "family", handle) media_list = self.get_media_list(sql, "family", handle) attribute_list = self.get_attribute_list(sql, "family", handle) lds_seal_list = self.get_lds_list(sql, "family", handle) source_list = self.get_source_list(sql, "family", handle) note_list = self.get_note_list(sql, "family", handle) self.db.family_map[str(handle)] = (str(handle), gid, father_handle, mother_handle, child_ref_list, (the_type0, the_type1), event_ref_list, media_list, attribute_list, lds_seal_list, source_list, note_list, change, (marker0, marker1), private) count += 1 self.callback(100 * count/total) # --------------------------------- # Process repository # --------------------------------- repositories = sql.query("""select * from repository;""") for repo in repositories: (handle, gid, the_type0, the_type1, name, change, marker0, marker1, private) = repo note_list = self.get_note_list(sql, "repository", handle) address_list = self.get_address_list(sql, "repository", handle) urls = self.get_url_list(sql, "repository", handle) self.db.repository_map[str(handle)] = (str(handle), gid, (the_type0, the_type1), name, note_list, address_list, urls, change, (marker0, marker1), private) count += 1 self.callback(100 * count/total) # --------------------------------- # Process place # --------------------------------- places = sql.query("""select * from place;""") for place in places: count += 1 (handle, gid, title, long, lat, change, marker0, marker1, private) = place main_loc = None # FIXME alt_location_list = self.get_location_list(sql, "place", handle) urls = self.get_url_list(sql, "place", handle) media_list = self.get_media_list(sql, "place", handle) source_list = self.get_source_list(sql, "place", handle) note_list = self.get_note_list(sql, "place", handle) self.db.place_map[str(handle)] = (str(handle), gid, title, long, lat, main_loc, alt_location_list, urls, media_list, source_list, note_list, change, (marker0, marker1), private) self.callback(100 * count/total) # --------------------------------- # Process source # --------------------------------- sources = sql.query("""select * from source;""") for source in sources: (from_type, handle, gid, title, author, pubinfo, abbrev, change, marker0, marker1, private) = source note_list = self.get_note_list(sql, "source", handle) media_list = self.get_media_list(sql, "source", handle) datamap = self.get_datamap(sql, "source", handle) reporef_list = self.get_repository_ref_list(sql, "source", handle) self.db.source_map[str(handle)] = (str(handle), gid, title, author, pubinfo, note_list, media_list, abbrev, change, datamap, reporef_list, (marker0, marker1), private) count += 1 self.callback(100 * count/total) # --------------------------------- # Process media # --------------------------------- media = sql.query("""select * from media;""") for med in media: (handle, gid, path, mime, desc, change, marker0, marker1, private) = med attribute_list = self.get_attribute_list(sql, "media", handle) source_list = self.get_source_list(sql, "media", handle) note_list = self.get_note_list(sql, "media", handle) date = self.get_date(sql, "media", handle) self.db.media_map[str(handle)] = (str(handle), gid, path, mime, desc, attribute_list, source_list, note_list, change, date, (marker0, marker1), private) count += 1 self.callback(100 * count/total) t = time.time() - t msg = ngettext('Import Complete: %d second','Import Complete: %d seconds', t ) % t self.db.transaction_commit(self.trans,_("SQL import")) self.db.enable_signals() self.db.request_rebuild() print msg return None def importData(db, filename, callback=None): g = SQLReader(db, filename, callback) g.process() #------------------------------------------------------------------------- # # Register the plugin # #------------------------------------------------------------------------- _name = _('SQLite Import') _description = _('SQLite is a common local database format') pmgr = PluginManager.get_instance() plugin = ImportPlugin(name = _('SQLite Database'), description = _("Import data from SQLite database"), import_function = importData, extension = "sql") pmgr.register_plugin(plugin)