Cyclone-Team/gramps
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

SQL Export/Import about 30% done: Notes are complete. People and Events close. The rest still to check.

Browse Source 
svn: r12917
This commit is contained in:
Doug Blank 2009-08-07 22:32:30 +00:00
parent 9eeb2d8b92
commit f3a7e259e5
2 changed files with 401 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

151
src/plugins/export/ExportSql.py
View File

@ -39,6 +39,18 @@ from gui.utils import ProgressMeter
import ExportOptions
from Utils import create_id
def lookup(index, event_ref_list):
if index < 0:
return None
else:
count = 0
for event_ref in event_ref_list:
(private, note_list, attribute_list, ref, role) = event_ref
if index == count:
return ref
count += 1
return None
def makeDB(db):
db.query("""drop table note;""")
db.query("""drop table person;""")
@ -59,37 +71,40 @@ def makeDB(db):
db.query("""drop table media_ref;""")
db.query("""drop table address;""")
db.query("""drop table attribute;""")
# Completed
db.query("""CREATE TABLE note (
handle CHARACTER(25),
gid CHARACTER(25),
text TEXT,
format TEXT,
format INTEGER,
note_type1 INTEGER,
note_type2 TEXT,
change TEXT,
marker0 TEXT,
change INTEGER,
marker0 INTEGER,
marker1 TEXT,
private BOOLEAN);""")
db.query("""CREATE TABLE name (
from_handle CHARACTER(25),
handle CHARACTER(25),
primary_name BOOLEAN,
private BOOLEAN,
first_name TEXT,
surname TEXT,
suffix TEXT,
title TEXT,
name_type0 TEXT,
name_type0 INTEGER,
name_type1 TEXT,
prefix TEXT,
patronymic TEXT,
group_as TEXT,
sort_as TEXT,
display_as TEXT,
sort_as INTEGER,
display_as INTEGER,
call TEXT);""")
db.query("""CREATE TABLE date (
type CHARACTER(10),
from_type CHARACTER(25),
from_handle CHARACTER(25),
calendar INTEGER,
modifier INTEGER,
quality INTEGER,
@ -108,11 +123,11 @@ def makeDB(db):
db.query("""CREATE TABLE person (
handle CHARACTER(25),
gid CHARACTER(25),
gender CHAR(1),
death_ref_index TEXT,
birth_ref_index TEXT,
change TEXT,
marker0 TEXT,
gender INTEGER,
death_ref_handle TEXT,
birth_ref_handle TEXT,
change INTEGER,
marker0 INTEGER,
marker1 TEXT,
private BOOLEAN);""")
@ -121,10 +136,10 @@ def makeDB(db):
gid CHARACTER(25),
father_handle CHARACTER(25),
mother_handle CHARACTER(25),
the_type0 TEXT,
the_type0 INTEGER,
the_type1 TEXT,
change TEXT,
marker0 TEXT,
change INTEGER,
marker0 INTEGER,
marker1 TEXT,
private BOOLEAN);""")
@ -134,19 +149,19 @@ def makeDB(db):
title TEXT,
long FLOAT,
lat FLOAT,
change TEXT,
marker0 TEXT,
change INTEGER,
marker0 INTEGER,
marker1 TEXT,
private BOOLEAN);""")
db.query("""CREATE TABLE event (
handle CHARACTER(25),
gid CHARACTER(25),
the_type0 TEXT,
the_type0 INTEGER,
the_type1 TEXT,
description TEXT,
change TEXT,
marker0 TEXT,
change INTEGER,
marker0 INTEGER,
marker1 TEXT,
private BOOLEAN);""")
@ -157,8 +172,8 @@ def makeDB(db):
author TEXT,
pubinfo TEXT,
abbrev TEXT,
change TEXT,
marker0 TEXT,
change INTEGER,
marker0 INTEGER,
marker1 TEXT,
private BOOLEAN);""")
@ -168,26 +183,26 @@ def makeDB(db):
path TEXT,
mime TEXT,
desc TEXT,
change TEXT,
marker0 TEXT,
change INTEGER,
marker0 INTEGER,
marker1 TEXT,
private BOOLEAN);""")
db.query("""CREATE TABLE repository (
handle CHARACTER(25),
gid CHARACTER(25),
the_type0 TEXT,
the_type0 INTEGER,
the_type1 TEXT,
name TEXT,
change TEXT,
marker0 TEXT,
change INTEGER,
marker0 INTEGER,
marker1 TEXT,
private BOOLEAN);""")
db.query("""CREATE TABLE link (
from_type CHARACTER(10),
from_type CHARACTER(25),
from_handle CHARACTER(25),
to_type CHARACTER(10),
to_type CHARACTER(25),
to_handle CHARACTER(25));""")
db.query("""CREATE TABLE markup (
@ -232,7 +247,7 @@ def makeDB(db):
from_type CHARACTER(25),
from_handle CHARACTER(25),
handle CHARACTER(25),
type CHARACTER(10),
type CHARACTER(25),
place TEXT,
famc CHARACTER(25),
temple TEXT,
@ -261,7 +276,7 @@ def makeDB(db):
db.query("""CREATE TABLE attribute (
handle CHARACTER(25),
from_type CHARACTER(10),
from_type CHARACTER(25),
from_handle CHARACTER(25),
the_type0 INTEGER,
the_type1 TEXT,
@ -440,7 +455,7 @@ def export_note(db, handle, gid, text, format, note_type0,
handle, gid, text, format, note_type0,
note_type1, change, marker0, marker1, private)
def export_name(db, handle, data):
def export_name(db, from_handle, handle, primary, data):
if data:
(private, source_list, note_list, date,
first_name, surname, suffix, title,
@ -448,7 +463,9 @@ def export_name(db, handle, data):
group_as, sort_as, display_as, call) = data
db.query("""INSERT into name (
from_handle,
handle,
primary_name,
private,
first_name,
surname,
@ -462,19 +479,20 @@ def export_name(db, handle, data):
sort_as,
display_as,
call
) values (?, ?, ?, ?, ?, ?, ?,
) values (?, ?, ?, ?, ?, ?, ?, ?, ?,
?, ?, ?, ?, ?, ?, ?);""",
handle, private, first_name, surname, suffix, title,
from_handle, handle, primary, private, first_name, surname, suffix, title,
name_type[0], name_type[1], prefix, patronymic, group_as,
sort_as, display_as, call)
export_date(db, "name", handle, date)
if date:
export_date(db, "name", handle, date)
export_list(db, "name", handle, "note", note_list)
export_source_list(db, "name", handle, source_list)
def export_date(db, date_type, handle, data):
if data:
(calendar, modifier, quality, dateval, text, sortval, newyear) = data
def export_date(db, date_type, handle, date):
if True:
(calendar, modifier, quality, dateval, text, sortval, newyear) = date
if len(dateval) == 4:
day1, month1, year1, slash1 = dateval
day2, month2, year2, slash2 = 0, 0, 0, 0
@ -483,7 +501,8 @@ def export_date(db, date_type, handle, data):
else:
raise ("ERROR:", dateval)
db.query("""INSERT INTO date (
type,
from_type,
from_handle,
calendar,
modifier,
quality,
@ -497,9 +516,9 @@ def export_date(db, date_type, handle, data):
slash2,
text,
sortval,
newyear) VALUES (?, ?, ?, ?, ?, ?, ?, ?,
newyear) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?,
?, ?, ?, ?, ?, ?, ?);""",
date_type, calendar, modifier, quality,
date_type, handle, calendar, modifier, quality,
day1, month1, year1, slash1,
day2, month2, year2, slash2,
text, sortval, newyear)
@ -520,7 +539,8 @@ def export_source_list(db, from_type, handle, source_list):
for source in source_list:
(date, private, note_list, confidence, ref, page) = source
export_source_ref(db, from_type, handle, ref, private, confidence, page)
export_date(db, "source", ref, date)
if date:
export_date(db, "source", ref, date)
export_list(db, "source", ref, "note", note_list)
def export_media_list(db, from_type, from_handle, media_list):
@ -588,11 +608,9 @@ def exportData(database, filename, option_box=None, callback=None):
len(database.family_map) +
len(database.repository_map) +
len(database.place_map) +
len(database.source_map) +
len(database.media_map) +
len(database.place_map) +
len(database.source_map))
count = 0
count = 0.0
db = Database(filename)
makeDB(db)
@ -622,7 +640,8 @@ def exportData(database, filename, option_box=None, callback=None):
(handle, gid, the_type, date, description, place,
source_list, note_list, media_list, attribute_list,
change, marker, private) = event
export_date(db, "event", event_handle, date)
if date:
export_date(db, "event", event_handle, date)
db.query("""INSERT INTO event (
handle,
gid,
@ -680,13 +699,12 @@ def exportData(database, filename, option_box=None, callback=None):
private, # 19
person_ref_list, # 20
) = person
db.query("""INSERT INTO person (
handle,
gid,
gender,
death_ref_index,
birth_ref_index,
death_ref_handle,
birth_ref_handle,
change,
marker0,
marker1,
@ -694,8 +712,8 @@ def exportData(database, filename, option_box=None, callback=None):
handle,
gid,
gender,
death_ref_index,
birth_ref_index,
lookup(death_ref_index, event_ref_list),
lookup(birth_ref_index, event_ref_list),
change,
marker[0],
marker[1],
@ -724,7 +742,8 @@ def exportData(database, filename, option_box=None, callback=None):
(street, city, county, state, country, postal, phone) = location
addr_handle = create_id()
export_address(db, "person", handle, addr_handle, street, city, county, state, country, postal, phone, private)
export_date(db, "address", addr_handle, date)
if date:
export_date(db, "address", addr_handle, date)
export_list(db, "address", addr_handle, "note", anote_list)
export_source_list(db, "address", addr_handle, source_list)
@ -736,15 +755,17 @@ def exportData(database, filename, option_box=None, callback=None):
famc, temple, status, lprivate) = ldsord
lds_handle = create_id()
export_lds(db, "person", handle, lds_handle, type, place, famc, temple, status, lprivate)
export_date(db, "lds", lds_handle, date)
if date:
export_date(db, "lds", lds_handle, date)
export_list(db, "lds", lds_handle, "note", lnote_list)
export_source_list(db, "lds", lds_handle, lsource_list)
# -------------------------------------
# Names
# -------------------------------------
export_name(db, handle, primary_name)
map(lambda name: export_name(db, handle, name), alternate_names)
export_name(db, handle, create_id(), True, primary_name)
map(lambda name: export_name(db, handle, create_id(), False, name),
alternate_names)
count += 1
callback(100 * count/total)
@ -794,7 +815,8 @@ def exportData(database, filename, option_box=None, callback=None):
famc, temple, status, lprivate) = ldsord
lds_handle = create_id()
export_lds(db, "family", handle, lds_handle, type, place, famc, temple, status, lprivate)
export_date(db, "lds", lds_handle, date)
if date:
export_date(db, "lds", lds_handle, date)
export_list(db, "lds", lds_handle, "note", lnote_list)
export_source_list(db, "lds", lds_handle, lsource_list)
@ -830,7 +852,8 @@ def exportData(database, filename, option_box=None, callback=None):
addr_handle = create_id()
export_address(db, "repository", handle, addr_handle, street, city, county, state,
country, postal, phone, private)
export_date(db, "address", addr_handle, date)
if date:
export_date(db, "address", addr_handle, date)
export_list(db, "address", addr_handle, "note", anote_list)
export_source_list(db, "address", addr_handle, asource_list)
@ -896,6 +919,12 @@ def exportData(database, filename, option_box=None, callback=None):
export_list(db, "source", handle, "note", note_list)
export_media_list(db, "source", handle, media_list)
# FIXME: reporef_list, datamap
#print "FIXME: reporef_list", reporef_list
#print "FIXME: datamap", datamap
#FIXME: reporef_list []
#FIXME: datamap {}
#FIXME: reporef_list [([], u'b2cfa6e37654b308559', '', (2, u''), False)]
#FIXME: datamap {}
count += 1
callback(100 * count/total)
@ -926,8 +955,8 @@ def exportData(database, filename, option_box=None, callback=None):
private) VALUES (?,?,?,?,?,?,?,?,?);""",
handle, gid, path, mime, desc,
change, marker[0], marker[1], private)
export_date(db, "media", handle, date)
if date:
export_date(db, "media", handle, date)
export_list(db, "media", handle, "note", note_list)
export_source_list(db, "media", handle, source_list)
export_attribute_list(db, "media", handle, attribute_list)

318
src/plugins/import/ImportSql.py
View File

@ -53,6 +53,19 @@ from Utils import gender as gender_map
from gui.utils import ProgressMeter
from Utils import create_id
def lookup(handle, event_ref_list):
if handle is None:
return -1
else:
count = 0
for event_ref in event_ref_list:
(private, note_list, attribute_list, ref, role) = event_ref
if handle == ref:
return count
count += 1
return -1
#-------------------------------------------------------------------------
#
# SQL Reader
@ -91,7 +104,8 @@ class Database(object):
class SQLReader(object):
def __init__(self, db, filename, callback):
print filename
if not callable(callback):
callback = lambda (percent): None # dummy
self.db = db
self.filename = filename
self.callback = callback
@ -107,15 +121,162 @@ class SQLReader(object):
return None
return sql
def get_event_refs(self, sql, from_type, from_handle):
results = sql.query("select * from event_ref where from_type = ? and from_handle = ?;",
from_type,
from_handle)
return [self.pack_event_ref(sql, result) for result in results]
def pack_event_ref(self, sql, data):
(from_type,
from_handle,
ref,
role0,
role1,
private) = data
note_list = self.get_links(sql, "event_ref", from_handle, "note")
attribute_list = self.get_attribute_list(sql, "event_ref", from_handle)
return (private, note_list, attribute_list, ref, (role0, role1))
def get_url_list(self, sql, from_type, from_handle):
# FIXME
return []
def get_attribute_list(self, sql, from_type, from_handle):
# FIXME
return []
def get_media_list(self, sql, from_type, from_handle):
# FIXME
return []
def get_sources(self, sql, handle):
results = sql.query("""select * from source where handle = ?;""",
handle)
return [self.pack_source(sql, result) for result in results]
def pack_source(self, sql, data):
(handle,
gid,
title,
author,
pubinfo,
abbrev,
change,
marker0,
marker1,
private) = data
# FIXME
note_list = self.get_links(sql, "source", handle, "note")
media_list = self.get_media_list(sql, "source", handle)
reporef_list = []
datamap = None
return (handle, gid, title,
author, pubinfo,
note_list,
media_list,
abbrev,
change, datamap,
reporef_list,
(marker0, marker1), private)
def get_links(self, sql, from_type, from_handle, to_type):
results = sql.query("""select to_handle from link where from_type = ? and from_handle = ? and to_type = ?;""",
from_type, from_handle, to_type)
return [str(result) for result in results]
def get_names(self, sql, handle, primary):
result = sql.query("""select * from name where from_handle = ? and primary_name = ?;""",
handle, primary)
return result
def package_name(self, sql, handle, primary):
names = self.get_names(sql, handle, primary)
result = [self.pack(sql, name) for name in names]
if primary:
if len(result) == 1:
return result[0]
elif len(result) == 0:
return None
else:
raise Exception("too many primary names")
else:
return result
def pack(self, sql, data):
# unpack name from SQL table:
(from_handle,
handle,
primary_name,
private,
first_name,
surname,
suffix,
title,
name_type0,
name_type1,
prefix,
patronymic,
group_as,
sort_as,
display_as,
call) = data
# FIXME
source_list = self.get_links(sql, "name", from_handle, "source")
note_list = self.get_links(sql, "name", from_handle, "note")
date = self.get_date(sql, "name", from_handle)
return (private, source_list, note_list, date,
first_name, surname, suffix, title,
(name_type0, name_type1), prefix, patronymic,
group_as, sort_as, display_as, call)
def get_date(self, sql, from_type, from_handle):
rows = sql.query("select * from date where from_type = ? and from_handle = ?;",
from_type, from_handle)
if len(rows) == 1:
(from_type,
from_handle,
calendar,
modifier,
quality,
day1,
month1,
year1,
slash1,
day2,
month2,
year2,
slash2,
text,
sortval,
newyear) = rows[0]
dateval = day1, month1, year1, slash1, day2, month2, year2, slash2
return (calendar, modifier, quality, dateval, text, sortval, newyear)
elif len(rows) == 0:
return None
else:
raise Exception("ERROR, wrong number of dates: %s" % rows)
def process(self):
progress = ProgressMeter(_('SQLite Import'))
progress.set_pass(_('Reading data...'), 1)
sql = self.openSQL()
progress.set_pass(_('Importing data...'), 100)
total = (sql.query("select count(*) from note;")[0][0] +
sql.query("select count(*) from person;")[0][0] +
sql.query("select count(*) from event;")[0][0] +
sql.query("select count(*) from family;")[0][0] +
sql.query("select count(*) from repository;")[0][0] +
sql.query("select count(*) from place;")[0][0] +
sql.query("select count(*) from media;")[0][0] +
sql.query("select count(*) from source;")[0][0])
self.trans = self.db.transaction_begin("",batch=True)
self.db.disable_signals()
count = 0.0
t = time.time()
# ---------------------------------
# Process note
# ---------------------------------
# FIXME: 11 note objects were referenced but not found
notes = sql.query("""select * from note;""")
for note in notes:
(handle,
@ -128,6 +289,7 @@ class SQLReader(object):
marker0,
marker1,
private) = note
styled_text = [text, []]
markups = sql.query("""select * from markup where handle = ?""", handle)
for markup in markups:
(mhandle,
@ -135,14 +297,156 @@ class SQLReader(object):
markup1,
value,
start_stop_list) = markup
ss_list = eval(start_stop_list)
styled_text[1] += [((markup0, markup1), value, ss_list)]
self.db.note_map[str(handle)] = (str(handle), gid, styled_text,
format, (note_type1, note_type2), change,
(marker0, marker1), private)
count += 1
self.callback(100 * count/total)
# ---------------------------------
# Process person
# ---------------------------------
people = sql.query("""select * from person;""")
for person in people:
if person is None:
continue
(handle, # 0
gid, # 1
gender, # 2
death_ref_handle, # 5
birth_ref_handle, # 6
change, # 17
marker0, # 18
marker1, # 18
private, # 19
) = person
# primary_name, # 3
primary_name = self.package_name(sql, handle, True)
# alternate_names, # 4
alternate_names = self.package_name(sql, handle, False) # list
# event_ref_list, # 7
event_ref_list = self.get_event_refs(sql, "person", handle)
# family_list, # 8
family_list = []
# parent_family_list, # 9
parent_family_list = []
# media_list, # 10
media_list = []
# address_list, # 11
address_list = []
# attribute_list, # 12
attribute_list = []
# urls, # 13
urls = self.get_url_list(sql, "person", handle)
# lds_ord_list, # 14
lds_ord_list = []
# psource_list, # 15
psource_list = self.get_links(sql, "person", handle, "source")
# pnote_list, # 16
pnote_list = []
# person_ref_list, # 20
person_ref_list = []
death_ref_index = lookup(death_ref_handle, event_ref_list)
birth_ref_index = lookup(birth_ref_handle, event_ref_list)
self.db.person_map[str(handle)] = (str(handle), # 0
gid, # 1
gender, # 2
primary_name, # 3
alternate_names, # 4
death_ref_index, # 5
birth_ref_index, # 6
event_ref_list, # 7
family_list, # 8
parent_family_list, # 9
media_list, # 10
address_list, # 11
attribute_list, # 12
urls, # 13
lds_ord_list, # 14
psource_list, # 15
pnote_list, # 16
change, # 17
(marker0, marker1), # 18
private, # 19
person_ref_list, # 20
)
count += 1
self.callback(100 * count/total)
# ---------------------------------
# Process event
# ---------------------------------
events = sql.query("""select * from event;""")
for event in events:
(handle,
gid,
the_type0,
the_type1,
description,
change,
marker0,
marker1,
private) = event
# FIXME:
source_list = self.get_links(sql, "event", handle, "source")
note_list = self.get_links(sql, "event", handle, "note")
media_list = self.get_links(sql, "event", handle, "media")
attribute_list = self.get_links(sql, "event", handle, "attribute")
date = self.get_date(sql, "event", handle)
place = None # self.get_place()
data = (str(handle), gid, (the_type0, the_type1), date, description, place,
source_list, note_list, media_list, attribute_list,
change, (marker0, marker1), private)
self.db.event_map[str(handle)] = data
count += 1
self.callback(100 * count/total)
# ---------------------------------
# Process family
# ---------------------------------
people = sql.query("""select * from family;""")
for person in people:
count += 1
self.callback(100 * count/total)
# ---------------------------------
# Process repository
# ---------------------------------
people = sql.query("""select * from repository;""")
for person in people:
count += 1
self.callback(100 * count/total)
# ---------------------------------
# Process place
# ---------------------------------
people = sql.query("""select * from place;""")
for person in people:
count += 1
self.callback(100 * count/total)
# ---------------------------------
# Process source
# ---------------------------------
people = sql.query("""select * from source;""")
for person in people:
count += 1
self.callback(100 * count/total)
# ---------------------------------
# Process media
# ---------------------------------
people = sql.query("""select * from media;""")
for person in people:
count += 1
self.callback(100 * count/total)
t = time.time() - t
msg = ngettext('Import Complete: %d second','Import Complete: %d seconds', t ) % t
self.db.transaction_commit(self.trans,_("CSV import"))
self.db.transaction_commit(self.trans,_("SQL import"))
self.db.enable_signals()
self.db.request_rebuild()
print msg
progress.close()
return None
def importData(db, filename, callback=None):