932 lines
36 KiB
Python
932 lines
36 KiB
Python
# -*- coding: utf-8 -*-
|
|
#
|
|
# Gramps - a GTK+/GNOME based genealogy program
|
|
#
|
|
# Copyright (C) 2008-2008 Kees Bakker
|
|
#
|
|
# This program is free software; you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License as published by
|
|
# the Free Software Foundation; either version 2 of the License, or
|
|
# (at your option) any later version.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License
|
|
# along with this program; if not, write to the Free Software
|
|
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
|
#
|
|
|
|
# $Id:$
|
|
|
|
"Import from Pro-Gen"
|
|
|
|
#-------------------------------------------------------------------------
|
|
#
|
|
# standard python modules
|
|
#
|
|
#-------------------------------------------------------------------------
|
|
import re
|
|
from gettext import gettext as _
|
|
import os
|
|
import struct
|
|
|
|
import time
|
|
|
|
#------------------------------------------------------------------------
|
|
#
|
|
# Set up logging
|
|
#
|
|
#------------------------------------------------------------------------
|
|
import logging
|
|
log = logging.getLogger('.ImportProGen')
|
|
|
|
#-------------------------------------------------------------------------
|
|
#
|
|
# GRAMPS modules
|
|
#
|
|
#-------------------------------------------------------------------------
|
|
import Errors
|
|
import Utils
|
|
import gen.lib
|
|
import const
|
|
from QuestionDialog import ErrorDialog
|
|
from PluginUtils import register_import
|
|
|
|
|
|
class ProgenError(Exception):
|
|
"""Error used to report Progen errors."""
|
|
def __init__(self, value=""):
|
|
Exception.__init__(self)
|
|
self.value = value
|
|
|
|
def __str__(self):
|
|
return self.value
|
|
|
|
|
|
def _importData(database, filename, cb=None):
|
|
|
|
try:
|
|
g = ProgenParser(database, filename)
|
|
except IOError, msg:
|
|
ErrorDialog(_("%s could not be opened") % filename, str(msg))
|
|
return
|
|
|
|
try:
|
|
status = g.parse_progen_file()
|
|
except ProgenError, msg:
|
|
ErrorDialog(_("Pro-Gen data error"), str(msg))
|
|
return
|
|
except IOError, msg:
|
|
ErrorDialog(_("%s could not be opened") % filename, str(msg))
|
|
return
|
|
|
|
|
|
def _find_from_handle(gramps_id, table):
|
|
"""
|
|
Find a handle corresponding to the specified GRAMPS ID.
|
|
|
|
The passed table contains the mapping. If the value is found, we return
|
|
it, otherwise we create a new handle, store it, and return it.
|
|
|
|
"""
|
|
intid = table.get(gramps_id)
|
|
if not intid:
|
|
intid = Utils.create_id()
|
|
table[gramps_id] = intid
|
|
return intid
|
|
|
|
|
|
def _read_mem(bname):
|
|
'''
|
|
Each record is 32 bytes. First a 4 byte reference to the next record
|
|
followed by 28 bytes of text.
|
|
The information forms a chain of records, that stops when a reference is 0
|
|
or smaller.
|
|
There are two special sequences:
|
|
<ESC> <CR> hard return
|
|
<ESC> <^Z> end of the memo field
|
|
'''
|
|
f = open(bname + '.mem')
|
|
recfmt = "i28s"
|
|
reclen = struct.calcsize( recfmt )
|
|
#print "# reclen = %d" % reclen
|
|
|
|
mems = []
|
|
while 1:
|
|
buf = f.read(reclen)
|
|
if not buf:
|
|
break
|
|
(recno, text) = struct.unpack(recfmt, buf)
|
|
mems.append([recno, text])
|
|
return mems
|
|
|
|
|
|
def _read_recs(table, bname):
|
|
'Read records from .PER or .REL file.'
|
|
f = open(bname + table.fileext)
|
|
recfmt = table.recfmt
|
|
log.info("# %s - recfmt = %s" % (table['name1'], recfmt))
|
|
reclen = struct.calcsize( recfmt )
|
|
log.info("# %s - reclen = %d" % (table['name1'], reclen))
|
|
|
|
recs = []
|
|
while 1:
|
|
buf = f.read(reclen)
|
|
if not buf:
|
|
break
|
|
tups = struct.unpack(recfmt, buf)
|
|
recs.append(tups)
|
|
|
|
log.info("# length %s.recs[] = %d" % (table['name1'], len(recs)))
|
|
print "# length %s.recs[] = %d" % (table['name1'], len(recs))
|
|
return recs
|
|
|
|
|
|
def _get_defname(fname):
|
|
'''
|
|
Get the name of the PG30 DEF file by looking at the user DEF file. And return
|
|
the name of the DEF file. fname is expected to be somewhere in the PG30 tree.
|
|
|
|
Contents of <fname> is something like:
|
|
=> \0
|
|
=> C:\PG30\NL\PG30-1.DEF
|
|
|
|
We will strip the C: and convert the rest to a native pathname. Next, this pathname
|
|
is compared with <fname>.
|
|
'''
|
|
lines = open( fname ).readlines()
|
|
if not lines[0].startswith(r'\0') or len(lines) < 2:
|
|
raise ProgenError(_("Not a Pro-Gen file"))
|
|
return None, '?'
|
|
|
|
defname = lines[1].lower()
|
|
defname = defname.strip()
|
|
# Strip drive, if any
|
|
defname = re.sub( r'^\w:', '', defname )
|
|
defname = defname.replace('\\', os.sep)
|
|
# Strip leading slash, if any.
|
|
if defname.startswith(os.sep):
|
|
defname = defname[1:]
|
|
|
|
# Using the directory of <fname>, go to the parent directory until
|
|
# the DEF is found.
|
|
dir_, f = os.path.split(os.path.abspath(fname))
|
|
while dir_:
|
|
newdefname = os.path.join(dir_, defname)
|
|
if os.path.exists(newdefname):
|
|
return newdefname, defname
|
|
dir_, f = os.path.split(dir_)
|
|
|
|
return None, defname
|
|
|
|
|
|
esc_ctrlz_pat = re.compile(r'\033\032.*')
|
|
def _get_mem_text(mems, i):
|
|
'Notice that Pro-Gen starts the mem numbering at 1.'
|
|
if i <= 0:
|
|
return ''
|
|
|
|
i = i - 1
|
|
recno = mems[i][0]
|
|
text = mems[i][1].decode('cp850')
|
|
if recno != 0:
|
|
text += _get_mem_text(mems, recno)
|
|
# ESC-^M is newline
|
|
text = text.replace('\033\r', '\n')
|
|
# ESC-^Z is end of string
|
|
text = esc_ctrlz_pat.sub('', text)
|
|
|
|
# TODO. Strip trailing empty lines.
|
|
#print text.encode('utf-8')
|
|
return text
|
|
|
|
|
|
|
|
# Example field:
|
|
# ['Voornaam', '47', '64', '4', '2', '15', '""', '""']
|
|
# item 0
|
|
# item 1 is a number indicating the fieldtype
|
|
# item 2
|
|
# item 3 is the size of the field
|
|
class PG30_Def_Table_Field:
|
|
'This class represents a field in one of the tables in the DEF file.'
|
|
def __init__(self, name, value):
|
|
self.fieldname = name
|
|
self.fields = value.split(',')
|
|
self.fields = [p.strip() for p in self.fields]
|
|
self.name = self.fields[0]
|
|
self.type_ = int(self.fields[1])
|
|
self.size = int(self.fields[3])
|
|
|
|
def __repr__(self):
|
|
return self.fieldname + ' -> ' + ', '.join(self.fields)
|
|
|
|
|
|
class PG30_Def_Table:
|
|
'This class represents a table in the DEF file.'
|
|
def __init__(self, name, lines):
|
|
self.name = name
|
|
self.parms = {}
|
|
self.recfmt = None
|
|
# Example line:
|
|
#f02=Persoon gewijzigd ,32,10,10, 1,68,"","INDI CHAN DATE"
|
|
line_pat = re.compile(r'(\w+) = (.*)', re.VERBOSE)
|
|
for l in lines:
|
|
#print l
|
|
m = line_pat.match(l)
|
|
if m:
|
|
# TODO. Catch duplicates?
|
|
self.parms[m.group(1)] = m.group(2)
|
|
|
|
self.fileext = self.parms.get('fileext', None)
|
|
if self.fileext:
|
|
self.fileext = self.fileext.lower()
|
|
#self.name1 = self.parms.get('name1', None)
|
|
|
|
# If there is a n_fields entry then this is a table that
|
|
# has details about the record format of another file (PER or REL).
|
|
if self.parms.has_key('n_fields'):
|
|
self.get_fields()
|
|
self.recfmt = self.get_recfmt()
|
|
self.nam2fld = {}
|
|
self.nam2idx = {}
|
|
self.recflds = [] # list of fields that use up space in a record
|
|
j = 0
|
|
for i, f in enumerate(self.flds):
|
|
#print "# field %s" % f
|
|
nam = f.name
|
|
self.nam2fld[nam] = f
|
|
if f.size != 0:
|
|
self.nam2idx[nam] = j
|
|
#print "# %s <= %d" % (f[0], j)
|
|
self.recflds.append(f)
|
|
j = j + 1
|
|
|
|
def __getitem__(self, i):
|
|
return self.parms.get(i, None)
|
|
|
|
def get_recfmt(self):
|
|
'Get the record format for struct.unpack'
|
|
# Example field:
|
|
# ['Voornaam', '47', '64', '4', '2', '15', '""', '""']
|
|
# item 0
|
|
# item 1 is a number indicating the fieldtype
|
|
# item 2
|
|
# item 3 is the size of the field
|
|
# ...
|
|
flds = self.flds
|
|
fmt = '='
|
|
for f in flds:
|
|
fldtyp = f.type_
|
|
if fldtyp == 2 or fldtyp == 3 or fldtyp == 22 or fldtyp == 23:
|
|
fmt += 'i'
|
|
elif fldtyp == 31:
|
|
pass
|
|
elif fldtyp == 32 or fldtyp == 44 or fldtyp == 45:
|
|
fmt += '%ds' % f.size
|
|
elif fldtyp == 41:
|
|
fmt += 'h'
|
|
elif fldtyp == 42 or fldtyp == 43 or fldtyp == 46 or fldtyp == 47:
|
|
fmt += 'i'
|
|
else:
|
|
pass # ???? Do we want to know?
|
|
return fmt
|
|
|
|
def get_fields(self):
|
|
# For example from pg30-1.def
|
|
#n_fields=58
|
|
#f01=Persoon record ,31, 6, 0, 1,17,"","INDI RFN"
|
|
#f02=Persoon gewijzigd ,32,10,10, 1,68,"","INDI CHAN DATE"
|
|
#f03=Voornaam ,47,64, 4, 2,15,"",""
|
|
|
|
n_fields = int(self.parms['n_fields'])
|
|
flds = []
|
|
for i in range(n_fields):
|
|
fld_name = 'f%02d' % (i+1)
|
|
fld = self.parms.get(fld_name, None)
|
|
flds.append(PG30_Def_Table_Field(fld_name, fld))
|
|
self.flds = flds
|
|
|
|
def get_record_field_index(self, fldname):
|
|
'Return the index number in the record tuple, based on the name.'
|
|
if not fldname in self.nam2idx:
|
|
raise ProgenError(_("Field '%(fldname)s' not found") % locals())
|
|
return self.nam2idx[fldname]
|
|
|
|
def convert_record_to_list(self, rec, mems):
|
|
flds = []
|
|
for i in range(len(rec)):
|
|
if self.field_ix_is_record_number(i):
|
|
flds.append("%d" % rec[i])
|
|
elif self.field_ix_is_mem_type(i):
|
|
flds.append(_get_mem_text(mems, rec[i]))
|
|
else:
|
|
# Not a record number, not a mem number. It must be just text.
|
|
fld = rec[i].strip()
|
|
# Convert to unicode
|
|
fld = fld.decode('cp850')
|
|
flds.append(fld)
|
|
#print ', '.join([f.encode('utf-8') for f in flds])
|
|
return flds
|
|
|
|
def get_field_names(self):
|
|
ret = []
|
|
for f in self.flds:
|
|
if f.size != 0:
|
|
ret.append(f.name)
|
|
return ret
|
|
|
|
def field_is_mem_type(self, fldname):
|
|
if not fldname in self.nam2fld:
|
|
return None
|
|
typ = self.nam2fld[fldname].type_
|
|
if typ == 46 or typ == 47:
|
|
return True
|
|
return False
|
|
|
|
# TODO. Integrate this into field_is_mem_type()
|
|
def field_ix_is_mem_type(self, ix):
|
|
typ = self.recflds[ix].type_
|
|
if typ == 46 or typ == 47:
|
|
return True
|
|
return False
|
|
|
|
def field_ix_is_record_number(self, ix):
|
|
typ = self.recflds[ix].type_
|
|
if typ == 2 or typ == 3 or typ == 22 or typ == 23:
|
|
return True
|
|
return False
|
|
|
|
def diag(self):
|
|
txt = self.name + '\n'
|
|
if self.parms.has_key('n_fields'):
|
|
txt += 'n_fields = %s\n' % self.parms['n_fields']
|
|
# Just grab a field
|
|
f = self.flds[1]
|
|
txt += '"%s"\n' % f
|
|
txt += 'recfmt = %s (length=%d)' % (self.recfmt, struct.calcsize(self.recfmt))
|
|
return txt
|
|
|
|
|
|
class PG30_Def:
|
|
'''
|
|
Utility class to read PG30-1.DEF and to get certain information
|
|
from it.
|
|
|
|
The contents of the DEF file is separated in sections that start
|
|
with [<section name>]. For example:
|
|
[general]
|
|
dateformat=DD-MM-YYYY
|
|
pointerlength=4
|
|
tables=2
|
|
|
|
'''
|
|
def __init__(self, fname):
|
|
#print fname
|
|
fname, deffname = _get_defname(fname)
|
|
if not fname:
|
|
raise ProgenError(_("Cannot find DEF file: %(deffname)s") % locals())
|
|
|
|
# This can throw a IOError
|
|
lines = open(fname).readlines()
|
|
lines = [l.strip() for l in lines]
|
|
content = '\n'.join(lines)
|
|
parts = re.split(r'\n(?=\[)', content)
|
|
self.parts = {}
|
|
self.tables = {}
|
|
for p in parts:
|
|
lines = p.splitlines()
|
|
# Get section name
|
|
k = re.sub(r'\[(.*)\]', r'\1', lines[0])
|
|
# Store section contents in a hashtable using that section name
|
|
self.parts[k] = lines[1:]
|
|
self.tables[k] = PG30_Def_Table(k, self.parts[k])
|
|
|
|
# Some sections are special: Table_1 and Table_2
|
|
|
|
def __getitem__(self, i):
|
|
return self.tables.get(i, None)
|
|
|
|
# TODO. Maybe rename to __repr__
|
|
def diag(self):
|
|
return '\n\n'.join([self.tables[t].diag() for t in self.tables])
|
|
|
|
|
|
class ProgenParser:
|
|
def __init__(self, dbase, file_):
|
|
self.bname, ext = os.path.splitext(file_)
|
|
if ext.lower() != '.def':
|
|
raise ProgenError(_("Expecting a file with .def extension"))
|
|
self.db = dbase
|
|
self.fname = file_
|
|
|
|
self.gid2id = {} # Maps person id
|
|
self.fid2id = {} # Maps family id
|
|
self.fm2fam = {}
|
|
self.pkeys = {} # Caching place handles
|
|
|
|
def parse_progen_file(self):
|
|
self.progress = Utils.ProgressMeter(_("Import from Pro-Gen"), '')
|
|
|
|
self.def_ = PG30_Def(self.fname)
|
|
#print self.def_.diag()
|
|
|
|
self.mems = _read_mem(self.bname)
|
|
self.pers = _read_recs(self.def_['Table_1'], self.bname)
|
|
self.rels = _read_recs(self.def_['Table_2'], self.bname)
|
|
|
|
self.trans = self.db.transaction_begin('', batch=True)
|
|
self.db.disable_signals()
|
|
|
|
self.create_persons()
|
|
self.create_families()
|
|
self.add_children()
|
|
|
|
self.db.transaction_commit(self.trans, _("Pro-Gen import"))
|
|
self.db.enable_signals()
|
|
self.db.request_rebuild()
|
|
self.progress.close()
|
|
|
|
|
|
def __find_person_handle(self, gramps_id):
|
|
"""
|
|
Return the database handle associated with the person's GRAMPS ID
|
|
"""
|
|
return _find_from_handle(gramps_id, self.gid2id)
|
|
|
|
def __find_family_handle(self, gramps_id):
|
|
"""
|
|
Return the database handle associated with the family's GRAMPS ID
|
|
"""
|
|
return _find_from_handle(gramps_id, self.fid2id)
|
|
|
|
def __find_or_create_person(self, gramps_id):
|
|
"""
|
|
Finds or creates a person based on the GRAMPS ID. If the ID is
|
|
already used (is in the db), we return the item in the db. Otherwise,
|
|
we create a new person, assign the handle and GRAMPS ID.
|
|
"""
|
|
person = gen.lib.Person()
|
|
intid = self.gid2id.get(gramps_id)
|
|
if self.db.has_person_handle(intid):
|
|
person.unserialize(self.db.get_raw_person_data(intid))
|
|
else:
|
|
intid = _find_from_handle(gramps_id, self.gid2id)
|
|
person.set_handle(intid)
|
|
person.set_gramps_id(gramps_id)
|
|
return person
|
|
|
|
def __find_or_create_family(self, gramps_id):
|
|
"""
|
|
Finds or creates a family based on the GRAMPS ID. If the ID is
|
|
already used (is in the db), we return the item in the db. Otherwise,
|
|
we create a new family, assign the handle and GRAMPS ID.
|
|
"""
|
|
family = gen.lib.Family()
|
|
intid = self.fid2id.get(gramps_id)
|
|
if self.db.has_family_handle(intid):
|
|
family.unserialize(self.db.get_raw_family_data(intid))
|
|
else:
|
|
intid = _find_from_handle(gramps_id, self.fid2id)
|
|
family.set_handle(intid)
|
|
family.set_gramps_id(gramps_id)
|
|
return family
|
|
|
|
def __get_or_create_place(self, place_name):
|
|
if not place_name:
|
|
return None
|
|
place = None
|
|
if place_name in self.pkeys:
|
|
place = self.db.get_place_from_handle(self.pkeys[place_name])
|
|
else:
|
|
# Create a new Place
|
|
place = gen.lib.Place()
|
|
place.set_title(place_name)
|
|
self.db.add_place(place, self.trans)
|
|
self.db.commit_place(place, self.trans)
|
|
self.pkeys[place_name] = place.get_handle()
|
|
return place
|
|
|
|
def __create_event_and_ref(self, type_, desc=None, date=None, place=None, source=None):
|
|
event = gen.lib.Event()
|
|
event.set_type(gen.lib.EventType(type_))
|
|
if desc:
|
|
event.set_description(desc)
|
|
if date:
|
|
event.set_date_object(date)
|
|
if place:
|
|
event.set_place_handle(place.get_handle())
|
|
if source:
|
|
event.add_source_reference(source)
|
|
self.db.add_event(event,self.trans)
|
|
self.db.commit_event(event,self.trans)
|
|
event_ref = gen.lib.EventRef()
|
|
event_ref.set_reference_handle(event.get_handle())
|
|
return event, event_ref
|
|
|
|
__date_pat1 = re.compile(r'(?P<day>\d{1,2}) (-|=) (?P<month>\d{1,2}) (-|=) (?P<year>\d{2,4})', re.VERBOSE)
|
|
__date_pat2 = re.compile(r'(?P<month>\d{1,2}) (-|=) (?P<year>\d{4})', re.VERBOSE)
|
|
__date_pat3 = re.compile(r'(?P<year>\d{4})', re.VERBOSE)
|
|
__date_pat4 = re.compile(ur'(v|vóór|voor|na|circa|ca|rond) \s* (?P<year>\d{4})', re.VERBOSE)
|
|
__date_pat5 = re.compile(r'(oo) (-|=) (oo) (-|=) (?P<year>\d{2,4})', re.VERBOSE)
|
|
def __create_date_from_text(self, txt, diag_msg=None):
|
|
'''
|
|
Pro-Gen has a text field for the date. It can be anything. Mostly it will be dd-mm-yyyy,
|
|
but we have seen:
|
|
yyyy
|
|
mm-yyyy
|
|
voor yyyy
|
|
dd=mm-yyyy (typo I guess)
|
|
00-00-yyyy
|
|
oo-oo-yyyy
|
|
dd-mm-00 (does this mean we do not know about the year?)
|
|
|
|
This function tries to parse the text and create a proper Gramps Date() object.
|
|
If all else fails we create a MOD_TEXTONLY Date() object.
|
|
'''
|
|
date = gen.lib.Date()
|
|
|
|
# dd-mm-yyyy
|
|
m = self.__date_pat1.match(txt)
|
|
if m:
|
|
day = int(m.group('day'))
|
|
month = int(m.group('month'))
|
|
year = int(m.group('year'))
|
|
if day and month and year:
|
|
date.set_yr_mon_day(year, month, day)
|
|
else:
|
|
date.set(gen.lib.Date.QUAL_NONE, gen.lib.Date.MOD_ABOUT, gen.lib.Date.CAL_GREGORIAN, (day, month, year, None))
|
|
return date
|
|
|
|
# mm-yyyy
|
|
m = self.__date_pat2.match(txt)
|
|
if m:
|
|
month = int(m.group('month'))
|
|
year = int(m.group('year'))
|
|
date.set(gen.lib.Date.QUAL_NONE, gen.lib.Date.MOD_ABOUT, gen.lib.Date.CAL_GREGORIAN, (0, month, year, None))
|
|
return date
|
|
|
|
# yyyy
|
|
m = self.__date_pat3.match(txt)
|
|
if m:
|
|
year = int(m.group('year'))
|
|
date.set(gen.lib.Date.QUAL_NONE, gen.lib.Date.MOD_ABOUT, gen.lib.Date.CAL_GREGORIAN, (0, 0, year, None))
|
|
return date
|
|
|
|
# voor|na|... yyyy
|
|
m = self.__date_pat4.match(txt)
|
|
if m:
|
|
year = int(m.group('year'))
|
|
if m.group(1) == 'voor' or m.group(1) == 'v' or m.group(1) == u'vóór':
|
|
date.set(gen.lib.Date.QUAL_NONE, gen.lib.Date.MOD_BEFORE, gen.lib.Date.CAL_GREGORIAN, (0, 0, year, None))
|
|
elif m.group(1) == 'na':
|
|
date.set(gen.lib.Date.QUAL_NONE, gen.lib.Date.MOD_AFTER, gen.lib.Date.CAL_GREGORIAN, (0, 0, year, None))
|
|
else:
|
|
date.set(gen.lib.Date.QUAL_NONE, gen.lib.Date.MOD_ABOUT, gen.lib.Date.CAL_GREGORIAN, (0, 0, year, None))
|
|
return date
|
|
|
|
# oo-oo-yyyy
|
|
m = self.__date_pat5.match(txt)
|
|
if m:
|
|
year = int(m.group('year'))
|
|
date.set(gen.lib.Date.QUAL_NONE, gen.lib.Date.MOD_ABOUT, gen.lib.Date.CAL_GREGORIAN, (0, 0, year, None))
|
|
return date
|
|
|
|
log.warning(_("date did not match: '%s' (%s)") % (txt.encode('utf-8'), diag_msg or ''))
|
|
# Hmmm. Just use the plain text.
|
|
date.set_as_text(txt)
|
|
return date
|
|
|
|
def create_persons(self):
|
|
self.progress.set_pass(_('Importing individuals'), len(self.pers))
|
|
table = self.def_['Table_1']
|
|
|
|
# Records in the PER file using PG30-1.DEF contain the following fields:
|
|
# (Note. We want this to be computed just once.
|
|
|
|
#log.info(table.get_field_names())
|
|
|
|
# I'm sure I can find a better way to do this through the local() dict)
|
|
first_name_ix = table.get_record_field_index('Voornaam')
|
|
surname_ix = table.get_record_field_index('Achternaam')
|
|
gender_ix = table.get_record_field_index('Geslacht')
|
|
patron_ix = table.get_record_field_index('Patroniem')
|
|
call_name_ix = table.get_record_field_index('Roepnaam')
|
|
alias_ix = table.get_record_field_index('Alias')
|
|
percode_ix = table.get_record_field_index('Persoon code')
|
|
title1_ix = table.get_record_field_index('Titel1')
|
|
title2_ix = table.get_record_field_index('Titel2')
|
|
title3_ix = table.get_record_field_index('Titel3')
|
|
father_ix = table.get_record_field_index('Vader')
|
|
mother_ix = table.get_record_field_index('Moeder')
|
|
occu_ix = table.get_record_field_index('Beroep')
|
|
|
|
per_klad_ix = table.get_record_field_index('Persoon klad')
|
|
per_info_ix = table.get_record_field_index('Persoon info')
|
|
|
|
addr_date_ix = table.get_record_field_index('Adres datum')
|
|
addr_street_ix = table.get_record_field_index('Adres straat')
|
|
addr_postal_ix = table.get_record_field_index('Adres postcode')
|
|
addr_place_ix = table.get_record_field_index('Adres plaats')
|
|
addr_country_ix = table.get_record_field_index('Adres land')
|
|
addr_tel_ix = table.get_record_field_index('Adres telefoon')
|
|
addr_info_ix = table.get_record_field_index('Adres info')
|
|
|
|
birth_date_ix = table.get_record_field_index('Geboorte datum')
|
|
birth_place_ix = table.get_record_field_index('Geboorte plaats')
|
|
birth_time_ix = table.get_record_field_index('Geboorte tijd')
|
|
birth_source_ix = table.get_record_field_index('Geboorte bron')
|
|
birth_cert_ix = table.get_record_field_index('Geboorte akte')
|
|
birth_source_text_ix = table.get_record_field_index('Geboorte brontekst')
|
|
birth_info_ix = table.get_record_field_index('Geboorte info')
|
|
|
|
bapt_date_ix = table.get_record_field_index('Doop datum')
|
|
bapt_place_ix = table.get_record_field_index('Doop plaats')
|
|
bapt_reli_ix = table.get_record_field_index('Gezindte')
|
|
bapt_wit_ix = table.get_record_field_index('Doop getuigen')
|
|
bapt_source_ix = table.get_record_field_index('Doop bron')
|
|
bapt_cert_ix = table.get_record_field_index('Doop akte')
|
|
bapt_source_text_ix = table.get_record_field_index('Doop brontekst')
|
|
bapt_info_ix = table.get_record_field_index('Doop info')
|
|
|
|
death_date_ix = table.get_record_field_index('Overlijden datum')
|
|
death_place_ix = table.get_record_field_index('Overlijden plaats')
|
|
death_time_ix = table.get_record_field_index('Overlijden tijd')
|
|
death_source_ix = table.get_record_field_index('Overlijden bron')
|
|
death_cert_ix = table.get_record_field_index('Overlijden akte')
|
|
death_source_text_ix = table.get_record_field_index('Overlijden brontekst')
|
|
death_info_ix = table.get_record_field_index('Overlijden info')
|
|
|
|
crem_date_ix = table.get_record_field_index('Crematie datum')
|
|
crem_place_ix = table.get_record_field_index('Crematie plaats')
|
|
crem_source_ix = table.get_record_field_index('Crematie bron')
|
|
crem_cert_ix = table.get_record_field_index('Crematie akte')
|
|
crem_source_text_ix = table.get_record_field_index('Crematie brontekst')
|
|
crem_info_ix = table.get_record_field_index('Crematie info')
|
|
|
|
bur_date_ix = table.get_record_field_index('Begrafenis datum')
|
|
bur_place_ix = table.get_record_field_index('Begrafenis plaats')
|
|
bur_source_ix = table.get_record_field_index('Begrafenis bron')
|
|
bur_cert_ix = table.get_record_field_index('Begrafenis akte')
|
|
bur_source_text_ix = table.get_record_field_index('Begrafenis brontekst')
|
|
bur_info_ix = table.get_record_field_index('Begrafenis info')
|
|
|
|
# The records are numbered 1..N
|
|
for i, rec in enumerate(self.pers):
|
|
pers_id = i + 1
|
|
father = rec[father_ix]
|
|
mother = rec[mother_ix]
|
|
if father >= 0 and mother >= 0:
|
|
recflds = table.convert_record_to_list(rec, self.mems)
|
|
|
|
first_name = recflds[first_name_ix]
|
|
surname = recflds[surname_ix]
|
|
gender = recflds[gender_ix]
|
|
if gender == 'M':
|
|
gender = gen.lib.Person.MALE
|
|
elif gender == 'V':
|
|
gender = gen.lib.Person.FEMALE
|
|
else:
|
|
gender = gen.lib.Person.UNKNOWN
|
|
|
|
person = self.__find_or_create_person("I%d" % pers_id)
|
|
diag_msg = "I%d: %s %s" % (pers_id, first_name.encode('utf-8'), surname.encode('utf-8'))
|
|
#log.info(diag_msg)
|
|
|
|
name = gen.lib.Name()
|
|
name.set_type(gen.lib.NameType.BIRTH)
|
|
name.set_surname(surname)
|
|
name.set_first_name(first_name)
|
|
if recflds[call_name_ix]:
|
|
name.set_call_name(recflds[call_name_ix])
|
|
person.set_primary_name(name)
|
|
person.set_gender(gender)
|
|
|
|
if recflds[alias_ix]:
|
|
# Alias skipped. Not sure if it is used.
|
|
# TODO. Log it
|
|
pass
|
|
|
|
if recflds[patron_ix]:
|
|
# Patronym skipped. Not sure if it is used.
|
|
# TODO. Log it
|
|
pass
|
|
|
|
if recflds[percode_ix]:
|
|
# Person code skipped. Not sure if it is used.
|
|
# TODO. Log it
|
|
pass
|
|
|
|
if recflds[occu_ix]:
|
|
event, event_ref = self.__create_event_and_ref(gen.lib.EventType.OCCUPATION, recflds[occu_ix])
|
|
person.add_event_ref(event_ref)
|
|
|
|
if recflds[birth_date_ix]:
|
|
place = self.__get_or_create_place(recflds[birth_place_ix])
|
|
time = recflds[birth_time_ix]
|
|
source = recflds[birth_source_ix]
|
|
cert = recflds[birth_cert_ix]
|
|
source_text = recflds[birth_source_text_ix]
|
|
info = recflds[birth_info_ix]
|
|
|
|
date = self.__create_date_from_text(recflds[birth_date_ix], diag_msg)
|
|
event, birth_ref = self.__create_event_and_ref(gen.lib.EventType.BIRTH, info, date, place)
|
|
person.set_birth_ref(birth_ref)
|
|
|
|
if recflds[bapt_date_ix]:
|
|
place = self.__get_or_create_place(recflds[bapt_place_ix])
|
|
reli = recflds[bapt_reli_ix]
|
|
witness = recflds[bapt_wit_ix]
|
|
source = recflds[bapt_source_ix]
|
|
cert = recflds[bapt_cert_ix]
|
|
source_text = recflds[bapt_source_text_ix]
|
|
info = recflds[bapt_info_ix]
|
|
|
|
date = self.__create_date_from_text(recflds[bapt_date_ix], diag_msg)
|
|
event, bapt_ref = self.__create_event_and_ref(gen.lib.EventType.BAPTISM, info, date, place)
|
|
# ???? reli
|
|
# ???? witness
|
|
person.add_event_ref(bapt_ref)
|
|
|
|
if recflds[death_date_ix]:
|
|
place = self.__get_or_create_place(recflds[death_place_ix])
|
|
time = recflds[death_time_ix]
|
|
source = recflds[death_source_ix]
|
|
cert = recflds[death_cert_ix]
|
|
source_text = recflds[death_source_text_ix]
|
|
info = recflds[death_info_ix]
|
|
|
|
date = self.__create_date_from_text(recflds[death_date_ix], diag_msg)
|
|
event, death_ref = self.__create_event_and_ref(gen.lib.EventType.DEATH, info, date, place)
|
|
person.set_death_ref(death_ref)
|
|
|
|
if recflds[bur_date_ix]:
|
|
place = self.__get_or_create_place(recflds[bur_place_ix])
|
|
source = recflds[bur_source_ix]
|
|
cert = recflds[bur_cert_ix]
|
|
source_text = recflds[bur_source_text_ix]
|
|
info = recflds[bur_info_ix]
|
|
|
|
date = self.__create_date_from_text(recflds[bur_date_ix], diag_msg)
|
|
event, burial_ref = self.__create_event_and_ref(gen.lib.EventType.BURIAL, info, date, place)
|
|
person.add_event_ref(burial_ref)
|
|
|
|
elif recflds[crem_date_ix]:
|
|
place = self.__get_or_create_place(recflds[crem_place_ix])
|
|
source = recflds[crem_source_ix]
|
|
cert = recflds[crem_cert_ix]
|
|
source_text = recflds[crem_source_text_ix]
|
|
info = recflds[crem_info_ix]
|
|
|
|
date = self.__create_date_from_text(recflds[crem_date_ix], diag_msg)
|
|
event, cremation_ref = self.__create_event_and_ref(gen.lib.EventType.CREMATION, info, date, place)
|
|
person.add_event_ref(cremation_ref)
|
|
|
|
self.db.commit_person(person, self.trans)
|
|
self.progress.step()
|
|
|
|
def create_families(self):
|
|
self.progress.set_pass(_('Importing families'), len(self.rels))
|
|
table = self.def_['Table_2']
|
|
|
|
# Records in the REL file using PG30-1.DEF contain the following fields:
|
|
# (Note. We want this to be computed just once.
|
|
|
|
#log.info(table.get_field_names())
|
|
|
|
man_ix = table.get_record_field_index('Man')
|
|
vrouw_ix = table.get_record_field_index('Vrouw')
|
|
f05 = table.get_record_field_index('Relatie code')
|
|
f06 = table.get_record_field_index('Relatie klad')
|
|
f07 = table.get_record_field_index('Relatie info')
|
|
|
|
f08 = table.get_record_field_index('Samenwonen datum')
|
|
f09 = table.get_record_field_index('Samenwonen plaats')
|
|
f10 = table.get_record_field_index('Samenwonen bron')
|
|
f11 = table.get_record_field_index('Samenwonen akte')
|
|
f12 = table.get_record_field_index('Samenwonen brontekst')
|
|
f13 = table.get_record_field_index('Samenwonen info')
|
|
|
|
f14 = table.get_record_field_index('Ondertrouw datum')
|
|
f15 = table.get_record_field_index('Ondertrouw plaats')
|
|
f16 = table.get_record_field_index('Ondertrouw getuigen')
|
|
f17 = table.get_record_field_index('Ondertrouw bron')
|
|
f18 = table.get_record_field_index('Ondertrouw akte')
|
|
f19 = table.get_record_field_index('Ondertrouw brontekst')
|
|
f20 = table.get_record_field_index('Ondertrouw info')
|
|
|
|
f21 = table.get_record_field_index('Wettelijk datum')
|
|
f22 = table.get_record_field_index('Wettelijk plaats')
|
|
f23 = table.get_record_field_index('Wettelijk getuigen')
|
|
f24 = table.get_record_field_index('Wettelijk bron')
|
|
f25 = table.get_record_field_index('Wettelijk akte')
|
|
f26 = table.get_record_field_index('Wettelijk brontekst')
|
|
f27 = table.get_record_field_index('Wettelijk info')
|
|
|
|
f28 = table.get_record_field_index('Kerkelijk datum')
|
|
f29 = table.get_record_field_index('Kerkelijk plaats')
|
|
f30 = table.get_record_field_index('Kerk')
|
|
f31 = table.get_record_field_index('Kerkelijk getuigen')
|
|
f32 = table.get_record_field_index('Kerkelijk bron')
|
|
f33 = table.get_record_field_index('Kerkelijk akte')
|
|
f34 = table.get_record_field_index('Kerkelijk brontekst')
|
|
f35 = table.get_record_field_index('Kerkelijk info')
|
|
|
|
f36 = table.get_record_field_index('Scheiding datum')
|
|
f37 = table.get_record_field_index('Scheiding plaats')
|
|
f38 = table.get_record_field_index('Scheiding bron')
|
|
f39 = table.get_record_field_index('Scheiding akte')
|
|
f40 = table.get_record_field_index('Scheiding brontekst')
|
|
f41 = table.get_record_field_index('Scheiding info')
|
|
|
|
# The records are numbered 1..N
|
|
self.progress.set_pass(_('Adding families'), len(self.rels))
|
|
for i, rec in enumerate(self.rels):
|
|
fam_id = i + 1
|
|
husband = rec[man_ix]
|
|
wife = rec[vrouw_ix]
|
|
if husband > 0 or wife > 0:
|
|
self.highest_fam_id = fam_id
|
|
fam = self.__find_or_create_family("F%d" % fam_id)
|
|
husband_handle = None
|
|
if husband > 0:
|
|
husband_handle = self.__find_person_handle("I%d" % husband)
|
|
fam.set_father_handle(husband_handle)
|
|
husband_person = self.db.get_person_from_handle(husband_handle)
|
|
husband_person.add_family_handle(fam.get_handle())
|
|
self.db.commit_person(husband_person, self.trans)
|
|
wife_handle = None
|
|
if wife > 0:
|
|
wife_handle = self.__find_person_handle("I%d" % wife)
|
|
fam.set_mother_handle(wife_handle)
|
|
wife_person = self.db.get_person_from_handle(wife_handle)
|
|
wife_person.add_family_handle(fam.get_handle())
|
|
self.db.commit_person(wife_person, self.trans)
|
|
self.fm2fam[husband_handle, wife_handle] = fam
|
|
self.db.commit_family(fam, self.trans)
|
|
|
|
self.progress.step()
|
|
|
|
def add_children(self):
|
|
# Once more to record the father and mother
|
|
table = self.def_['Table_1']
|
|
self.progress.set_pass(_('Adding children'), len(self.pers))
|
|
|
|
father_ix = table.get_record_field_index('Vader')
|
|
mother_ix = table.get_record_field_index('Moeder')
|
|
|
|
# The records are numbered 1..N
|
|
for i, rec in enumerate(self.pers):
|
|
pers_id = i + 1
|
|
father = rec[father_ix]
|
|
mother = rec[mother_ix]
|
|
if father > 0 or mother > 0:
|
|
# Find the family with this father and mother
|
|
person_handle = self.__find_person_handle("I%d" % pers_id)
|
|
father_handle = father > 0 and self.__find_person_handle("I%d" % father) or None
|
|
mother_handle = mother > 0 and self.__find_person_handle("I%d" % mother) or None
|
|
if father > 0 and not father_handle:
|
|
log.warning(_("cannot find father for I%s (father=%d)") % (pers_id, father))
|
|
elif mother > 0 and not mother_handle:
|
|
log.warning(_("cannot find mother for I%s (mother=%d)") % (pers_id, mother))
|
|
else:
|
|
fam = self.fm2fam.get((father_handle, mother_handle), None)
|
|
if not fam:
|
|
# Family not present in REL. Create a new one.
|
|
self.highest_fam_id = self.highest_fam_id + 1
|
|
fam_id = self.highest_fam_id
|
|
fam = self.__find_or_create_family("F%d" % fam_id)
|
|
if father_handle:
|
|
fam.set_father_handle(father_handle)
|
|
father_person = self.db.get_person_from_handle(father_handle)
|
|
father_person.add_family_handle(fam.get_handle())
|
|
self.db.commit_person(father_person, self.trans)
|
|
if mother_handle:
|
|
fam.set_mother_handle(mother_handle)
|
|
mother_person = self.db.get_person_from_handle(mother_handle)
|
|
mother_person.add_family_handle(fam.get_handle())
|
|
self.db.commit_person(mother_person, self.trans)
|
|
|
|
if fam:
|
|
childref = gen.lib.ChildRef()
|
|
childref.set_reference_handle(person_handle)
|
|
fam.add_child_ref(childref)
|
|
self.db.commit_family(fam, self.trans)
|
|
person = self.db.get_person_from_handle(person_handle)
|
|
person.add_parent_family_handle(fam.get_handle())
|
|
self.db.commit_person(person, self.trans)
|
|
self.progress.step()
|
|
|
|
|
|
_mime_type = "application/x-progen"
|
|
import gtk
|
|
_filter = gtk.FileFilter()
|
|
_filter.set_name(_('Pro-Gen files'))
|
|
_filter.add_mime_type(_mime_type)
|
|
_format_name = _('Pro-Gen')
|
|
|
|
register_import(_importData, _filter, [_mime_type], 0, _format_name)
|