Cyclone-Team/gramps
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #326 from prculley/progen_prog

Browse Source 
Progen import fixes and testing
This commit is contained in:
Sam Manzi 2017-01-26 15:58:29 +11:00 committed by GitHub
commit 2cb8c2801a
15 changed files with 1589 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitattributes vendored
View File

@ -13,3 +13,6 @@
*.ged -text
*.GED -text
*.csv -text
*.DEF -text
*.IXI -text
*.REM -text

236
data/tests/PG30-1GB.DEF Normal file
View File

@ -0,0 +1,236 @@
[PRO-GEN]
version=3.0b
type=def
format=2
[general]
dateformat=DD/MM/YYYY
pointerlength=4
tables=2
[Table_1]
name1=Person
name2=Persons
fileext=.PER
indexext=.IXP
n_lines=42
hiddenlines=4,5,6,7,8,10,12,13,14,15,19,25,30,32,33,34,35,36,40
t01=É͵INDIVIDUAL ÆÍÍÍÍÍÍÍÍÍÍÍÍÍ͵pg30-1gbÆÍÍÍÍÍÍÍÍ͵LAST CHANGE ÆÍ»
t02=º ÛGiven name: º
t03=º Surname : Sex: º
t04=º Patronym : º
t05=º Call name : Alias: Code: º
t06=º Title 1 : Title 2: Title 3: º
t07=º Father : º
t08=º Mother : º
t09=º Occupation: º
t10=º Scratch : º
t11=º Info : º
t12=ÇÄ´ÛAddressÃÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄĶ
t13=º Date : Street: º
t14=º Zip : Place: Country: º
t15=º Phone : Info: º
t16=ÇÄ´ÛBirthÃÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄĶ
t17=º Date : Place: Time: º
t18=º Source: Reference: º
t19=º Text : º
t20=º Info : º
t21=ÇÄ´ÛChristeningÃÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄĶ
t22=º Date : Place: Religion: º
t23=º Witn. : º
t24=º Source: Reference: º
t25=º Text : º
t26=º Info : º
t27=ÇÄ´ÛDeathÃÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄĶ
t28=º Date : Place: Time: º
t29=º Source: Reference: º
t30=º Text : º
t31=º Info : º
t32=ÇÄ´CÛremationÃÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄĶ
t33=º Date : Place: º
t34=º Source: Reference: º
t35=º Text : º
t36=º Info : º
t37=ÇÄ´BÛurialÃÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄĶ
t38=º Date : Place: º
t39=º Source: Reference: º
t40=º Text : º
t41=º Info : º
t42=ÈÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍͼ
n_fields=58
f01=Person_record ,31, 6, 0, 1,15,"","INDI RFN"
f02=Person_last_change ,32,10,10, 1,68,"","INDI CHAN DATE"
f03=Given_name ,47,64, 4, 2,15,"",""
f04=Surname ,47,40, 4, 3,15,"","INDI NAME"
f05=Sex ,45, 1, 1, 3,67,"MMFFmmff??","INDI SEX"
f06=Patronym ,47,40, 4, 4,15,"","INDI _PATR"
f07=Call_name ,47,18, 4, 5,15,"","INDI NAME NICK/INDI NAME ALIA/INDI CHR NICK"
f08=Alias ,47,19, 4, 5,41,"","INDI NAME _ALIA/INDI NAME COMM"
f09=Person_code ,47,12, 4, 5,67,"","INDI REFN/INDI CODE"
f10=Title1 ,47,16, 4, 6,15,"","INDI TITL"
f11=Title2 ,47,16, 4, 6,41,"","INDI _TITL2"
f12=Title3 ,47,12, 4, 6,67,"","INDI _TITL3"
f13=Father , 2,64, 4, 7,15,"",""
f14=Mother , 3,64, 4, 8,15,"",""
f15=Occupation ,47,64, 4, 9,15,"","INDI OCCU"
f16=Person_scratch ,46,64, 4,10,15,"","INDI _COMM/INDI COMM"
f17=Person_info ,46,64, 4,11,15,"","INDI NOTE"
f18=Address_date ,44,10,10,13,11,"","INDI RESI DATE"
f19=Address_street ,47,49, 4,13,30,"","INDI RESI ADDR"
f20=Address_zip ,47,11, 4,14,11,"","INDI RESI ADDR POST/INDI RESI POST"
f21=Address_place ,47,27, 4,14,30,"","INDI RESI ADDR CITY/INDI RESI PLAC"
f22=Address_country ,47,12, 4,14,67,"","INDI RESI ADDR CTRY/INDI RESI CTRY"
f23=Address_phone ,47,12, 4,15,11,"","INDI RESI PHON/INDI PHON"
f24=Address_info ,46,49, 4,15,30,"","INDI RESI NOTE/INDI ADDR"
f25=Birth_date ,44,10,10,17,11,"","INDI BIRT DATE"
f26=Birth_place ,47,30, 4,17,30,"","INDI BIRT PLAC"
f27=Birth_time ,45, 5, 5,17,67,"09::..","INDI BIRT TIME"
f28=Birth_source ,47,44, 4,18,11,"","INDI BIRT SOUR/INDI BIRT SOUR TITL"
f29=Birth_ref ,45,12,12,18,67,"","INDI BIRT SOUR REFN"
f30=Birth_text ,46,68, 4,19,11,"","INDI BIRT SOUR TEXT"
f31=Birth_info ,46,68, 4,20,11,"","INDI BIRT NOTE"
f32=Christening_date ,44,10,10,22,11,"","INDI CHR DATE"
f33=Christening_place ,47,26, 4,22,30,"","INDI CHR PLAC"
f34=Religion ,47,12, 4,22,67,"","INDI CHR RELI/INDI RELI"
f35=Christening_witness ,47,68, 4,23,11,"","INDI CHR _WITN/INDI CHR WITN"
f36=Christening_source ,47,44, 4,24,11,"","INDI CHR SOUR/INDI CHR SOUR TITL"
f37=Christening_ref ,45,12,12,24,67,"","INDI CHR SOUR REFN"
f38=Christening_text ,46,68, 4,25,11,"","INDI CHR SOUR TEXT"
f39=Christening_info ,46,68, 4,26,11,"","INDI CHR NOTE"
f40=Death_date ,44,10,10,28,11,"","INDI DEAT DATE"
f41=Death_place ,47,30, 4,28,30,"","INDI DEAT PLAC"
f42=Death_time ,45, 5, 5,28,67,"09::..","INDI DEAT TIME"
f43=Death_source ,47,44, 4,29,11,"","INDI DEAT SOUR/INDI DEAT SOUR TITL"
f44=Death_ref ,45,12,12,29,67,"","INDI DEAT SOUR REFN"
f45=Death_text ,46,68, 4,30,11,"","INDI DEAT SOUR TEXT"
f46=Death_info ,46,68, 4,31,11,"","INDI DEAT NOTE"
f47=Cremation_date ,44,10,10,33,11,"","INDI CREM DATE"
f48=Cremation_place ,47,49, 4,33,30,"","INDI CREM PLAC"
f49=Cremation_source ,47,44, 4,34,11,"","INDI CREM SOUR/INDI CREM SOUR TITL"
f50=Cremation_ref ,45,12,12,34,67,"","INDI CREM SOUR REFN"
f51=Cremation_text ,46,68, 4,35,11,"","INDI CREM SOUR TEXT"
f52=Cremation_info ,46,68, 4,36,11,"","INDI CREM NOTE"
f53=Burial_date ,44,10,10,38,11,"","INDI BURI DATE"
f54=Burial_place ,47,49, 4,38,30,"","INDI BURI PLAC"
f55=Burial_source ,47,44, 4,39,11,"","INDI BURI SOUR/INDI BURI SOUR TITL"
f56=Burial_ref ,45,12,12,39,67,"","INDI BURI SOUR REFN"
f57=Burial_text ,46,68, 4,40,11,"","INDI BURI SOUR TEXT"
f58=Burial_info ,46,68, 4,41,11,"","INDI BURI NOTE"
n_replace=4
r01=Surname,Patronym
r02=Birth_date,Christening_date/Birth_place,Christening_place
r03=Death_date,Burial_date/Death_place,Burial_place
r04=Burial_date,Cremation_date/Burial_place,Cremation_place
[Table_2]
name1=Marriage
name2=Marriages
fileext=.REL
indexext=.IXR
n_lines=35
hiddenlines=5,7,8,9,10,11,16,22,28,30,31,32,33,34
t01=É͵MARRIAGE ÆÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍ͵pg30-1gbÆÍÍÍÍÍÍÍÍ͵LAST CHANGE ÆÍ»
t02=º ÛHusband: º
t03=º Wife : º
t04=º Code : º
t05=º Scratch: º
t06=º Info : º
t07=ÇÄ´ÛLiving togetherÃÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄĶ
t08=º Date : Place: º
t09=º Source: Reference: º
t10=º Text : º
t11=º Info : º
t12=ÇÄ´Publication of the ÛbannsÃÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄĶ
t13=º Date : Place: º
t14=º Witn. : º
t15=º Source: Reference: º
t16=º Text : º
t17=º Info : º
t18=ÇÄ´Civil ÛmarriageÃÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄĶ
t19=º Date : Place: º
t20=º Witn. : º
t21=º Source: Reference: º
t22=º Text : º
t23=º Info : º
t24=ÇÄ´ÛChurch marriageÃÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄĶ
t25=º Date : Place: Church: º
t26=º Witn. : º
t27=º Source: Reference: º
t28=º Text : º
t29=º Info : º
t30=ÇÄ´ÛDivorceÃÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄÄĶ
t31=º Date : Place: º
t32=º Source: Reference: º
t33=º Text : º
t34=º Info : º
t35=ÈÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍÍͼ
n_fields=41
f01=Relation_record ,31, 6, 0, 1,13,"","FAM RFN"
f02=Relation_last_change,32,10,10, 1,68,"","FAM CHAN DATE"
f03=Husband ,22,67, 4, 2,12,"","FAM HUSB"
f04=Wife ,23,67, 4, 3,12,"","FAM WIFE"
f05=Relation_code ,47,12, 4, 4,12,"","FAM REFN/FAM CODE"
f06=Relation_scratch ,46,67, 4, 5,12,"","FAM _COMM/FAM COMM"
f07=Relation_info ,46,67, 4, 6,12,"","FAM NOTE"
f08=Living_date ,44,10,10, 8,11,"","FAM _LIV DATE"
f09=Living_place ,47,49, 4, 8,30,"","FAM _LIV PLAC"
f10=Living_source ,47,44, 4, 9,11,"","FAM _LIV SOUR/FAM _LIV SOUR TITL"
f11=Living_ref ,45,12,12, 9,67,"","FAM _LIV SOUR REFN"
f12=Living_text ,46,68, 4,10,11,"","FAM _LIV SOUR TEXT"
f13=Living_info ,46,68, 4,11,11,"","FAM _LIV NOTE"
f14=Banns_date ,44,10,10,13,11,"","FAM MARB DATE/FAM REGS DATE"
f15=Banns_place ,47,49, 4,13,30,"","FAM MARB PLAC/FAM REGS PLAC"
f16=banns_witnesses ,47,68, 4,14,11,"","FAM MARB _WITN/FAM MARB WITN"
f17=Banns_source ,47,44, 4,15,11,"","FAM MARB SOUR/FAM MARB SOUR TITL/FAM REGS SOUR"
f18=Banns_ref ,45,12,12,15,67,"","FAM MARB SOUR REFN/FAM REGS SOUR REFN"
f19=Banns_text ,46,68, 4,16,11,"","FAM MARB SOUR TEXT"
f20=Banns_info ,46,68, 4,17,11,"","FAM MARB NOTE"
f21=Civil_date ,44,10,10,19,11,"","FAM MARR(Civil) DATE/FAM MARR DATE"
f22=Civil_place ,47,49, 4,19,30,"","FAM MARR(Civil) PLAC/FAM MARR PLAC"
f23=Civil_witnesses ,47,68, 4,20,11,"","FAM MARR(Civil) _WITN/FAM MARR _WITN/FAM MARR WITN/FAM WITN"
f24=Civil_source ,47,44, 4,21,11,"","FAM MARR(Civil) SOUR/FAM MARR SOUR/FAM MARR SOUR TITL"
f25=Civil_ref ,45,12,12,21,67,"","FAM MARR(Civil) SOUR REFN/FAM MARR SOUR REFN"
f26=Civil_text ,46,68, 4,22,11,"","FAM MARR(Civil) SOUR TEXT/FAM MARR SOUR TEXT"
f27=Civil_info ,46,68, 4,23,11,"","FAM MARR(Civil) NOTE/FAM MARR NOTE"
f28=Church_date ,44,10,10,25,11,"","FAM MARR(Church) DATE/FAM ORDI DATE"
f29=Church_place ,47,28, 4,25,30,"","FAM MARR(Church) PLAC/FAM ORDI PLAC"
f30=Church ,47,12, 4,25,67,"","FAM MARR(Church) _CHUR/FAM ORDI _CHUR/FAM ORDI RELI"
f31=Church_witnesses ,47,68, 4,26,11,"","FAM MARR(Church) _WITN/FAM ORDI _WITN/FAM ORDI WITN"
f32=Church_source ,47,44, 4,27,11,"","FAM MARR(Church) SOUR/FAM ORDI SOUR/FAM ORDI SOUR TITL"
f33=Church_ref ,45,12,12,27,67,"","FAM MARR(Church) SOUR REFN/FAM ORDI SOUR REFN"
f34=Church_text ,46,68, 4,28,11,"","FAM MARR(Church) SOUR TEXT/FAM ORDI SOUR TEXT"
f35=Church_info ,46,68, 4,29,11,"","FAM MARR(Church) NOTE/FAM ORDI NOTE"
f36=Divorce_date ,44,10,10,31,11,"","FAM DIV DATE/FAM DIVO DATE"
f37=Divorce_place ,47,49, 4,31,30,"","FAM DIV PLAC/FAM DIVO PLAC"
f38=Divorce_source ,47,44, 4,32,11,"","FAM DIV SOUR/FAM DIV SOUR TITL"
f39=Divorce_ref ,45,12,12,32,67,"","FAM DIV SOUR REFN"
f40=Divorce_text ,46,68, 4,33,11,"","FAM DIV SOUR TEXT"
f41=Divorce_info ,46,68, 4,34,11,"","FAM DIV NOTE"
n_replace=3
r01=Banns_date,Living_date/Banns_place,Living_place
r02=Civil_date,Church_date/Civil_place,Church_place
r03=Church_date,Banns_date/Church_place,Banns_place
[Genealogical]
male=M
female=F
function_father=[FATHER]
function_mother=[MOTHER]
function_husband=[HUSBAND]
function_wife=[WIFE]
function_age=[AGE]
function_initials=[INITIALS]
field_father=Father
field_mother=Mother
field_givenname=Given_name
field_surname=Surname
field_sex=Sex
field_birthdate=Birth_date
field_christeningdate=Christening_date
field_deathdate=Death_date
field_burialdate=Burial_date
field_husband=Husband
field_wife=Wife
field_marriagedate=Civil_date
field_churchdate=Church_date
field_divorcedate=Divorce_date

2
data/tests/SAMPLE.DEF Normal file
View File

@ -0,0 +1,2 @@
\0
PG30-1GB.DEF

22
data/tests/SAMPLE.IXI Normal file
View File

@ -0,0 +1,22 @@
[PRO-GEN]
version=3.21
type=ixi
indexformat=2
[Table_1]
current_record=0
current_index=1
indexes=1
sel_1=
sort_1=
mark_1=
marked_1=0
[Table_2]
current_record=0
current_index=1
indexes=1
sel_1=
sort_1=
mark_1=
marked_1=0

BIN
data/tests/SAMPLE.IXP Normal file
View File

Binary file not shown.

BIN
data/tests/SAMPLE.IXR Normal file
View File

Binary file not shown.

BIN
data/tests/SAMPLE.MEM Normal file
View File

Binary file not shown.

BIN
data/tests/SAMPLE.PER Normal file
View File

Binary file not shown.

BIN
data/tests/SAMPLE.REL Normal file
View File

Binary file not shown.

20
data/tests/SAMPLE.REM Normal file
View File

@ -0,0 +1,20 @@
File of remarks for GEDCOM import.
GEDCOM file : D:\Users\PRC\Documents\Gramps\data\tests\exp_sample_ged.ged
PRO-GEN file : C:\PG30\GB\DATA\SAMPLE
Date : 13/01/2017
Rec. Remarks (P=person, R=Marriage)
--------------------------------------------------------------------------------
P1 (@I0000@) Surname (Anna Nana /Hansdotter/) has already been entered.
2 FAMC @F0008@
P25 (@I0024@) Death_date (BEF 23 JUL 1930) too long.
2 FAMC @F0010@
P43 (@I0042@) Surname (Frank /Neilsen/) has already been entered.
2 FAMC @F0005@
P47 (@I0046@) Address_date (FROM 1 JAN 1964 TO 3 MAR 1970) too long.
(@I0046@) Address_date (I think 1970 to 1971) has already been entered.
2 FAMC @F0016@
2 FAMC @F0016@
2 FAMC @F0016@
--------------------------------------------------------------------------------

BIN
data/tests/SAMPLE.X11 Normal file
View File

Binary file not shown.

BIN
data/tests/SAMPLE.X21 Normal file
View File

Binary file not shown.

1222
data/tests/SAMPLE.gramps Normal file
View File

File diff suppressed because it is too large Load Diff

150
gramps/plugins/importer/importprogen.py
View File

@ -4,7 +4,7 @@
#
# Copyright (C) 2008-2011 Kees Bakker
# Copyright (C) 2008 Brian G. Matherly
# Copyright (C) 2013-2016 Alois Poettker <alois.poettker@gmx.de>
# Copyright (C) 2013-2017 Alois Poettker <alois.poettker@gmx.de>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@ -57,7 +57,7 @@ from gramps.gen.lib import (Address, Attribute, AttributeType, ChildRef, Citatio
Name, NameType, NameOriginType, Note, NoteType, Person,
Place, PlaceName, Source, SrcAttribute, Surname, Tag)
from gramps.gen.utils.id import create_id
from gramps.gui.utils import ProgressMeter
from gramps.gen.updatecallback import UpdateCallback
from gramps.gen.utils.libformatting import ImportInfo
class ProgenError(Exception):
@ -101,6 +101,7 @@ def _find_from_handle(progen_id, table):
if not intid:
intid = create_id()
table[progen_id] = intid
return intid
def _read_mem(bname):
@ -118,23 +119,24 @@ def _read_mem(bname):
else:
fname = bname + '.mem'
file_ = open(fname, "rb")
LOG.debug("The current system is %s-endian", sys.byteorder)
with open(fname, "rb") as file_:
LOG.debug("The current system is %s-endian", sys.byteorder)
# The input file comes from [what was originally] a DOS machine so will be
# little-endian, regardless of the 'native' byte order of the host system
recfmt = "<i28s"
reclen = struct.calcsize(str(recfmt))
# print("# reclen = %d" % reclen)
# The input file comes from [what was originally] a DOS machine so will
# be little-endian, regardless of the 'native' byte order of the host
recfmt = "<i28s"
reclen = struct.calcsize(str(recfmt))
# print("# reclen = %d" % reclen)
mems = []
while 1:
buf = file_.read(reclen)
if not buf:
break
(recno, text) = struct.unpack(recfmt, buf)
mems.append([recno, text])
return mems
mems = []
while 1:
buf = file_.read(reclen)
if not buf:
break
(recno, text) = struct.unpack(recfmt, buf)
mems.append([recno, text])
return mems
def _read_recs(table, bname):
"""
@ -145,23 +147,23 @@ def _read_recs(table, bname):
else:
fname = bname + table.fileext.lower()
file_ = open(fname, "rb")
recfmt = table.recfmt
LOG.info("# %s - recfmt = %s" % (table['name1'], recfmt))
reclen = struct.calcsize(str(recfmt))
LOG.info("# %s - reclen = %d" % (table['name1'], reclen))
with open(fname, "rb") as file_:
recfmt = table.recfmt
LOG.info("# %s - recfmt = %s" % (table['name1'], recfmt))
reclen = struct.calcsize(str(recfmt))
LOG.info("# %s - reclen = %d" % (table['name1'], reclen))
recs = []
while 1:
buf = file_.read(reclen)
if not buf:
break
tups = struct.unpack(recfmt, buf)
recs.append(tups)
recs = []
while 1:
buf = file_.read(reclen)
if not buf:
break
tups = struct.unpack(recfmt, buf)
recs.append(tups)
LOG.info("# length %s.recs[] = %d" % (table['name1'], len(recs)))
LOG.info("# length %s.recs[] = %d" % (table['name1'], len(recs)))
return recs
return recs
def _get_defname(fname):
"""
@ -175,7 +177,8 @@ def _get_defname(fname):
# We will strip the C: and convert the rest to a native pathname. Next, this
# pathname is compared with <fname>.
lines = open(fname).readlines()
with open(fname) as file_:
lines = file_.readlines()
if not lines[0].startswith(r'\0') or len(lines) < 2:
raise ProgenError(_("Not a Pro-Gen file"))
@ -214,10 +217,11 @@ def _get_mem_text(mems, i):
return
i -= 1
recno = mems[i][0]
recno = mems[i][0] - 1
text = mems[i][1].decode('cp850')
if recno > 0:
text += _get_mem_text(mems, recno)
while recno >= 0:
text += mems[recno][1].decode('cp850')
recno = mems[recno][0] - 1
text = text.replace('\033\r', '\n') # ESC-^M is newline
text = ESC_CTRLZ.sub('', text) # ESC-^Z is end of string
@ -364,6 +368,7 @@ class PG30DefTable(object):
fmt += 'i'
else:
pass # ???? Do we want to know?
return fmt
def get_fields(self):
@ -484,7 +489,7 @@ class PG30Def(object):
TAGOBJECTS = ['Person', 'Family', 'Event', 'Place', 'Citation', 'Source', 'Note']
class ProgenParser:
class ProgenParser(UpdateCallback):
"""
Main class to import and parse Pro-Gen files.
"""
@ -511,14 +516,13 @@ class ProgenParser:
self.user.notify_error(_("Pro-Gen data error"), str(error_msg))
return
self.progress = ProgressMeter( # TODO no-parent
_("Import from Pro-Gen"), '')
self.progress.set_pass(_('Initializing'))
self.mems = _read_mem(self.bname)
self.pers = _read_recs(self.def_['Table_1'], self.bname)
self.rels = _read_recs(self.def_['Table_2'], self.bname)
self.set_total(2 * len(self.pers) + len(self.rels))
# self.reset(_("Import from Pro-Gen")) # non-functional for now
with DbTxn(_("Pro-Gen import"), self.dbase, batch=True) as self.trans:
self.dbase.disable_signals()
@ -530,12 +534,11 @@ class ProgenParser:
self.dbase.enable_signals()
self.dbase.request_rebuild()
self.progress.close()
def __init__(self, data_base, file_name, user_handle):
def __init__(self, data_base, file_name, user):
"""
Pro-Gen defines his own set of (static) person and family identifiers.
"""
UpdateCallback.__init__(self, user.callback)
# Sometime their match the GRAMPS localisation, sometimes not. To be on
# a safe and uniform path person and family identifiers for (alphabetical)
# German (de), English (en) and Dutch (nl) language defined here.
@ -544,7 +547,7 @@ class ProgenParser:
file_name = self.bname + '.def'
self.dbase = data_base
self.fname = file_name
self.user = user_handle
self.user = user
self.language = 0
self.mems = None # Memory area
@ -567,7 +570,6 @@ class ProgenParser:
self.opt_death_info2cause = True # Death info to Death cause
# Miscalaneous
self.progress = None # Prgress bar
self.trans = None # Transaction identifier
self.def_ = None # PG30 definitions
self.high_fam_id = -1
@ -816,7 +818,6 @@ class ProgenParser:
"""
Finds or creates a Citation based on Source, Name, Date, Page, Note, Attribute.
"""
if not source_title:
return None
@ -861,7 +862,7 @@ class ProgenParser:
citation.set_page('%s' % page)
# process Note
note = self.__create_note(note_text, NoteType.CUSTOM, "Pro-Gen Export")
note = self.__create_note(note_text, NoteType.CUSTOM, "Pro-Gen Import")
if note and note.handle:
citation.add_note(note.handle)
@ -883,6 +884,9 @@ class ProgenParser:
if not note_text:
return None
if isinstance(note_text, list):
note_text = '\n'.join(note_text)
note = Note()
note.set(note_text)
note_type = NoteType()
@ -1114,8 +1118,7 @@ class ProgenParser:
pix = table.get_record_field_index(pid)
person_ix.append(pix)
# The records are numbered 1..N
self.progress.set_pass(_('Importing individuals'), len(self.pers))
# self.set_text(_('Importing individuals')) # non-functional for now
# Male / Female symbols
male_sym = self.def_.tables['Genealogical'].parms['male']
@ -1123,8 +1126,8 @@ class ProgenParser:
ind_id = self.opt_ind_id -1 # Option: Individuals IDs interator
for i, rec in enumerate(self.pers):
# Progress at the begin due to approx. ton's of 'not recflds[1]'
self.progress.step()
# Update at the begin due to approx. ton's of 'not recflds[1]'
self.update()
recflds = table.convert_record_to_list(rec, self.mems)
if not recflds[1]:
@ -1268,10 +1271,10 @@ class ProgenParser:
if event_ref:
person.add_event_ref(event_ref)
# process F16 Person Note, F17 Person Info
note = recflds[person_ix[16]] # F16: INDI _COMM / INDI COMM
info = recflds[person_ix[17]] # F17: INDI NOTE
note_text = [_f for _f in [info, note] if _f]
# process F16 Person Comment, F17 Person Note
comm = recflds[person_ix[16]] # F16: INDI _COMM / INDI COMM
note = recflds[person_ix[17]] # F17: INDI NOTE
note_text = [_f for _f in [comm, note] if _f]
note = self.__create_note(note_text, NoteType.PERSON)
if note and note.handle:
person.add_note(note.handle)
@ -1294,11 +1297,16 @@ class ProgenParser:
address = Address()
if date:
address.set_date_object(date)
address.set_street(street)
address.set_city(recflds[person_ix[21]])
address.set_postal_code(postal_code)
address.set_country(country)
address.set_phone(phone)
if street:
address.set_street(street)
if recflds[person_ix[21]]:
address.set_city(recflds[person_ix[21]])
if postal_code:
address.set_postal_code(postal_code)
if country:
address.set_country(country)
if phone:
address.set_phone(phone)
# Option 1: add Notes to Address
note = self.__create_note(info, NoteType.ADDRESS)
@ -1498,12 +1506,11 @@ class ProgenParser:
family_ix.append(fix)
# The records are numbered 1..N
self.progress.set_pass(_('Importing families'), len(self.rels))
# self.set_text(_('Importing families')) # non-functional for now
fam_id = self.opt_fam_id -1 # Option: Family IDs interator
for i, rec in enumerate(self.rels):
# Progress at the begin
self.progress.step()
# Update at the begin
self.update()
husband = rec[family_ix[3]] # F03: FAM HUSB
wife = rec[family_ix[4]] # F04: FAM WIFE
@ -1605,7 +1612,7 @@ class ProgenParser:
if date or place or info or citation:
desc = source_text
desc = [_f for _f in [info, source_text] if _f]
desc = [_f for _f in [source_text, info] if _f]
desc = desc and '; '.join(desc) or None
event, marl_ref = self.__create_event_and_ref \
(EventType.MARR_BANNS, desc, date, place, citation, '',
@ -1692,9 +1699,9 @@ class ProgenParser:
if attr:
family.add_attribute(attr)
note = recflds[family_ix[6]] # F06: FAM _COMM/FAM COMM
info = recflds[family_ix[7]] # F07: FAM NOTE
note_text = [_f for _f in [info, note] if _f]
comm = recflds[family_ix[6]] # F06: FAM _COMM/FAM COMM
note = recflds[family_ix[7]] # F07: FAM NOTE
note_text = [_f for _f in [comm, note] if _f]
if note_text:
cnt = None
if len(note_text) > 0:
@ -1765,11 +1772,10 @@ class ProgenParser:
(self.person_identifier[14][self.language].lower()) # F14: Mother
# The records are numbered 1..N
self.progress.set_pass(_('Adding children'), len(self.pers))
# self.set_text(_('Adding children')) # non-functional for now
for i, rec in enumerate(self.pers):
# Progress at the begin
self.progress.step()
# Update at the begin
self.update()
ind_id = i +1
# print(("Person ID %d " % ind_id) + " ".join(("%s" % r) for r in rec))

9
gramps/plugins/test/test_imports.py
View File

@ -29,13 +29,14 @@ from time import localtime, strptime
from unittest.mock import patch
#import logging
from gramps.gen.utils.config import config
config.set('preferences.date-format', 0)
from gramps.gen.db.utils import import_as_dict
from gramps.gen.merge.diff import diff_dbs, to_struct
from gramps.gen.simple import SimpleAccess
from gramps.gen.utils.id import set_det_id
from gramps.cli.user import User
from gramps.gen.const import TEMP_DIR, DATA_DIR
from gramps.gen.utils.config import config
from gramps.test.test_util import capture
from gramps.plugins.export.exportxml import XmlWriter
@ -212,6 +213,8 @@ def make_tst_function(tstfile, file_name):
skp_imp_adds = False
else:
skp_imp_adds = True
config.set('preferences.default-source', False)
config.set('preferences.tag-on-import', False)
try:
os.remove(fres)
os.remove(fout)
@ -288,8 +291,8 @@ else:
_tstfiles = []
for _tstfile in os.listdir(TEST_DIR):
(fname, ext) = os.path.splitext(os.path.basename(_tstfile))
if ext in (".gramps", ".difs", ".bak") \
or not fname.startswith("imp_"):
if _tstfile != "SAMPLE.DEF" and (ext in (".gramps", ".difs", ".bak") \
or not fname.startswith("imp_")):
continue
test_func = make_tst_function(_tstfile, fname)
clname = 'Import_{0}'.format(_tstfile)