Merge pull request #326 from prculley/progen_prog

Progen import fixes and testing

.gitattributes

@@ -13,3 +13,6 @@
 *.ged -text
 *.GED -text
 *.csv -text
+*.DEF -text
+*.IXI -text
+*.REM -text

data/tests/PG30-1GB.DEF

@@ -0,0 +1,236 @@
+[PRO-GEN]
+version=3.0b
+type=def
+format=2
+
+[general]
+dateformat=DD/MM/YYYY
+pointerlength=4
+tables=2
+
+[Table_1]
+name1=Person
+name2=Persons
+fileext=.PER
+indexext=.IXP
+n_lines=42
+hiddenlines=4,5,6,7,8,10,12,13,14,15,19,25,30,32,33,34,35,36,40
+t01=<3D>͵INDIVIDUAL       <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>͵pg30-1gb<67><62><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>͵LAST CHANGE           <20>ͻ
+t02=<3D> <20>Given name:                                                                  <20>
+t03=<3D> Surname   :                                                Sex:              <20>
+t04=<3D> Patronym  :                                                                  <20>
+t05=<3D> Call name :                    Alias:                     Code:              <20>
+t06=<3D> Title 1   :                  Title 2:                  Title 3:              <20>
+t07=<3D> Father    :                                                                  <20>
+t08=<3D> Mother    :                                                                  <20>
+t09=<3D> Occupation:                                                                  <20>
+t10=<3D> Scratch   :                                                                  <20>
+t11=<3D> Info      :                                                                  <20>
+t12=<3D>Ĵ<EFBFBD>Address<73><73><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>Ķ
+t13=<3D> Date  :            Street:                                                   <20>
+t14=<3D> Zip   :             Place:                             Country:              <20>
+t15=<3D> Phone :              Info:                                                   <20>
+t16=<3D>Ĵ<EFBFBD>Birth<74><68><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>Ķ
+t17=<3D> Date  :             Place:                                Time:              <20>
+t18=<3D> Source:                                              Reference:              <20>
+t19=<3D> Text  :                                                                      <20>
+t20=<3D> Info  :                                                                      <20>
+t21=<3D>Ĵ<EFBFBD>Christening<6E><67><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>Ķ
+t22=<3D> Date  :             Place:                            Religion:              <20>
+t23=<3D> Witn. :                                                                      <20>
+t24=<3D> Source:                                              Reference:              <20>
+t25=<3D> Text  :                                                                      <20>
+t26=<3D> Info  :                                                                      <20>
+t27=<3D>Ĵ<EFBFBD>Death<74><68><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>Ķ
+t28=<3D> Date  :             Place:                                Time:              <20>
+t29=<3D> Source:                                              Reference:              <20>
+t30=<3D> Text  :                                                                      <20>
+t31=<3D> Info  :                                                                      <20>
+t32=<3D>ĴC<C4B4>remation<6F><6E><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>Ķ
+t33=<3D> Date  :             Place:                                                   <20>
+t34=<3D> Source:                                              Reference:              <20>
+t35=<3D> Text  :                                                                      <20>
+t36=<3D> Info  :                                                                      <20>
+t37=<3D>ĴB<C4B4>urial<61><6C><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>Ķ
+t38=<3D> Date  :             Place:                                                   <20>
+t39=<3D> Source:                                              Reference:              <20>
+t40=<3D> Text  :                                                                      <20>
+t41=<3D> Info  :                                                                      <20>
+t42=<3D><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ͼ
+n_fields=58
+f01=Person_record       ,31, 6, 0, 1,15,"","INDI RFN"
+f02=Person_last_change  ,32,10,10, 1,68,"","INDI CHAN DATE"
+f03=Given_name          ,47,64, 4, 2,15,"",""
+f04=Surname             ,47,40, 4, 3,15,"","INDI NAME"
+f05=Sex                 ,45, 1, 1, 3,67,"MMFFmmff??","INDI SEX"
+f06=Patronym            ,47,40, 4, 4,15,"","INDI _PATR"
+f07=Call_name           ,47,18, 4, 5,15,"","INDI NAME NICK/INDI NAME ALIA/INDI CHR NICK"
+f08=Alias               ,47,19, 4, 5,41,"","INDI NAME _ALIA/INDI NAME COMM"
+f09=Person_code         ,47,12, 4, 5,67,"","INDI REFN/INDI CODE"
+f10=Title1              ,47,16, 4, 6,15,"","INDI TITL"
+f11=Title2              ,47,16, 4, 6,41,"","INDI _TITL2"
+f12=Title3              ,47,12, 4, 6,67,"","INDI _TITL3"
+f13=Father              , 2,64, 4, 7,15,"",""
+f14=Mother              , 3,64, 4, 8,15,"",""
+f15=Occupation          ,47,64, 4, 9,15,"","INDI OCCU"
+f16=Person_scratch      ,46,64, 4,10,15,"","INDI _COMM/INDI COMM"
+f17=Person_info         ,46,64, 4,11,15,"","INDI NOTE"
+f18=Address_date        ,44,10,10,13,11,"","INDI RESI DATE"
+f19=Address_street      ,47,49, 4,13,30,"","INDI RESI ADDR"
+f20=Address_zip         ,47,11, 4,14,11,"","INDI RESI ADDR POST/INDI RESI POST"
+f21=Address_place       ,47,27, 4,14,30,"","INDI RESI ADDR CITY/INDI RESI PLAC"
+f22=Address_country     ,47,12, 4,14,67,"","INDI RESI ADDR CTRY/INDI RESI CTRY"
+f23=Address_phone       ,47,12, 4,15,11,"","INDI RESI PHON/INDI PHON"
+f24=Address_info        ,46,49, 4,15,30,"","INDI RESI NOTE/INDI ADDR"
+f25=Birth_date          ,44,10,10,17,11,"","INDI BIRT DATE"
+f26=Birth_place         ,47,30, 4,17,30,"","INDI BIRT PLAC"
+f27=Birth_time          ,45, 5, 5,17,67,"09::..","INDI BIRT TIME"
+f28=Birth_source        ,47,44, 4,18,11,"","INDI BIRT SOUR/INDI BIRT SOUR TITL"
+f29=Birth_ref           ,45,12,12,18,67,"","INDI BIRT SOUR REFN"
+f30=Birth_text          ,46,68, 4,19,11,"","INDI BIRT SOUR TEXT"
+f31=Birth_info          ,46,68, 4,20,11,"","INDI BIRT NOTE"
+f32=Christening_date    ,44,10,10,22,11,"","INDI CHR DATE"
+f33=Christening_place   ,47,26, 4,22,30,"","INDI CHR PLAC"
+f34=Religion            ,47,12, 4,22,67,"","INDI CHR RELI/INDI RELI"
+f35=Christening_witness ,47,68, 4,23,11,"","INDI CHR _WITN/INDI CHR WITN"
+f36=Christening_source  ,47,44, 4,24,11,"","INDI CHR SOUR/INDI CHR SOUR TITL"
+f37=Christening_ref     ,45,12,12,24,67,"","INDI CHR SOUR REFN"
+f38=Christening_text    ,46,68, 4,25,11,"","INDI CHR SOUR TEXT"
+f39=Christening_info    ,46,68, 4,26,11,"","INDI CHR NOTE"
+f40=Death_date          ,44,10,10,28,11,"","INDI DEAT DATE"
+f41=Death_place         ,47,30, 4,28,30,"","INDI DEAT PLAC"
+f42=Death_time          ,45, 5, 5,28,67,"09::..","INDI DEAT TIME"
+f43=Death_source        ,47,44, 4,29,11,"","INDI DEAT SOUR/INDI DEAT SOUR TITL"
+f44=Death_ref           ,45,12,12,29,67,"","INDI DEAT SOUR REFN"
+f45=Death_text          ,46,68, 4,30,11,"","INDI DEAT SOUR TEXT"
+f46=Death_info          ,46,68, 4,31,11,"","INDI DEAT NOTE"
+f47=Cremation_date      ,44,10,10,33,11,"","INDI CREM DATE"
+f48=Cremation_place     ,47,49, 4,33,30,"","INDI CREM PLAC"
+f49=Cremation_source    ,47,44, 4,34,11,"","INDI CREM SOUR/INDI CREM SOUR TITL"
+f50=Cremation_ref       ,45,12,12,34,67,"","INDI CREM SOUR REFN"
+f51=Cremation_text      ,46,68, 4,35,11,"","INDI CREM SOUR TEXT"
+f52=Cremation_info      ,46,68, 4,36,11,"","INDI CREM NOTE"
+f53=Burial_date         ,44,10,10,38,11,"","INDI BURI DATE"
+f54=Burial_place        ,47,49, 4,38,30,"","INDI BURI PLAC"
+f55=Burial_source       ,47,44, 4,39,11,"","INDI BURI SOUR/INDI BURI SOUR TITL"
+f56=Burial_ref          ,45,12,12,39,67,"","INDI BURI SOUR REFN"
+f57=Burial_text         ,46,68, 4,40,11,"","INDI BURI SOUR TEXT"
+f58=Burial_info         ,46,68, 4,41,11,"","INDI BURI NOTE"
+n_replace=4
+r01=Surname,Patronym
+r02=Birth_date,Christening_date/Birth_place,Christening_place
+r03=Death_date,Burial_date/Death_place,Burial_place
+r04=Burial_date,Cremation_date/Burial_place,Cremation_place
+
+[Table_2]
+name1=Marriage
+name2=Marriages
+fileext=.REL
+indexext=.IXR
+n_lines=35
+hiddenlines=5,7,8,9,10,11,16,22,28,30,31,32,33,34
+t01=<3D>͵MARRIAGE       <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>͵pg30-1gb<67><62><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>͵LAST CHANGE           <20>ͻ
+t02=<3D> <20>Husband:                                                                     <20>
+t03=<3D> Wife   :                                                                     <20>
+t04=<3D> Code   :                                                                     <20>
+t05=<3D> Scratch:                                                                     <20>
+t06=<3D> Info   :                                                                     <20>
+t07=<3D>Ĵ<EFBFBD>Living together<65><72><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>Ķ
+t08=<3D> Date  :             Place:                                                   <20>
+t09=<3D> Source:                                              Reference:              <20>
+t10=<3D> Text  :                                                                      <20>
+t11=<3D> Info  :                                                                      <20>
+t12=<3D>ĴPublication of the <20>banns<6E><73><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>Ķ
+t13=<3D> Date  :             Place:                                                   <20>
+t14=<3D> Witn. :                                                                      <20>
+t15=<3D> Source:                                              Reference:              <20>
+t16=<3D> Text  :                                                                      <20>
+t17=<3D> Info  :                                                                      <20>
+t18=<3D>ĴCivil <20>marriage<67><65><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>Ķ
+t19=<3D> Date  :             Place:                                                   <20>
+t20=<3D> Witn. :                                                                      <20>
+t21=<3D> Source:                                              Reference:              <20>
+t22=<3D> Text  :                                                                      <20>
+t23=<3D> Info  :                                                                      <20>
+t24=<3D>Ĵ<EFBFBD>Church marriage<67><65><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>Ķ
+t25=<3D> Date  :             Place:                              Church:              <20>
+t26=<3D> Witn. :                                                                      <20>
+t27=<3D> Source:                                              Reference:              <20>
+t28=<3D> Text  :                                                                      <20>
+t29=<3D> Info  :                                                                      <20>
+t30=<3D>Ĵ<EFBFBD>Divorce<63><65><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>Ķ
+t31=<3D> Date  :             Place:                                                   <20>
+t32=<3D> Source:                                              Reference:              <20>
+t33=<3D> Text  :                                                                      <20>
+t34=<3D> Info  :                                                                      <20>
+t35=<3D><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ͼ
+n_fields=41
+f01=Relation_record     ,31, 6, 0, 1,13,"","FAM RFN"
+f02=Relation_last_change,32,10,10, 1,68,"","FAM CHAN DATE"
+f03=Husband             ,22,67, 4, 2,12,"","FAM HUSB"
+f04=Wife                ,23,67, 4, 3,12,"","FAM WIFE"
+f05=Relation_code       ,47,12, 4, 4,12,"","FAM REFN/FAM CODE"
+f06=Relation_scratch    ,46,67, 4, 5,12,"","FAM _COMM/FAM COMM"
+f07=Relation_info       ,46,67, 4, 6,12,"","FAM NOTE"
+f08=Living_date         ,44,10,10, 8,11,"","FAM _LIV DATE"
+f09=Living_place        ,47,49, 4, 8,30,"","FAM _LIV PLAC"
+f10=Living_source       ,47,44, 4, 9,11,"","FAM _LIV SOUR/FAM _LIV SOUR TITL"
+f11=Living_ref          ,45,12,12, 9,67,"","FAM _LIV SOUR REFN"
+f12=Living_text         ,46,68, 4,10,11,"","FAM _LIV SOUR TEXT"
+f13=Living_info         ,46,68, 4,11,11,"","FAM _LIV NOTE"
+f14=Banns_date          ,44,10,10,13,11,"","FAM MARB DATE/FAM REGS DATE"
+f15=Banns_place         ,47,49, 4,13,30,"","FAM MARB PLAC/FAM REGS PLAC"
+f16=banns_witnesses     ,47,68, 4,14,11,"","FAM MARB _WITN/FAM MARB WITN"
+f17=Banns_source        ,47,44, 4,15,11,"","FAM MARB SOUR/FAM MARB SOUR TITL/FAM REGS SOUR"
+f18=Banns_ref           ,45,12,12,15,67,"","FAM MARB SOUR REFN/FAM REGS SOUR REFN"
+f19=Banns_text          ,46,68, 4,16,11,"","FAM MARB SOUR TEXT"
+f20=Banns_info          ,46,68, 4,17,11,"","FAM MARB NOTE"
+f21=Civil_date          ,44,10,10,19,11,"","FAM MARR(Civil) DATE/FAM MARR DATE"
+f22=Civil_place         ,47,49, 4,19,30,"","FAM MARR(Civil) PLAC/FAM MARR PLAC"
+f23=Civil_witnesses     ,47,68, 4,20,11,"","FAM MARR(Civil) _WITN/FAM MARR _WITN/FAM MARR WITN/FAM WITN"
+f24=Civil_source        ,47,44, 4,21,11,"","FAM MARR(Civil) SOUR/FAM MARR SOUR/FAM MARR SOUR TITL"
+f25=Civil_ref           ,45,12,12,21,67,"","FAM MARR(Civil) SOUR REFN/FAM MARR SOUR REFN"
+f26=Civil_text          ,46,68, 4,22,11,"","FAM MARR(Civil) SOUR TEXT/FAM MARR SOUR TEXT"
+f27=Civil_info          ,46,68, 4,23,11,"","FAM MARR(Civil) NOTE/FAM MARR NOTE"
+f28=Church_date         ,44,10,10,25,11,"","FAM MARR(Church) DATE/FAM ORDI DATE"
+f29=Church_place        ,47,28, 4,25,30,"","FAM MARR(Church) PLAC/FAM ORDI PLAC"
+f30=Church              ,47,12, 4,25,67,"","FAM MARR(Church) _CHUR/FAM ORDI _CHUR/FAM ORDI RELI"
+f31=Church_witnesses    ,47,68, 4,26,11,"","FAM MARR(Church) _WITN/FAM ORDI _WITN/FAM ORDI WITN"
+f32=Church_source       ,47,44, 4,27,11,"","FAM MARR(Church) SOUR/FAM ORDI SOUR/FAM ORDI SOUR TITL"
+f33=Church_ref          ,45,12,12,27,67,"","FAM MARR(Church) SOUR REFN/FAM ORDI SOUR REFN"
+f34=Church_text         ,46,68, 4,28,11,"","FAM MARR(Church) SOUR TEXT/FAM ORDI SOUR TEXT"
+f35=Church_info         ,46,68, 4,29,11,"","FAM MARR(Church) NOTE/FAM ORDI NOTE"
+f36=Divorce_date        ,44,10,10,31,11,"","FAM DIV DATE/FAM DIVO DATE"
+f37=Divorce_place       ,47,49, 4,31,30,"","FAM DIV PLAC/FAM DIVO PLAC"
+f38=Divorce_source      ,47,44, 4,32,11,"","FAM DIV SOUR/FAM DIV SOUR TITL"
+f39=Divorce_ref         ,45,12,12,32,67,"","FAM DIV SOUR REFN"
+f40=Divorce_text        ,46,68, 4,33,11,"","FAM DIV SOUR TEXT"
+f41=Divorce_info        ,46,68, 4,34,11,"","FAM DIV NOTE"
+n_replace=3
+r01=Banns_date,Living_date/Banns_place,Living_place
+r02=Civil_date,Church_date/Civil_place,Church_place
+r03=Church_date,Banns_date/Church_place,Banns_place
+
+[Genealogical]
+male=M
+female=F
+function_father=[FATHER]
+function_mother=[MOTHER]
+function_husband=[HUSBAND]
+function_wife=[WIFE]
+function_age=[AGE]
+function_initials=[INITIALS]
+field_father=Father
+field_mother=Mother
+field_givenname=Given_name
+field_surname=Surname
+field_sex=Sex
+field_birthdate=Birth_date
+field_christeningdate=Christening_date
+field_deathdate=Death_date
+field_burialdate=Burial_date
+field_husband=Husband
+field_wife=Wife
+field_marriagedate=Civil_date
+field_churchdate=Church_date
+field_divorcedate=Divorce_date

data/tests/SAMPLE.DEF

@@ -0,0 +1,2 @@
+\0
+PG30-1GB.DEF

data/tests/SAMPLE.IXI

@@ -0,0 +1,22 @@
+[PRO-GEN]
+version=3.21
+type=ixi
+indexformat=2
+
+[Table_1]
+current_record=0
+current_index=1
+indexes=1
+sel_1=
+sort_1=
+mark_1=
+marked_1=0
+
+[Table_2]
+current_record=0
+current_index=1
+indexes=1
+sel_1=
+sort_1=
+mark_1=
+marked_1=0

data/tests/SAMPLE.REM

@@ -0,0 +1,20 @@
+File of remarks for GEDCOM import.
+
+GEDCOM file   : D:\Users\PRC\Documents\Gramps\data\tests\exp_sample_ged.ged
+PRO-GEN file  : C:\PG30\GB\DATA\SAMPLE
+Date          : 13/01/2017
+
+Rec.   Remarks (P=person, R=Marriage)
+--------------------------------------------------------------------------------
+    P1 (@I0000@) Surname (Anna Nana /Hansdotter/) has already been entered.
+2 FAMC @F0008@
+   P25 (@I0024@) Death_date (BEF 23 JUL 1930) too long.
+2 FAMC @F0010@
+   P43 (@I0042@) Surname (Frank /Neilsen/) has already been entered.
+2 FAMC @F0005@
+   P47 (@I0046@) Address_date (FROM 1 JAN 1964 TO 3 MAR 1970) too long.
+       (@I0046@) Address_date (I think 1970 to 1971) has already been entered.
+2 FAMC @F0016@
+2 FAMC @F0016@
+2 FAMC @F0016@
+--------------------------------------------------------------------------------

gramps/plugins/importer/importprogen.py

@@ -4,7 +4,7 @@
 #
 # Copyright (C) 2008-2011  Kees Bakker
 # Copyright (C) 2008       Brian G. Matherly
-# Copyright (C) 2013-2016  Alois Poettker <alois.poettker@gmx.de>
+# Copyright (C) 2013-2017  Alois Poettker <alois.poettker@gmx.de>
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -57,7 +57,7 @@ from gramps.gen.lib import (Address, Attribute, AttributeType, ChildRef, Citatio
                             Name, NameType, NameOriginType, Note, NoteType, Person,
                             Place, PlaceName, Source, SrcAttribute, Surname, Tag)
 from gramps.gen.utils.id import create_id
-from gramps.gui.utils import ProgressMeter
+from gramps.gen.updatecallback import UpdateCallback
 from gramps.gen.utils.libformatting import ImportInfo
 
 class ProgenError(Exception):
@@ -101,6 +101,7 @@ def _find_from_handle(progen_id, table):
     if not intid:
         intid = create_id()
         table[progen_id] = intid
+
     return intid
 
 def _read_mem(bname):
@@ -118,11 +119,11 @@ def _read_mem(bname):
     else:
         fname = bname + '.mem'
 
-    file_ = open(fname, "rb")
+    with open(fname, "rb") as file_:
         LOG.debug("The current system is %s-endian", sys.byteorder)
 
-    # The input file comes from [what was originally] a DOS machine so will be
+        # The input file comes from [what was originally] a DOS machine so will
-    # little-endian, regardless of the 'native' byte order of the host system
+        # be little-endian, regardless of the 'native' byte order of the host
         recfmt = "<i28s"
         reclen = struct.calcsize(str(recfmt))
         # print("# reclen = %d" % reclen)
@@ -134,6 +135,7 @@ def _read_mem(bname):
                 break
             (recno, text) = struct.unpack(recfmt, buf)
             mems.append([recno, text])
+
         return mems
 
 def _read_recs(table, bname):
@@ -145,7 +147,7 @@ def _read_recs(table, bname):
     else:
         fname = bname + table.fileext.lower()
 
-    file_ = open(fname, "rb")
+    with open(fname, "rb") as file_:
         recfmt = table.recfmt
         LOG.info("# %s - recfmt = %s" % (table['name1'], recfmt))
         reclen = struct.calcsize(str(recfmt))
@@ -175,7 +177,8 @@ def _get_defname(fname):
     # We will strip the C: and convert the rest to a native pathname. Next, this
     # pathname is compared with <fname>.
 
-    lines = open(fname).readlines()
+    with open(fname) as file_:
+        lines = file_.readlines()
     if not lines[0].startswith(r'\0') or len(lines) < 2:
         raise ProgenError(_("Not a Pro-Gen file"))
 
@@ -214,10 +217,11 @@ def _get_mem_text(mems, i):
         return
 
     i -= 1
-    recno = mems[i][0]
+    recno = mems[i][0] - 1
     text = mems[i][1].decode('cp850')
-    if recno > 0:
+    while recno >= 0:
-        text += _get_mem_text(mems, recno)
+        text += mems[recno][1].decode('cp850')
+        recno = mems[recno][0] - 1
 
     text = text.replace('\033\r', '\n')   # ESC-^M is newline
     text = ESC_CTRLZ.sub('', text)   # ESC-^Z is end of string
@@ -364,6 +368,7 @@ class PG30DefTable(object):
                 fmt += 'i'
             else:
                 pass   # ???? Do we want to know?
+
         return fmt
 
     def get_fields(self):
@@ -484,7 +489,7 @@ class PG30Def(object):
 
 
 TAGOBJECTS = ['Person', 'Family', 'Event', 'Place', 'Citation', 'Source', 'Note']
-class ProgenParser:
+class ProgenParser(UpdateCallback):
     """
     Main class to import and parse Pro-Gen files.
     """
@@ -511,14 +516,13 @@ class ProgenParser:
             self.user.notify_error(_("Pro-Gen data error"), str(error_msg))
             return
 
-        self.progress = ProgressMeter( # TODO no-parent
-                                      _("Import from Pro-Gen"), '')
-        self.progress.set_pass(_('Initializing'))
-
         self.mems = _read_mem(self.bname)
         self.pers = _read_recs(self.def_['Table_1'], self.bname)
         self.rels = _read_recs(self.def_['Table_2'], self.bname)
 
+        self.set_total(2 * len(self.pers) + len(self.rels))
+        # self.reset(_("Import from Pro-Gen"))  # non-functional for now
+
         with DbTxn(_("Pro-Gen import"), self.dbase, batch=True) as self.trans:
             self.dbase.disable_signals()
 
@@ -530,12 +534,11 @@ class ProgenParser:
             self.dbase.enable_signals()
             self.dbase.request_rebuild()
 
-        self.progress.close()
+    def __init__(self, data_base, file_name, user):
-
-    def __init__(self, data_base, file_name, user_handle):
         """
         Pro-Gen defines his own set of (static) person and family identifiers.
         """
+        UpdateCallback.__init__(self, user.callback)
         # Sometime their match the GRAMPS localisation, sometimes not. To be on
         # a safe and uniform path person and family identifiers for (alphabetical)
         # German (de), English (en) and Dutch (nl) language defined here.
@@ -544,7 +547,7 @@ class ProgenParser:
             file_name = self.bname + '.def'
         self.dbase = data_base
         self.fname = file_name
-        self.user = user_handle
+        self.user = user
         self.language = 0
 
         self.mems = None   # Memory area
@@ -567,7 +570,6 @@ class ProgenParser:
         self.opt_death_info2cause = True   # Death info to Death cause
 
         # Miscalaneous
-        self.progress = None   # Prgress bar
         self.trans = None   # Transaction identifier
         self.def_ = None   # PG30 definitions
         self.high_fam_id = -1
@@ -816,7 +818,6 @@ class ProgenParser:
         """
         Finds or creates a Citation based on Source, Name, Date, Page, Note, Attribute.
         """
-
         if not source_title:
             return None
 
@@ -861,7 +862,7 @@ class ProgenParser:
             citation.set_page('%s' % page)
 
             # process Note
-            note = self.__create_note(note_text, NoteType.CUSTOM, "Pro-Gen Export")
+            note = self.__create_note(note_text, NoteType.CUSTOM, "Pro-Gen Import")
             if note and note.handle:
                 citation.add_note(note.handle)
 
@@ -883,6 +884,9 @@ class ProgenParser:
         if not note_text:
             return None
 
+        if isinstance(note_text, list):
+            note_text = '\n'.join(note_text)
+
         note = Note()
         note.set(note_text)
         note_type = NoteType()
@@ -1114,8 +1118,7 @@ class ProgenParser:
             pix = table.get_record_field_index(pid)
             person_ix.append(pix)
 
-        # The records are numbered 1..N
+        # self.set_text(_('Importing individuals'))  # non-functional for now
-        self.progress.set_pass(_('Importing individuals'), len(self.pers))
 
         # Male / Female symbols
         male_sym = self.def_.tables['Genealogical'].parms['male']
@@ -1123,8 +1126,8 @@ class ProgenParser:
 
         ind_id = self.opt_ind_id -1   # Option: Individuals IDs interator
         for i, rec in enumerate(self.pers):
-            # Progress at the begin due to approx. ton's of 'not recflds[1]'
+            # Update at the begin due to approx. ton's of 'not recflds[1]'
-            self.progress.step()
+            self.update()
 
             recflds = table.convert_record_to_list(rec, self.mems)
             if not recflds[1]:
@@ -1268,10 +1271,10 @@ class ProgenParser:
                 if event_ref:
                     person.add_event_ref(event_ref)
 
-            # process F16 Person Note, F17 Person Info
+            # process F16 Person Comment, F17 Person Note
-            note = recflds[person_ix[16]]   # F16: INDI _COMM / INDI COMM
+            comm = recflds[person_ix[16]]   # F16: INDI _COMM / INDI COMM
-            info = recflds[person_ix[17]]   # F17: INDI NOTE
+            note = recflds[person_ix[17]]   # F17: INDI NOTE
-            note_text = [_f for _f in [info, note] if _f]
+            note_text = [_f for _f in [comm, note] if _f]
             note = self.__create_note(note_text, NoteType.PERSON)
             if note and note.handle:
                 person.add_note(note.handle)
@@ -1294,10 +1297,15 @@ class ProgenParser:
                 address = Address()
                 if date:
                     address.set_date_object(date)
+                if street:
                     address.set_street(street)
+                if recflds[person_ix[21]]:
                     address.set_city(recflds[person_ix[21]])
+                if postal_code:
                     address.set_postal_code(postal_code)
+                if country:
                     address.set_country(country)
+                if phone:
                     address.set_phone(phone)
 
                 # Option 1: add Notes to Address
@@ -1498,12 +1506,11 @@ class ProgenParser:
             family_ix.append(fix)
 
         # The records are numbered 1..N
-        self.progress.set_pass(_('Importing families'), len(self.rels))
+        # self.set_text(_('Importing families'))  # non-functional for now
-
         fam_id = self.opt_fam_id -1   # Option: Family IDs interator
         for i, rec in enumerate(self.rels):
-            # Progress at the begin
+            # Update at the begin
-            self.progress.step()
+            self.update()
 
             husband = rec[family_ix[3]]   # F03: FAM HUSB
             wife = rec[family_ix[4]]   # F04: FAM WIFE
@@ -1605,7 +1612,7 @@ class ProgenParser:
 
                 if date or place or info or citation:
                     desc = source_text
-                    desc = [_f for _f in [info, source_text] if _f]
+                    desc = [_f for _f in [source_text, info] if _f]
                     desc = desc and '; '.join(desc) or None
                     event, marl_ref = self.__create_event_and_ref \
                         (EventType.MARR_BANNS, desc, date, place, citation, '',
@@ -1692,9 +1699,9 @@ class ProgenParser:
                         if attr:
                             family.add_attribute(attr)
 
-                note = recflds[family_ix[6]]   # F06: FAM _COMM/FAM COMM
+                comm = recflds[family_ix[6]]   # F06: FAM _COMM/FAM COMM
-                info = recflds[family_ix[7]]   # F07: FAM NOTE
+                note = recflds[family_ix[7]]   # F07: FAM NOTE
-                note_text = [_f for _f in [info, note] if _f]
+                note_text = [_f for _f in [comm, note] if _f]
                 if note_text:
                     cnt = None
                     if len(note_text) > 0:
@@ -1765,11 +1772,10 @@ class ProgenParser:
             (self.person_identifier[14][self.language].lower())   # F14: Mother
 
         # The records are numbered 1..N
-        self.progress.set_pass(_('Adding children'), len(self.pers))
+        # self.set_text(_('Adding children'))  # non-functional for now
-
         for i, rec in enumerate(self.pers):
-            # Progress at the begin
+            # Update at the begin
-            self.progress.step()
+            self.update()
 
             ind_id = i +1
             # print(("Person ID %d  " % ind_id) + " ".join(("%s" % r) for r in rec))

gramps/plugins/test/test_imports.py

@@ -29,13 +29,14 @@ from time import localtime, strptime
 from unittest.mock import patch
 #import logging
 
+from gramps.gen.utils.config import config
+config.set('preferences.date-format', 0)
 from gramps.gen.db.utils import import_as_dict
 from gramps.gen.merge.diff import diff_dbs, to_struct
 from gramps.gen.simple import SimpleAccess
 from gramps.gen.utils.id import set_det_id
 from gramps.cli.user import User
 from gramps.gen.const import TEMP_DIR, DATA_DIR
-from gramps.gen.utils.config import config
 from gramps.test.test_util import capture
 from gramps.plugins.export.exportxml import XmlWriter
 
@@ -212,6 +213,8 @@ def make_tst_function(tstfile, file_name):
             skp_imp_adds = False
         else:
             skp_imp_adds = True
+            config.set('preferences.default-source', False)
+            config.set('preferences.tag-on-import', False)
         try:
             os.remove(fres)
             os.remove(fout)
@@ -288,8 +291,8 @@ else:
     _tstfiles = []
     for _tstfile in os.listdir(TEST_DIR):
         (fname, ext) = os.path.splitext(os.path.basename(_tstfile))
-        if ext in (".gramps", ".difs", ".bak") \
+        if _tstfile != "SAMPLE.DEF" and (ext in (".gramps", ".difs", ".bak") \
-                or not fname.startswith("imp_"):
+                or not fname.startswith("imp_")):
             continue
         test_func = make_tst_function(_tstfile, fname)
         clname = 'Import_{0}'.format(_tstfile)