* src/plugins/PHPGedViewConnector.py: Enhanced version.

svn: r4667

gramps2/ChangeLog

@@ -1,3 +1,6 @@
+2005-05-24  Martin Hawlisch  <Martin.Hawlisch@gmx.de>
+	* src/plugins/PHPGedViewConnector.py: Enhanced version.
+	
 2005-05-23  Alex Roitman  <shura@gramps-project.org>
 	* various: merge changes made in gramps20 branch with main trunk.
 

gramps2/src/plugins/PHPGedViewConnector.py

@@ -23,12 +23,14 @@
 "Download a GEDCOM file from a phpGedView server"
 
 
+import httplib
 import urllib2
 import gtk
 import gtk.glade
 import os
 from random import randint
 from tempfile import NamedTemporaryFile
+from tempfile import mkstemp
 from gettext import gettext as _
 
 #
@@ -72,15 +74,19 @@ class PHPGedViewConnector:
         POS_ALL:    "all",
         }
     
-    def __init__(self,url):
+    def __init__(self,url,progressbar_cb=None):
         self.url = url
         self.sessionname = None
         self.sessionid = None
         self.connected = False
+        self.progressbar_cb = progressbar_cb
     
+    def update_progressbar(self,text=None,pos=0,max=0):
+        if self.progressbar_cb:
+            self.progressbar_cb(text,pos,max)
+            
     def get_version(self):
         response = self.fetch_url_to_array( [("action","version",),])
-        print response
         if response and response[0] == "SUCCESS":
             version = response[1]
             return version
@@ -88,7 +94,6 @@ class PHPGedViewConnector:
 
     def list_gedcoms(self):
         response = self.fetch_url_to_array( [("action","listgedcoms",),])
-        print response
         if response and response[0] == "SUCCESS":
             gedcoms = []
             for i in range(1,len(response)):
@@ -100,9 +105,10 @@ class PHPGedViewConnector:
     
     
     def connect_to_gedcom(self, filename=None, username=None, password=None):
+        self.gedname = None
         params = []
         params.append( ("action","connect",))
-        if file:
+        if filename:
             params.append( ("ged",filename,))
         if username:
             params.append( ("username",username,))
@@ -110,17 +116,17 @@ class PHPGedViewConnector:
             params.append( ("password",password,))
         
         response = self.fetch_url_to_array( params)
-        print response
         if response and response[0] == "SUCCESS":
             session = response[1].split("\t")
             self.sessionname = session[0]
             self.sessionid = session[1]
             self.connected = True
+            if filename:
+                self.gedname = filename
             return True
         return False
 
     def list_xrefs(self, type = TYPE_ALL, pos=POS_ALL, xref=None):
-        print type
         result = []
         types = []
         if type == self.TYPE_ALL:
@@ -136,7 +142,6 @@ class PHPGedViewConnector:
             if xref:
                 request.append( ("xref", xref))
             result_part = self.fetch_url_to_array( request)
-            print result_part
             if result_part[0] == "SUCCESS":
                 for x in range(1,len(result_part)):
                     txt = result_part[x]
@@ -150,15 +155,24 @@ class PHPGedViewConnector:
                         break
         return result
     
-    def get_record(self, xref):
+    def get_records(self, xref):
-        if not xref:
+        if not xref or len(xref) == 0:
             return None
+        
+        # merge xref list to a semicolon separated string
+        xref_str = ""
+        try:
+            for x in xref:
+                xref_str += x+";"
+        except TypeError:
+            xref_str = xref
+        
         result = []
         request = []
         request.append( ("action", "get"))
-        request.append( ("xref", xref))
+        request.append( ("xref", xref_str))
         result_part = self.fetch_url_to_array( request)
-        print result_part
+        #print result_part
         if result_part[0] == "SUCCESS":
             for x in range(1,len(result_part)):
                 txt = result_part[x].strip()
@@ -166,15 +180,70 @@ class PHPGedViewConnector:
                     result.append( txt)
         return result
 
+    def fetch_full_gedcom( self, outfile=None):
+        print outfile
+        if outfile is None:
+            gedname = self.gedname
+            if not gedname:
+                gedname = "temp.ged"
+            filenameparts = gedname.split(".")
+            (outfiled,outfilename) = mkstemp("."+filenameparts[1],filenameparts[0]+"_")
+            outfile = os.fdopen(outfiled,"w")
+        else:
+            outfilename = outfile.name
+        print outfile
+        
+        outfile.write("0 HEAD\n")
+        outfile.write("1 SOUR phpGedView\n")
+        outfile.write("2 VERS %s\n" % self.get_version())
+        outfile.write("2 NAME phpGedView\n")
+        outfile.write("2 DATA %s\n" % self.url)
+        outfile.write("1 CHAR UTF-8\n")
+        outfile.write("1 GEDC\n")
+        outfile.write("2 VERS 5.5\n")
+        outfile.write("2 FORM LINEAGE-LINKED\n")
+        outfile.write("1 NOTE Dowloaded using GRAMPS PHPGedViewConnector\n")
+        
+        self.update_progressbar(_("Fetching index list..."))
+        steps = (   self.TYPE_INDI,
+                    self.TYPE_FAM,
+                    self.TYPE_SOUR,
+                    self.TYPE_REPO,
+                    self.TYPE_NOTE,
+                    self.TYPE_OBJE
+                )
+        xref_list = []
+        i = 0
+        for type in steps:
+            self.update_progressbar( _("Fetching index list..."), i, len(steps))
+            xref_list += self.list_xrefs( type)
+            i += 1
+        self.update_progressbar( _("Fetching records..."))
+        i = 0
+        junk_size = 100
+        for i in range(len(xref_list)/junk_size+1):
+            self.update_progressbar( _("Fetching records..."), i*junk_size, len(xref_list))
+            record = self.get_records(xref_list[i*junk_size:(i+1)*junk_size])
+            if record:
+                for r in record:
+                    outfile.write(r+"\n")
+            outfile.flush()
+            i += 1
+
+        outfile.write("0 TRLR\n")
+        outfile.flush()
+        outfile.close()
+        return outfilename
+
+
     def get_variable(self, name="PEDIGREE_ROOT_ID"):
         if not name:
             return None
         result = []
         request = []
         request.append( ("action", "getvar"))
-        request.append( ("var", var))
+        request.append( ("var", name))
         result_part = self.fetch_url_to_array( request)
-        print result_part
         if result_part[0] == "SUCCESS":
             for x in range(1,len(result_part)):
                 txt = result_part[x].strip()
@@ -183,7 +252,7 @@ class PHPGedViewConnector:
         return result
 
     def fetch_url_to_array(self, params):
-        request_url = self.url + "client.php?"
+        request_url = self.url + "gdbi.php?"
         for param in params:
             request_url += "%s=%s&" % (param)
         if self.sessionname and self.sessionid:
@@ -232,6 +301,14 @@ class phpGedViewImporter:
         self.dialog = top.get_widget('importer')
         self.dialog.show()
 
+    def filter_url(self, url):
+        if url[:7] != "http://":
+            url = "http://"+url
+        if url[-1:] != "/":
+            url = url + "/"
+        print url
+        return url
+
     def update_progressbar(self,text,step=0,max=0):
         self.progressbar.set_text(text)
         if max > 0:
@@ -242,57 +319,26 @@ class phpGedViewImporter:
             gtk.main_iteration()
         
     def on_next_pressed_cb(self, widget, event=None, data=None):
+        import ReadGedcom
         if event:
             print event.type
         
         if not self.url or self.url != self.url_entry.get_text():
             # url entered the first time or url changed
-            self.url = self.url_entry.get_text()
+            self.url = self.filter_url( self.url_entry.get_text())
-            self.validate_server()
+            if self.validate_server():
+                self.url_entry.set_text( self.url)
         else:
             self.update_progressbar(_("Logging in..."))
             if self.connector.connect_to_gedcom(self.file_combo.get_active_text(), self.username_entry.get_text(), self.password_entry.get_text()):
-                if self.file_combo.get_active_text():
+
-                    gedname = self.file_combo.get_active_text()
+                self.update_progressbar( _("Fetching GEDCOM..."))
-                else:
+                
-                    gedname = "temp.ged"
+                fn = self.connector.fetch_full_gedcom()
-                print "gedname"
-                filenameparts = gedname.split(".")
-                outfile = NamedTemporaryFile("w",-1,"."+filenameparts[1],filenameparts[0]+"_")
-                print "WRITING TO: "+outfile.name
-                outfile.write("0 HEAD\n")
-                outfile.write("1 CHAR UTF-8\n")
                 
-                self.update_progressbar(_("Fetching index list..."))
-                steps = (   PHPGedViewConnector.TYPE_INDI,
-                            PHPGedViewConnector.TYPE_FAM,
-                            PHPGedViewConnector.TYPE_SOUR,
-                            PHPGedViewConnector.TYPE_REPO,
-                            PHPGedViewConnector.TYPE_NOTE,
-                            PHPGedViewConnector.TYPE_OBJE
-                        )
-                xref_list = []
-                i = 0
-                for type in steps:
-                    self.update_progressbar( _("Fetching index list..."), i, len(steps))
-                    xref_list += self.connector.list_xrefs( type)
-                    i += 1
-                self.update_progressbar( _("Fetching records..."))
-                i = 0
-                for xref in xref_list[:10]:
-                    self.update_progressbar( _("Fetching records..."), i, len(xref_list))
-                    record = self.connector.get_record(xref)
-                    for r in record:
-                        outfile.write(r+"\n")
-                    outfile.flush()
-                    i += 1
-
-                outfile.flush()
-
                 self.update_progressbar( _("Importing GEDCOM..."))
-                
+
-                import ReadGedcom
+                ReadGedcom.importData(self.db, fn)
-                ReadGedcom.importData(self.db, outfile.name)
                 # done. bye.
                 self.dialog.destroy()
                 
@@ -304,53 +350,91 @@ class phpGedViewImporter:
     def validate_server(self):
         try:
             self.update_progressbar(_("Connecting..."))
-            self.connector = PHPGedViewConnector(self.url)
+            self.connector = PHPGedViewConnector(self.url,self.update_progressbar)
             self.update_progressbar(_("Get version..."))
             version = self.connector.get_version()
-            self.version_label.set_text(_("Version %s") % version)
+            if version:
-            self.update_progressbar(_("Reading file list..."))
+                self.version_label.set_text(_("Version %s") % version)
-            files = self.connector.list_gedcoms()
+                self.update_progressbar(_("Reading file list..."))
-            list_store = self.file_combo.get_model()
+                files = self.connector.list_gedcoms()
-            list_store.clear()
+                list_store = self.file_combo.get_model()
-            for file in files:
+                list_store.clear()
-                list_store.append([file[0],])
+                for file in files:
-            self.file_combo.show()
+                    list_store.append([file[0],])
-            self.username_entry.show()
+                self.file_combo.show()
-            self.password_entry.show()
+                self.username_entry.show()
+                self.password_entry.show()
+                return True
             
-        except (ValueError, urllib2.URLError), e:
+        except (ValueError, urllib2.URLError, httplib.InvalidURL), e:
             print e
             self.version_label.set_text(_("Error: Invalid URL"))
-            self.version_label.set_text(_("Error: Unable to connect to phpGedView"))
         self.update_progressbar(_("done."))
+        return False
 
 
 
-
+# TODO: This should go into PHPGedViewConnector
-
+def filter_url( url):
-#-------------------------------------------------------------------------
+    url = url.split("?")[0]     # strip params
-#
+    if url[-1:] == "/":         # strip trailing slash
-#
+        url = url[:-1]
-#
+    if url[-4:] in (".php",".htm") or url[-5:] in (".html"):      # strip script name
-#-------------------------------------------------------------------------
+        idx = url.rfind("/")
-from PluginMgr import register_tool
+        if idx > 1:
-
+            url = url[:idx]
-def phpGedViewImporterCaller(database,active_person,callback,parent=None):
+    if url[:7] != "http://":    # add protocol
-    phpGedViewImporter(database)
+        url = "http://"+url
-
+    if url[-1:] != "/":         # readd trailing slash
-register_tool(
+        url = url + "/"
-    phpGedViewImporterCaller,
+    return url
-    _("Import the gedcom from a phpGedView driven website"),
-    category=_("Import"),
-    description=_("phpGedView is an open source web application that generates dynamic webpages"
-                    " out of a GEDCOM file. This plugin uses the gedcom access protocol to"
-                    " retrieve the gedcom file from the webserver.")
-    )
-
 
 
 # for Testing
 if __name__ == "__main__":
+    def dummy_progress( text,pos=0,max=0):
+        if max > 0:
+            percent = pos*100/max
+            print "%s: %d%%" % (text,percent)
+        else:
+            print text
     
-    phpGedViewImporter(None)
+    try:
-    gtk.main()
+        f = open("/tmp/sites.txt")
+        l = f.readline()
+        while l:
+            l = filter_url(l.strip())
+            print l
+            try:
+                c = PHPGedViewConnector(l,dummy_progress)
+                c.connect_to_gedcom()
+                v = c.get_version()
+                if v:
+                    print("%s\t\t%s" % (v,l))
+                    c.fetch_full_gedcom()
+            except KeyboardInterrupt:
+                exit
+            l = f.readline()
+    except IOError:
+        phpGedViewImporter(None)
+        gtk.main()
+
+else:
+    #-------------------------------------------------------------------------
+    #
+    #
+    #
+    #-------------------------------------------------------------------------
+    from PluginMgr import register_tool
+    
+    def phpGedViewImporterCaller(database,active_person,callback,parent=None):
+        phpGedViewImporter(database)
+    
+    register_tool(
+        phpGedViewImporterCaller,
+        _("Import the gedcom from a phpGedView driven website"),
+        category=_("Import"),
+        description=_("phpGedView is an open source web application that generates dynamic webpages"
+                        " out of a GEDCOM file. This plugin uses the gedcom access protocol to"
+                        " retrieve the gedcom file from the webserver.")
+        )