From 340e9a8eb793eb41258f3d2e105dd9a77d8c77d7 Mon Sep 17 00:00:00 2001 From: Martin Hawlisch Date: Tue, 24 May 2005 21:39:49 +0000 Subject: [PATCH] * src/plugins/PHPGedViewConnector.py: Enhanced version. svn: r4667 --- ChangeLog | 3 + src/plugins/PHPGedViewConnector.py | 266 +++++++++++++++++++---------- 2 files changed, 178 insertions(+), 91 deletions(-) diff --git a/ChangeLog b/ChangeLog index 31d0383bf..936833a59 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,6 @@ +2005-05-24 Martin Hawlisch + * src/plugins/PHPGedViewConnector.py: Enhanced version. + 2005-05-23 Alex Roitman * various: merge changes made in gramps20 branch with main trunk. diff --git a/src/plugins/PHPGedViewConnector.py b/src/plugins/PHPGedViewConnector.py index ce64287b1..000733611 100644 --- a/src/plugins/PHPGedViewConnector.py +++ b/src/plugins/PHPGedViewConnector.py @@ -23,12 +23,14 @@ "Download a GEDCOM file from a phpGedView server" +import httplib import urllib2 import gtk import gtk.glade import os from random import randint from tempfile import NamedTemporaryFile +from tempfile import mkstemp from gettext import gettext as _ # @@ -72,15 +74,19 @@ class PHPGedViewConnector: POS_ALL: "all", } - def __init__(self,url): + def __init__(self,url,progressbar_cb=None): self.url = url self.sessionname = None self.sessionid = None self.connected = False + self.progressbar_cb = progressbar_cb + def update_progressbar(self,text=None,pos=0,max=0): + if self.progressbar_cb: + self.progressbar_cb(text,pos,max) + def get_version(self): response = self.fetch_url_to_array( [("action","version",),]) - print response if response and response[0] == "SUCCESS": version = response[1] return version @@ -88,7 +94,6 @@ class PHPGedViewConnector: def list_gedcoms(self): response = self.fetch_url_to_array( [("action","listgedcoms",),]) - print response if response and response[0] == "SUCCESS": gedcoms = [] for i in range(1,len(response)): @@ -100,9 +105,10 @@ class PHPGedViewConnector: def connect_to_gedcom(self, filename=None, username=None, password=None): + self.gedname = None params = [] params.append( ("action","connect",)) - if file: + if filename: params.append( ("ged",filename,)) if username: params.append( ("username",username,)) @@ -110,17 +116,17 @@ class PHPGedViewConnector: params.append( ("password",password,)) response = self.fetch_url_to_array( params) - print response if response and response[0] == "SUCCESS": session = response[1].split("\t") self.sessionname = session[0] self.sessionid = session[1] self.connected = True + if filename: + self.gedname = filename return True return False def list_xrefs(self, type = TYPE_ALL, pos=POS_ALL, xref=None): - print type result = [] types = [] if type == self.TYPE_ALL: @@ -136,7 +142,6 @@ class PHPGedViewConnector: if xref: request.append( ("xref", xref)) result_part = self.fetch_url_to_array( request) - print result_part if result_part[0] == "SUCCESS": for x in range(1,len(result_part)): txt = result_part[x] @@ -150,15 +155,24 @@ class PHPGedViewConnector: break return result - def get_record(self, xref): - if not xref: + def get_records(self, xref): + if not xref or len(xref) == 0: return None + + # merge xref list to a semicolon separated string + xref_str = "" + try: + for x in xref: + xref_str += x+";" + except TypeError: + xref_str = xref + result = [] request = [] request.append( ("action", "get")) - request.append( ("xref", xref)) + request.append( ("xref", xref_str)) result_part = self.fetch_url_to_array( request) - print result_part + #print result_part if result_part[0] == "SUCCESS": for x in range(1,len(result_part)): txt = result_part[x].strip() @@ -166,15 +180,70 @@ class PHPGedViewConnector: result.append( txt) return result + def fetch_full_gedcom( self, outfile=None): + print outfile + if outfile is None: + gedname = self.gedname + if not gedname: + gedname = "temp.ged" + filenameparts = gedname.split(".") + (outfiled,outfilename) = mkstemp("."+filenameparts[1],filenameparts[0]+"_") + outfile = os.fdopen(outfiled,"w") + else: + outfilename = outfile.name + print outfile + + outfile.write("0 HEAD\n") + outfile.write("1 SOUR phpGedView\n") + outfile.write("2 VERS %s\n" % self.get_version()) + outfile.write("2 NAME phpGedView\n") + outfile.write("2 DATA %s\n" % self.url) + outfile.write("1 CHAR UTF-8\n") + outfile.write("1 GEDC\n") + outfile.write("2 VERS 5.5\n") + outfile.write("2 FORM LINEAGE-LINKED\n") + outfile.write("1 NOTE Dowloaded using GRAMPS PHPGedViewConnector\n") + + self.update_progressbar(_("Fetching index list...")) + steps = ( self.TYPE_INDI, + self.TYPE_FAM, + self.TYPE_SOUR, + self.TYPE_REPO, + self.TYPE_NOTE, + self.TYPE_OBJE + ) + xref_list = [] + i = 0 + for type in steps: + self.update_progressbar( _("Fetching index list..."), i, len(steps)) + xref_list += self.list_xrefs( type) + i += 1 + self.update_progressbar( _("Fetching records...")) + i = 0 + junk_size = 100 + for i in range(len(xref_list)/junk_size+1): + self.update_progressbar( _("Fetching records..."), i*junk_size, len(xref_list)) + record = self.get_records(xref_list[i*junk_size:(i+1)*junk_size]) + if record: + for r in record: + outfile.write(r+"\n") + outfile.flush() + i += 1 + + outfile.write("0 TRLR\n") + outfile.flush() + outfile.close() + return outfilename + + def get_variable(self, name="PEDIGREE_ROOT_ID"): if not name: return None result = [] request = [] request.append( ("action", "getvar")) - request.append( ("var", var)) + request.append( ("var", name)) result_part = self.fetch_url_to_array( request) - print result_part if result_part[0] == "SUCCESS": for x in range(1,len(result_part)): txt = result_part[x].strip() @@ -183,7 +252,7 @@ class PHPGedViewConnector: return result def fetch_url_to_array(self, params): - request_url = self.url + "client.php?" + request_url = self.url + "gdbi.php?" for param in params: request_url += "%s=%s&" % (param) if self.sessionname and self.sessionid: @@ -232,6 +301,14 @@ class phpGedViewImporter: self.dialog = top.get_widget('importer') self.dialog.show() + def filter_url(self, url): + if url[:7] != "http://": + url = "http://"+url + if url[-1:] != "/": + url = url + "/" + print url + return url + def update_progressbar(self,text,step=0,max=0): self.progressbar.set_text(text) if max > 0: @@ -242,57 +319,26 @@ class phpGedViewImporter: gtk.main_iteration() def on_next_pressed_cb(self, widget, event=None, data=None): + import ReadGedcom if event: print event.type if not self.url or self.url != self.url_entry.get_text(): # url entered the first time or url changed - self.url = self.url_entry.get_text() - self.validate_server() + self.url = self.filter_url( self.url_entry.get_text()) + if self.validate_server(): + self.url_entry.set_text( self.url) else: self.update_progressbar(_("Logging in...")) if self.connector.connect_to_gedcom(self.file_combo.get_active_text(), self.username_entry.get_text(), self.password_entry.get_text()): - if self.file_combo.get_active_text(): - gedname = self.file_combo.get_active_text() - else: - gedname = "temp.ged" - print "gedname" - filenameparts = gedname.split(".") - outfile = NamedTemporaryFile("w",-1,"."+filenameparts[1],filenameparts[0]+"_") - print "WRITING TO: "+outfile.name - outfile.write("0 HEAD\n") - outfile.write("1 CHAR UTF-8\n") + + self.update_progressbar( _("Fetching GEDCOM...")) + + fn = self.connector.fetch_full_gedcom() - self.update_progressbar(_("Fetching index list...")) - steps = ( PHPGedViewConnector.TYPE_INDI, - PHPGedViewConnector.TYPE_FAM, - PHPGedViewConnector.TYPE_SOUR, - PHPGedViewConnector.TYPE_REPO, - PHPGedViewConnector.TYPE_NOTE, - PHPGedViewConnector.TYPE_OBJE - ) - xref_list = [] - i = 0 - for type in steps: - self.update_progressbar( _("Fetching index list..."), i, len(steps)) - xref_list += self.connector.list_xrefs( type) - i += 1 - self.update_progressbar( _("Fetching records...")) - i = 0 - for xref in xref_list[:10]: - self.update_progressbar( _("Fetching records..."), i, len(xref_list)) - record = self.connector.get_record(xref) - for r in record: - outfile.write(r+"\n") - outfile.flush() - i += 1 - - outfile.flush() - self.update_progressbar( _("Importing GEDCOM...")) - - import ReadGedcom - ReadGedcom.importData(self.db, outfile.name) + + ReadGedcom.importData(self.db, fn) # done. bye. self.dialog.destroy() @@ -304,53 +350,91 @@ class phpGedViewImporter: def validate_server(self): try: self.update_progressbar(_("Connecting...")) - self.connector = PHPGedViewConnector(self.url) + self.connector = PHPGedViewConnector(self.url,self.update_progressbar) self.update_progressbar(_("Get version...")) version = self.connector.get_version() - self.version_label.set_text(_("Version %s") % version) - self.update_progressbar(_("Reading file list...")) - files = self.connector.list_gedcoms() - list_store = self.file_combo.get_model() - list_store.clear() - for file in files: - list_store.append([file[0],]) - self.file_combo.show() - self.username_entry.show() - self.password_entry.show() + if version: + self.version_label.set_text(_("Version %s") % version) + self.update_progressbar(_("Reading file list...")) + files = self.connector.list_gedcoms() + list_store = self.file_combo.get_model() + list_store.clear() + for file in files: + list_store.append([file[0],]) + self.file_combo.show() + self.username_entry.show() + self.password_entry.show() + return True - except (ValueError, urllib2.URLError), e: + except (ValueError, urllib2.URLError, httplib.InvalidURL), e: print e self.version_label.set_text(_("Error: Invalid URL")) - self.version_label.set_text(_("Error: Unable to connect to phpGedView")) self.update_progressbar(_("done.")) + return False - - -#------------------------------------------------------------------------- -# -# -# -#------------------------------------------------------------------------- -from PluginMgr import register_tool - -def phpGedViewImporterCaller(database,active_person,callback,parent=None): - phpGedViewImporter(database) - -register_tool( - phpGedViewImporterCaller, - _("Import the gedcom from a phpGedView driven website"), - category=_("Import"), - description=_("phpGedView is an open source web application that generates dynamic webpages" - " out of a GEDCOM file. This plugin uses the gedcom access protocol to" - " retrieve the gedcom file from the webserver.") - ) - +# TODO: This should go into PHPGedViewConnector +def filter_url( url): + url = url.split("?")[0] # strip params + if url[-1:] == "/": # strip trailing slash + url = url[:-1] + if url[-4:] in (".php",".htm") or url[-5:] in (".html"): # strip script name + idx = url.rfind("/") + if idx > 1: + url = url[:idx] + if url[:7] != "http://": # add protocol + url = "http://"+url + if url[-1:] != "/": # readd trailing slash + url = url + "/" + return url # for Testing if __name__ == "__main__": + def dummy_progress( text,pos=0,max=0): + if max > 0: + percent = pos*100/max + print "%s: %d%%" % (text,percent) + else: + print text - phpGedViewImporter(None) - gtk.main() + try: + f = open("/tmp/sites.txt") + l = f.readline() + while l: + l = filter_url(l.strip()) + print l + try: + c = PHPGedViewConnector(l,dummy_progress) + c.connect_to_gedcom() + v = c.get_version() + if v: + print("%s\t\t%s" % (v,l)) + c.fetch_full_gedcom() + except KeyboardInterrupt: + exit + l = f.readline() + except IOError: + phpGedViewImporter(None) + gtk.main() + +else: + #------------------------------------------------------------------------- + # + # + # + #------------------------------------------------------------------------- + from PluginMgr import register_tool + + def phpGedViewImporterCaller(database,active_person,callback,parent=None): + phpGedViewImporter(database) + + register_tool( + phpGedViewImporterCaller, + _("Import the gedcom from a phpGedView driven website"), + category=_("Import"), + description=_("phpGedView is an open source web application that generates dynamic webpages" + " out of a GEDCOM file. This plugin uses the gedcom access protocol to" + " retrieve the gedcom file from the webserver.") + )