1 #!/usr/bin/env python

        2 """

        3 USAGE:

        4   python dangerSync.py --dev --user=connolly_sim --passwd=xyz --get=contact

        5 or

        6   python dangerSync.py --prod --user=XYZ --passwd=PDQ  --get=contact

        7 

        8 

        9 $Id: dangerSync.py,v 1.20 2007/04/17 22:24:02 connolly Exp $

       10 see change log at end

       11 

       12 """

       13 

       14 import shelve

       15 import types, operator

       16 from xml.sax import saxutils

       17 from xmlrpclib import Server

       18 from warnings import warn

       19 

       20 # http://www.python.org/doc/current/lib/module-getopt.html

       21 import getopt, sys

       22 

       23 def main():

       24     try:

       25         opts, args = getopt.getopt(sys.argv[1:],

       26                                    "hdru:p:s:g:t:z:x:n:t:",

       27                                    ["help",

       28                                     "dev", "prod",

       29                                     "user=", "passwd=", "session=", "pgp",

       30                                     "get=", "post=", "zap=",

       31                                     "xml=", "newer=",

       32 				    "test"])

       33     except getopt.GetoptError:

       34         usage()

       35         sys.exit(2)

       36 

       37     db_get = None

       38     db_post = None

       39     db_zap = None

       40     db_dump = None

       41     newer = None

       42     

       43     peer = Anchored()

       44 

       45     for o, a in opts:

       46         if o in ("-h", "--help"):

       47             usage()

       48             sys.exit()

       49 	elif o in ("-t", "--test"):

       50 	    _test()

       51 	    sys.exit()

       52         elif o == '-u' or o == '--user':

       53             userName = a # e.g. connolly_sim or danc

       54         elif o == "-p" or o == '--passwd':

       55             passwd = a

       56             print peer.login(userName, passwd)

       57         elif o == '--pgp':

       58             import credstore

       59             # str() to fix:

       60             # TypeError: cannot marshal <type 'dbus.String'> objects

       61             print peer.login(userName, str(credstore.decrypt()))

       62         elif o == "-d" or o == '--dev':

       63             pass # this is the default

       64         elif o == '--client':

       65             peer.setClient(a)

       66         elif o == "-r" or o == '--prod':

       67             peer.useProduction()

       68         elif o == "-s" or o == '--session':

       69             print "resuming...", a

       70             peer.resume(a)

       71         elif o == '-g' or o == '--get':

       72             db_get = a

       73         elif o == '-t' or o == '--post':

       74             db_post = a

       75         elif o == '-z' or o == '--zap':

       76             db_zap = a

       77         elif o == '-x' or o == '--xml':

       78             db_dump = a

       79         elif o == '-n' or o == '--newer':

       80             newer = a

       81 

       82 

       83     if newer:

       84         db = shelve.open('event', flag="r")

       85         dump_newer(db, newer)

       86 

       87     if db_dump:

       88         dump_kinds(db_dump.split(","))

       89 

       90     if db_zap:

       91         for name in db_zap.split(","):

       92             zap(peer, name)

       93 

       94     if db_get:

       95         for name in db_get.split(","):

       96             db = shelve.open(name, flag='c')

       97             download(db, peer, name)

       98 

       99     if db_post:

      100         for name in db_post.split(","):

      101             db = shelve.open(name, flag="r")

      102             upload(db, peer, name)

      103 

      104 

      105 def _test():

      106     import doctest

      107     doctest.testmod()

      108 

      109 def zap(peer, kind):

      110     print >>sys.stderr, "@@zap kind: ", kind

      111     

      112     ids = peer.allItemIds(kind)

      113     print >>sys.stderr, "@@deleting: ", ids

      114     peer.delete(kind, ids)

      115 

      116 

      117 def download(db, peer, kind):

      118     print >>sys.stderr, "@@download kind: ", kind

      119     

      120     anchor = db.get('anchor', None)

      121     if anchor:

      122         peer.anchor = anchor

      123         ids = peer.deletedItemIds(kind)

      124         for id in ids:

      125             try:

      126                 del db[`id`]

      127             except KeyError:

      128                 warn("what happened to %d?" % id)

      129         ids = peer.changedItemIds(kind)

      130     else:

      131         ids = peer.allItemIds(kind)

      132 

      133     for record in peer.get(kind, ids):

      134         k = `record['id']`

      135         print >>sys.stderr, "@@getting: ", k, record.get('title', '')

      136         db[k] = record

      137     db['anchor'] = peer.resetAnchor()

      138 

      139 

      140 def upload(db, peer, kind):

      141     print >>sys.stderr, "@@upload kind: ", kind

      142 

      143     records = []

      144     for k in db.keys():

      145         if k == 'anchor': continue

      146         ki = int(k)

      147         print >>sys.stderr, "@@queing for upload: ", k

      148         records.append(db[k])

      149 

      150     i = 0

      151     while i < len(records):

      152         recs = records[i:i+30]

      153         ids_old = []

      154         for r in recs:

      155             ids_old.append(r.get('id', None))

      156         print >>sys.stderr, "@@uploading %d records... " % len(recs)

      157         ids_new = peer.post(kind, recs)

      158         print zip(ids_old, ids_new)

      159         i = i + 30

      160 

      161 

      162 

      163 def dump_kinds(kinds):

      164     dump_start()

      165     for name in kinds:

      166         db = shelve.open(name, flag="r")

      167         todo = db.keys()

      168         todo.sort()

      169         for k in todo:

      170             if k == 'anchor':

      171                 dump_anchor(name, db[k])

      172             else:

      173                 dumpItem(name, db[k], 0)

      174     print "</rdf:RDF>"

      175 

      176 def dump_start():

      177     print "<rdf:RDF xmlns:rdf='http://www.w3.org/1999/02/22-rdf-syntax-ns#' xmlns='http://dev.w3.org/2001/palmagent/danger#'>"

      178 

      179 def dump_anchor(kind, anchor):

      180     print "<rdf:Description rdf:about='#%s' rdf:value='%s'/>" % (kind, anchor)

      181     

      182 def dump_newer(db, newer):

      183     dump_start()

      184     todo = db.keys()

      185     todo.sort()

      186     for k in todo:

      187         if k == 'anchor':

      188             dump_anchor('event', db[k])

      189         else:

      190             item = db[k]

      191             if item.get('repeat_type', 0) == 0:

      192                 if item.get('end_date', '9999') < newer: continue

      193                 if item.get('start_date', '9999') < newer: continue

      194             else:

      195                 if (item.get('repeat_end_date', '9999') or '9999') < newer: continue

      196                 

      197             dumpItem('event', db[k], 0)

      198     print "</rdf:RDF>"

      199 

      200 def usage():

      201     print __doc__

      202 

      203 

      204 def dumpItem(name, item, indent):

      205     spaces = ' ' * indent

      206 

      207     if (item == ''):

      208         #hmm... it seems that empty strings

      209         #mean "value not recorded" rather than

      210         #     "value is an empty string".

      211         # any exceptions?

      212         #print "%s<%s />" % (spaces, name)

      213         pass

      214     elif operator.isNumberType(item):

      215         print "%s<%s rdf:datatype='http://www.w3.org/2001/XMLSchema#integer'>%s</%s>" % (spaces, name, item, name)

      216     elif isinstance(item, types.StringTypes):

      217         print "%s<%s>%s</%s>" % (spaces, name, saxutils.escape(item).encode('UTF-8'), name)

      218     elif operator.isSequenceType(item):

      219         if len(item) == 0:

      220             print "%s<%s rdf:parseType='Collection'/>" % (spaces, name)

      221         else:

      222             print "%s<%s rdf:parseType='Collection'>" % (spaces, name)

      223             for j in item:

      224                 dumpItem("rdf:Description", j, indent + 2)

      225             print "%s</%s>" % (spaces, name)

      226 

      227     elif operator.isMappingType(item):

      228         id = item.get('id', None)

      229         if id is None:

      230             print "%s<%s>" % (spaces, name)

      231         else:

      232             print "%s<%s rdf:ID='_%s'>" % (spaces, name, id)

      233         for name2, value2 in item.iteritems():

      234             dumpItem(name2, value2, indent + 2)

      235         print "%s</%s>" % (spaces, name)

      236     else:

      237         print "%s  {unknown item %s }" % (spaces, item)

      238 

      239 

      240 def asDate(item):

      241     """punctuate item as per ISO8601 and W3C XML Schema datatypes

      242 

      243     >>> asDate("20041225")

      244     '2004-12-25'

      245     >>> asDate("20041202T170000Z")

      246     '2004-12-02T17:00:00Z'

      247     """

      248     if len(item) == 16:

      249 	return "%s-%s-%s:%s:%s" % (item[:4], item[4:6], item[6:11],

      250 				   item[11:13] ,item[13:])

      251     elif len(item) == 8:

      252 	return "%s-%s-%s" % (item[:4], item[4:6], item[6:8])

      253     else:

      254 	raise ValueError, item

      255 

      256 

      257 #######

      258 import xmlrpclib

      259  

      260 DEBUG=0

      261  

      262 class PimSyncServer (xmlrpclib.ServerProxy):

      263     def __init__ (self, url):

      264         self._url = url

      265         xmlrpclib.ServerProxy.__init__ (self, url, verbose=DEBUG)

      266  

      267  

      268 class DangerService:

      269     """ A connection to Danger PIMAPI user data calls XMLRPC service

      270     http://developer.danger.com/wiki/space/user+data+calls

      271 

      272     maintains session state, but not anchor state

      273     """

      274 

      275     def __init__(self):

      276         """ default to connect to the developer service

      277         """

      278         self._addr = "http://sync.developer.danger.com:5014/pimapi"

      279         self._carrier = "danger"

      280         self._client = {}

      281 

      282     def useProduction(self):

      283         self._addr = "http://pimapi.prod1.dngr.net/pimapi"

      284         self._carrier = 'voicestream'

      285 

      286     def setClient(self, uastring, version='1.0'):

      287         self._client = {'Client': uastring,

      288                         'Version': version}

      289 

      290     def login(self, username, password):

      291         self._server = PimSyncServer(self._addr)

      292         # pprint(self._server.get_carriers(), sys.stderr)

      293         self._session = self._server.create_session (self._carrier, username,

      294                                                      password,

      295                                                      self._client)

      296         return self._session

      297 

      298     def resume(self, session):

      299         self._server = PimSyncServer(self._addr)

      300         self._session = session

      301 

      302     # keep sessions open just in case

      303     #def __del__ (self):

      304     #    self._server.end_session(self._session)

      305  

      306     def allItemIds(self, type):

      307         return self._server.get_all_item_ids(self._session, type)

      308  

      309     def post(self, type, vector):

      310         return self._server.new_item (self._session, type, vector)

      311  

      312     def put(self, string, vector):

      313         return self._server.update_item (self._session, string, vector)

      314  

      315     def get(self, string, vector):

      316         return self._server.get_item (self._session, string, vector)

      317  

      318     def delete(self, string, vector):

      319         return self._server.delete_item (self._session, string, vector)

      320 

      321 

      322 class Anchored(DangerService):

      323     def resetAnchor(self):

      324         self.anchor = self._server.get_sync_anchor (self._session)

      325         return self.anchor

      326     

      327     def deletedItemIds(self, type):

      328         return self._server.get_deleted_item_ids(self._session, type,

      329                                                  self.anchor)

      330  

      331     def changedItemIds(self, type):

      332         return self._server.get_changed_item_ids (self._session,

      333                                                   type, self.anchor)

      334  

      335 

      336  

      337     

      338 if __name__ == '__main__':

      339     main()

      340 

      341 

      342 # $Log: dangerSync.py,v $

      343 # Revision 1.19  2007/01/01 01:06:21  connolly

      344 # show title of record in download()

      345 #

      346 # Revision 1.18  2006/07/14 07:22:16  connolly

      347 # include client info in login for write operations

      348 #

      349 # Revision 1.17  2006/06/30 01:25:37  connolly

      350 # use rdf:Description rather than foo_item in lists

      351 #

      352 # Revision 1.16  2005/02/17 22:50:45  connolly

      353 # when dumping rdf/xml, sort items by key

      354 #

      355 # Revision 1.15  2004/12/31 18:38:52  connolly

      356 # emit flattened lists as well as collections

      357 #

      358 # Revision 1.14  2004/12/31 17:13:02  connolly

      359 # date punctuation is now taken care of in dangerSync.py

      360 # dangerSync.py now has some doctest stuff, invoked using --test

      361 #

      362 # Revision 1.13  2004/12/31 16:40:49  connolly

      363 # open read-only when we can so that Makefiles work

      364 #

      365 # Revision 1.12  2004/11/13 16:46:18  connolly

      366 # prepend _ to make numeric IDs into XML names

      367 #

      368 # Revision 1.11  2004/10/23 18:41:31  connolly

      369 # oops; fixed previously-untested empty string handling

      370 #

      371 # Revision 1.10  2004/10/23 18:21:19  connolly

      372 # skipping empty strings

      373 #

      374 # Revision 1.9  2004/04/19 16:27:48  connolly

      375 # update namespace name

      376 #

      377 # Revision 1.8  2004/02/23 07:40:01  connolly

      378 # - handle KeyError in delete

      379 # - fixed --newer for repeating events

      380 #

      381 # Revision 1.7  2004/02/16 00:04:00  connolly

      382 # added --newer to get events that affect the future

      383 #

      384 # Revision 1.6  2004/02/15 20:31:15  connolly

      385 # on post, print correspondence between old ids and new

      386 #

      387 # Revision 1.5  2004/02/15 08:04:42  connolly

      388 # - use item id as rdf:ID

      389 #

      390 # Revision 1.4  2004/02/15 07:52:51  connolly

      391 # - moved zap before post

      392 # - dump as RDF/XML

      393 # - resume session (not successfully tested)

      394 #

      395 # Revision 1.3  2004/02/13 14:10:18  connolly

      396 # zap, upload seem to be working...

      397 # well enough to contact,task,note,event data

      398 # from my one phone to the other

      399 #

      400 # Revision 1.2  2004/02/12 22:18:45  connolly

      401 # downloading all 4 kinds from production server worked.

      402 # upload code untested

      403 #

      404 # Revision 1.1  2004/02/12 22:02:38  connolly

      405 # anchored downloading works

      406 #