2 hipAgent.py -- danger sidekick hiptop agent, via desktop interface
3 $Id: hipAgent.py,v 1.28 2007/09/03 17:28:24 connolly Exp $
9 $ python hipAgent.py --number=2025551212 --passwd=xyz --importItinerary <<EOF
10 2005-02-26 lv MCI 15:00 ar ORD 16:27 Saturday AMERICAN AIRLINES #1094
11 2005-02-26 lv ORD 17:29 ar BOS 20:46 Saturday AMERICAN AIRLINES #874
14 See parseFlight() below if you'd rather use a different format.
16 There are some test routines, but they're probably bitrotten.
18 python hipAgent.py --number telnum --passwd passwd --datebook
19 to grab your calendar homepage and extract some pointers
21 python hipAgent.py --number telnum --passwd passwd --contact who
22 to grab a pointer to the page about a contact named who
26 login technique reverse-engineered using ClientCookie
27 http://wwwsearch.sourceforge.net/ClientCookie/
28 The technique described in http://hipme.com/software/appointment/
32 LICENSE: Share and Enjoy.
33 Copyright (c) 2001 W3C (MIT, INRIA, Keio)
35 http://www.w3.org/Consortium/Legal/copyright-software-19980720
43 import urllib, urllib2
44 from urllib2 import urlopen
45 from string import lower
49 import dangerSync # for posting events
50 import hipsrv # for asHiptopDate
53 # http://www.mnot.net/python/HtmlDom.py
54 # <AaronSw> it lets me do: d = fetch(url); print xml.xpath.Evaluate("//*[@class='rss:item']/text()", d)
55 #import HtmlDom # http://www.mnot.net/python/HtmlDom.py
62 sys.stderr.write('%s ' % a)
63 sys.stderr.write("\n")
66 class TMobileDangerAgent:
69 import ClientCookie # http://wwwsearch.sourceforge.net/ClientCookie/
70 ClientCookie.HTTP_DEBUG = 0
71 ClientCookie.CLIENTCOOKIE_DEBUG = 0
72 from ClientForm import ParseResponse # http://wwwsearch.sourceforge.net/ClientForm/
74 self._icon = 15 # default to airplane
76 def login(self, phone, passwd):
77 """set up cookies based on login credentials
79 we cheat here, using module state of ClientCookie.
80 We should keep an opener object here.
81 This interface will stay the same when we fix
82 that; i.e. the caller shouldn't assume
86 progress("loggin in with creds: ", phone, passwd)
88 form = ParseResponse(urlopen("https://my.t-mobile.com/login/MyTmobileLogin.aspx"))[0]
90 progress("=== start page form: ", form)
92 form["Login1:txtMSISDN"] = phone
93 form["Login1:txtPassword"] = passwd
94 arg = 'Login1$btnLogin'
95 form.find_control("__EVENTTARGET").readonly = False
96 form.find_control("__EVENTARGUMENT").readonly = False
97 form["__EVENTTARGET"] = ':'.join(arg.split('$'))
98 form["__EVENTARGUMENT"] = arg
99 request = form.click()
101 resp2 = ClientCookie.urlopen(request)
102 progress("=== login response headers: ", resp2.info())
105 logging.debug("resp2:" + html)
106 i = html.index("http://login.sidekick.dngr.com/")
108 addr = addr[:addr.index('"')]
110 progress("desktop addr:", addr)
111 resp3 = ClientCookie.urlopen(addr)
113 progress("=== desktop interface headers: ", resp3.info())
115 progress("logged in.")
118 def urlopen(self, addr, data=None):
119 req = urllib2.Request(addr, data=data)
120 resp = ClientCookie.urlopen(req)
124 def pageOfContactNamed(self, name):
125 letter = lower(name[0])
126 listAddr = 'http://address.sidekick.dngr.com/index?high=%s&low=%s' \
128 listDoc = parseContent(self.urlopen(listAddr))
129 progress("parsed as: ", listDoc)
130 nodes = listDoc.xpathNewContext().xpathEval( \
131 '//a[@class="text" and contains(text(), "%s")]' % name)
133 progress("got %d nodes about '%s'" % (len(nodes), name))
138 return nodes[0].prop('href')
140 def aboutDay(self, fp, yyyy_mm_dd):
141 """return a list of (addr, title) for each item on a day.
143 @@TODO: put time in there too.
147 <td align=right valign=top class=timeGridCell rowspan=1><a href='add?h=19&min=0&date=2003-06-12' class='pmTimeDisplay'>7:00</a></td>
148 <td height=20 class=pmGridCell></td>
149 <td valign=top width='100%' height='120' class='eventGridCell' colspan='1' rowspan='5'>
150 <table cellpadding=2 cellspacing=0 border=0>
152 <td width='100%' height='120' valign=top><span class='eventGridCellText'><a href='http://calendar.sidekick.dngr.com/event?id=310' class='data'><b>Royals</b> </a></span></td>
154 <td height='120' width=1><img src='http://img.sidekick.dngr.com/img/voicestream/pixel.gif' height='120' width='1' border=0></td>
162 pgAddr = 'http://calendar.sidekick.dngr.com/day?date=' + yyyy_mm_dd
163 doc = parseContent(self.urlopen(pgAddr))
164 ctxt = doc.xpathNewContext()
165 nodes = ctxt.xpathEval("//a[@class='data']")
167 dayTerm = '<#theDay%s>' % yyyy_mm_dd
168 fp.write('<%s> foaf:topic %s.\n' % (pgAddr, dayTerm))
169 fp.write('%s xsdt:date "%s".\n' % (dayTerm, yyyy_mm_dd))
173 addr = ev.prop('href')
175 eid = int(addr[addr.find("id=")+3:])
177 eid = hash(addr) # darn. 2 names for same event
178 title = ev.content #@@n3-escape
179 evTerm = "<#event_%s>" % eid
180 fp.write('%s k:temporallyIntersects %s;\n' % (evTerm, dayTerm))
181 fp.write(' rdf:value """%s"""; is foaf:topic of <%s>.\n' %
183 events.append((addr, title))
189 def aboutMonth(self, year, month):
190 calAddr = 'http://calendar.sidekick.dngr.com/month?date=%04d-%d' % (year, month)
191 txt = self.urlopen(calAddr).read()
192 doc = HtmlDom._HTMLReader().fromString(txt)
193 nodes = xml.xpath.Evaluate('//TR/TD[@CLASS="monthWeekNav"]', doc)
197 nodes = xml.xpath.Evaluate( \
198 'SPAN/A[@CLASS="data"]', wkn)
200 addr = nodes[0].prop('href')
206 def aboutEvent(self, fp, id):
207 progress("== grabbing event:", id)
209 pgAddr = 'http://calendar.sidekick.dngr.com/edit?id=%d' % id
210 doc = parseContent(self.urlopen(pgAddr))
213 ctxt = doc.xpathNewContext()
214 form = ctxt.xpathEval("//form[@id='calendar_add_form']")[0]
215 ctxt.xpathFreeContext()
217 props = formProps(doc, form)
218 progress("event properties:", props.keys())
220 evTerm = "<#event_%s>" % id
222 fp.write('<%s> foaf:topic [= %s; \n' % (pgAddr, evTerm))
225 'e_icon_id', # hint at type
241 'provides_starttime',
245 'provides_starttime',
251 fp.write(' dngr:%s """%s""";\n' % (p, props[p]))
256 def addTimedEvent(self, yyyy_mm_dd, t1, title, desc,
257 reminder_increment='minutes',
260 addr = 'http://calendar.sidekick.dngr.com/add?type=timed&date=%s' % yyyy_mm_dd
261 forms = ParseResponse(self.urlopen(addr))
262 progress("forms: ", [ f.name for f in forms])
263 form = [ f for f in forms if f.name == "calendar_add_form" ][0]
264 progress("add form:", form)
265 form['e_title'] = title
266 form['e_location'] = where
267 form['hour_start'] = [str(int(t1[0:2]))] # strip leading 0
269 # the form has a limited selection of minute values
270 # but the danger service groks others.
271 form.find_control('min_start')._selected = t1[3:5]
273 #@@form['reminder_type'] = '0' # in desktop iface, this is on a separate form
274 #@@form['reminder_increment_0'] = reminder_increment
275 #@@form['reminder_length_0'] = str(reminder_length)
276 form['e_notes'] = desc
277 form['e_icon_id'] = ['%d' % self._icon]
279 progress('POSTing add request')
280 res = ClientCookie.urlopen(form.click()) #@@
281 progress('POST reply headers: ', res.info())
283 # debugging... save response
284 #open(title, "w").write(res.read())
287 #Set-Cookie: calendar_last_visited=%2Fevent%3Fid%3D439; expires=...
291 def parseContent(pg):
292 import libxml2 # http://xmlsoft.org/python.html
296 progress("Content: ", txt[:50])
298 # xhtml-ize a little bit
299 # fix &foo= ==> &foo=
300 txt = re.sub(r'&(\w+)=', r'&\1=', txt)
301 txt = txt.replace(' ', ' ')
302 txt = txt.replace('©', '©')
303 txt = txt.replace(' SELECTED>', ' selected="selected">')
304 txt = txt.replace(' CHECKED>', ' checked="checked">')
306 print >>sys.stderr, "@@txt type", type(txt)
308 #@@return libxml2.htmlParseDoc(txt, 'us-ascii')
309 return libxml2.htmlParseDoc(txt, None)
312 def parseFlight(flt):
313 """ separate flight line into date, departure, short description
315 >>> parseFlight('2005-02-08 lv MCI 15:00 ar ORD 16:27 Tuesday AMERICAN AIRLINES #1094')
316 ('2005-02-08', '15:00', 'flt#1094 ->ORD')
319 desc = 'flt' + flt[flt.rindex("#"):] + ' ->' + flt[27:30]
320 return flt[:10], flt[18:23], desc
323 def importItinerary(phone, passwd, fp):
324 a = TMobileDangerAgent()
325 a.login(phone, passwd)
329 date, ti, desc = parseFlight(flt)
330 progress("== adding event: [%s] %s" % (desc, flt))
332 a.addTimedEvent(date, ti, desc, flt, 'minutes', 45)
334 def importTimedEvents(phone, passwd, fp, icon):
335 """import timed events from fp
337 read tab-delimited event descriptions:
338 yyyy-mm-dd hh:mm title desc location
339 and import them into the sidekick calendar
342 a = TMobileDangerAgent()
343 a.login(phone, passwd)
344 if icon: a._icon = icon
351 progress("== adding event:", ev.split("\t"))
353 dt, ti, titl, desc, loc = ev.split("\t")
354 a.addTimedEvent(dt, ti,
356 'minutes', 45, #@@hardcoded
359 def importTimedRDF(phone, passwd, client, kb, icon):
360 """import timed events from RDF kb
362 supports timed events only
363 summary, dtstart required
364 loc, description optional
373 progress("@@scanning RDF:", kb)
375 for ev in calitems.eachEvent(kb):
376 progress("== found event:", ev['title'])
380 progress("peer.post", pprint.pformat(records))
382 peer = dangerSync.Anchored()
384 peer.setClient(client)
385 peer.login(phone, passwd)
386 ids = peer.post("event", records)
387 progress("post ids:", ids)
389 def testContact(phone, passwd, contact):
390 a = TMobileDangerAgent()
391 a.login(phone, passwd)
393 progress("== grabbing contact")
395 pg = a.pageOfContactNamed(contact)
397 progress("== got it", pg)
400 def testCalPage(phone, passwd):
401 a = TMobileDangerAgent()
402 a.login(phone, passwd)
404 progress("== grabbing calendar")
406 pg = a.urlopen('http://calendar.sidekick.dngr.com/')
408 progress("=== calendar headers: ", pg.info())
410 doc = parseContent(pg)
411 print doc.children #@@
413 ctxt = doc.xpathNewContext()
414 res = ctxt.xpathEval("//a[@class='data']")
417 progress("a data link:", n.prop('href'), n.children)
420 ctxt.xpathFreeContext()
425 def formProps(doc, form):
426 """return a dictionary of field name/value pairs.
428 @@sometimes there's more than one value per name; that
429 case isn't handled here.
432 ctxt = doc.xpathNewContext()
433 ctxt.setContextNode(form)
437 fields = ctxt.xpathEval(".//input")
439 ty = field.prop('type') or 'text' # defaults to text
440 name = field.prop('name')
442 if ty == 'hidden' or ty == 'text':
443 v = field.prop('value')
445 elif (ty == 'radio'):
446 if field.prop('checked'):
447 v = field.prop('value') or field.content
449 elif ty == 'image' or ty == 'submit':
452 progress("NOTIMPL: form input; name:", name,
456 fields = ctxt.xpathEval(".//select")
459 name = f.prop('name')
460 progress("form select; name:", name)
463 v = o.prop('value') or o.content
464 #progress(" option value:", v, "selected?", o.prop('selected'))
465 if o.prop('selected'): props[name] = v
468 fields = ctxt.xpathEval(".//textarea")
471 name = f.prop('name')
472 progress("textarea name:", f.prop('name'))
473 props[name] = f.content
475 ctxt.xpathFreeContext()
480 def testCalDay(phone, passwd, dt):
483 a = TMobileDangerAgent()
484 a.login(phone, passwd)
486 progress("== grabbing calendar day", dt)
488 res = a.aboutDay(sys.stdout, dt)
490 progress("== res:", res)
493 def testEvent(phone, passwd, id):
496 a = TMobileDangerAgent()
497 a.login(phone, passwd)
499 progress("== grabbing event", id)
501 a.aboutEvent(sys.stdout, id)
504 def testTrip(outFp, phone, passwd, id):
505 a = TMobileDangerAgent()
506 a.login(phone, passwd)
509 outFp.write("""@prefix dngr: <http://dev.w3.org/2001/palmagent/dngr@@#>.
510 @prefix foaf: <http://xmlns.com/foaf/0.1/>.
511 @prefix xsdt: <http://www.w3.org/2001/XMLSchema#>.
512 @prefix k: <http://opencyc.sourceforge.net/daml/cyc.daml#>.
513 @prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>.
516 progress("== grabbing trip event", id)
518 tripProps = a.aboutEvent(outFp, id)
519 days = int(tripProps['e_day_dur'])
520 year = int(tripProps['year_start'])
521 month = int(tripProps['month_start'])
522 day = int(tripProps['day_start'])
523 secs = calendar.timegm((year, month, day, 0, 0, 0, 0, 0, 0))
526 year, month, day, hour, minute, second, wday, jday, dst = time.gmtime(secs)
527 ymd = "%04d-%02d-%02d" % (year, month, day)
528 progress("trip includes day:", ymd)
529 events = a.aboutDay(outFp, ymd)
530 for (addr, title) in events:
531 progress("found event", title, " on ", ymd)
532 qs = addr[addr.rindex('id=')+3:] # ...id=434&foo => 434&foo
533 try: qs = qs[:qs.index('&')]
534 except ValueError: pass
536 a.aboutEvent(outFp, eid)
538 secs = secs + 60 * 60 * 24
547 if __name__ == '__main__':
551 opts, args = getopt.getopt(sys.argv[1:],
553 ["datebook", "contact=",
554 "number=", "passwd=",
559 "testEvent=", "testTrip=",
564 except getopt.GetoptError:
565 # print help information and exit:
576 if o in ("-p", "--passwd"):
578 elif o in ("-n", "--number"):
580 elif o == '--client':
582 elif o in ("-i", "--icon"):
584 elif o in ("-n", "--contact"):
586 testContact(number, passwd, contact)
587 elif o in ("-d", "--datebook"):
588 testCalPage(number, passwd)
589 elif o in ("--testDay",):
591 testCalDay(number, passwd, dt)
592 elif o in ("--testEvent",):
594 testEvent(number, passwd, evid)
595 elif o in ("--output",):
597 elif o in ("--testTrip",):
599 testTrip(outFp, number, passwd, evid)
600 elif o in ("--test",):
602 elif o in ("--importItinerary",):
603 importItinerary(number, passwd, sys.stdin)
604 elif o in ("--importTimed",):
605 importTimedEvents(number, passwd, sys.stdin, icon)
606 elif o in ("--importRDF",):
608 kb = calitems.fn2kb(a)
609 progress("loaded RDF:", kb)
610 import credstore #@@ do this only if cmd line flag
611 importTimedRDF(number, str(credstore.decrypt()), client, kb, icon)
614 # $Log: hipAgent.py,v $
615 # Revision 1.27 2007/05/04 20:41:09 connolly
616 # tweak dates before uploading to danger
618 # Revision 1.26 2007/01/06 00:03:46 connolly
619 # used credstore to import
621 # Revision 1.25 2006/12/30 06:43:18 connolly
622 # handle all-day events in importRDF
623 # show results of post
624 # import ClientCookie, libxml2 only if needed
626 # Revision 1.24 2006/07/14 07:19:22 connolly
627 # using hipsrv for date/tz conversion
628 # using dangerSync for RDF upload, since t-mobile's web site login changed
629 # and I figured out the XMLRPC write issues
631 # Revision 1.23 2006/05/16 19:59:14 connolly
632 # normalize space in ical descriptions
634 # Revision 1.22 2006/05/03 20:34:26 connolly
635 # support import of RDF events without locations
636 # use logging for debug notices
638 # Revision 1.21 2006/03/26 05:37:39 connolly
639 # not quite so verbose diagnostics
641 # Revision 1.20 2006/03/26 05:32:10 connolly
642 # new login sequence seems to work
644 # Revision 1.19 2005/12/27 20:37:17 connolly
645 # select 2nd form by name
647 # Revision 1.18 2005/11/08 23:03:25 connolly
648 # location support in importTimed
650 # Revision 1.17 2005/09/08 03:11:56 connolly
651 # add --icon command-line option
653 # Revision 1.16 2005/02/05 04:33:51 connolly
654 # updated __doc__ a bit
656 # Revision 1.15 2005/02/05 04:20:14 connolly
657 # strip leading 0s on hours
659 # Revision 1.14 2005/02/05 04:14:18 connolly
660 # uploaded an itinerary!
662 # Revision 1.13 2005/02/05 04:02:30 connolly
663 # yes! added an event! now... just need to override list choices for minutes
665 # Revision 1.12 2003/11/08 17:46:10 connolly
666 # timed events, for basketball schedule, based on importItinerary
668 # Revision 1.11 2003/07/19 19:15:39 connolly
669 # cleaned up event naming
671 # Revision 1.10 2003/07/19 18:49:19 connolly
672 # parseDoc api change?
674 # Revision 1.9 2003/05/16 13:57:50 connolly
677 # Revision 1.8 2003/04/18 16:46:26 connolly
678 # --testTrip now writes real N3
680 # Revision 1.7 2003/03/27 23:56:22 connolly
681 # generalized --testApr just a bit, to --importItin
683 # Revision 1.6 2003/03/13 05:06:05 connolly
684 # testDay: get a day's worth of stuff in N3
685 # testApr: POSTing some events
687 # Revision 1.5 2003/03/03 16:23:44 connolly
688 # - got one test for aboutDay working.
689 # - started playing with pure-python HTML/DOM/XPath
690 # replacement for libxml2 (not winning yet)
692 # Revision 1.4 2003/01/30 16:33:22 connolly
693 # 1Jan: clues for eliminating dependency on libxml2 in favor of PyMXL
695 # Revision 1.3 2002/12/19 22:08:17 connolly
696 # initial foray into addressbook navigation
697 # converted options processing to getopt
699 # Revision 1.2 2002/12/14 21:10:12 connolly
700 # extracting links from html using libxml2 works
702 # Revision 1.1 2002/12/14 18:46:34 connolly