[csw-devel] SF.net SVN: opencsw:[574] twitter/sweets/sweets.py

[skayser at users.sourceforge.net]

Revision: 574
          http://opencsw.svn.sourceforge.net/opencsw/?rev=574&view=rev
Author:   skayser
Date:     2011-10-25 09:57:29 +0000 (Tue, 25 Oct 2011)
Log Message:
-----------
twitter/sweets: switch from pycurl to urllib, add state tracking, print exception msgs

Modified Paths:
--------------
    twitter/sweets/sweets.py

Modified: twitter/sweets/sweets.py
===================================================================
--- twitter/sweets/sweets.py	2011-10-25 08:44:42 UTC (rev 573)
+++ twitter/sweets/sweets.py	2011-10-25 09:57:29 UTC (rev 574)
@@ -5,15 +5,14 @@
 #   can be used for further processing (e.g. to generate RSS)
 #
 # TODO:
-# * Implement twitter_search (http://search.twitter.com/search.json?q=opencsw)
-# * twitter_search: catch and log twitter API HTTP status codes <> 200
-# * run: write result JSON to output file
-# * get_results: use refresh_url from response for follow up queries
+# * Add logging for debug purposes (URL fetches, reponses received)
+# * run: write or append result JSON to output file
 # * Rewrite file open/close blocks with "with" once python2.5+ on dev box
 #
 
 import json
-import pycurl
+import urllib
+import urllib2
 import sys
 import time
 import types
@@ -27,29 +26,30 @@
     f.close()
     return data
 
-def twitter_search_real(query, since=""):
+def twitter_search_real(query, since_id=""):
     """Search twitter, returns a single-line, JSON search result."""
-    output=StringIO.StringIO()
-    c = pycurl.Curl()
-    c.setopt(c.URL, 'http://search.twitter.com/search.json?q=' + query)
-    c.setopt(c.WRITEFUNCTION, output.write)
-    c.perform()
-    c.close()
-    return output.getvalue()
+    url_base = "http://search.twitter.com/search.json"
+    url_params = []
+    url_params.append(( "q" , query ))
+    if since_id: url_params.append(( "since_id" , since_id))
+    url = url_base + "?" + urllib.urlencode(url_params)
+    output = urllib2.urlopen(url).read()
+    return output
 
 twitter_search = twitter_search_real
 
-def get_results(query):
+def get_results(query, state):
     """Trigger twitter search and extract results from JSON response.""" 
-    json_data = twitter_search(query)
+    json_data = twitter_search(query, state.get('max_seen'))
     response = json.read(json_data)
+    state['max_seen'] = response.get('max_id')
     return response['results']
 
-def run(interval, query):
+def run(interval, query, state={}):
     """Run the main loop which queries Twitter and writes the output."""
     while 1:
         try:
-            results = get_results(query)
+            results = get_results(query, state)
             for result in results:
                 # Tweet URL #http://twitter.com/#!/<from_user>/status/<id_str>
                 msg = "%s - http://twitter.com/#!/%s/status/%s" % (
@@ -58,13 +58,14 @@
                         result['id_str']
                 )
                 print msg.encode(sys.stdout.encoding, 'replace')
-            if hasattr(interval, "__call__"): interval()
-            else: time.sleep(float(interval))
-        except pycurl.error:
-            print "Couldn't retrieve URL"
-        except json.ReadException:
-            print "Couldn't read JSON response"
+        except urllib2.URLError, e:
+            print "Couldn't retrieve URL:", e
+        except json.ReadException, e:
+            print "Couldn't read JSON response", e
 
+        if hasattr(interval, "__call__"): interval()
+        else: time.sleep(float(interval))
+
 if __name__ == '__main__':
     usage = "%prog -s <searchterm> -t <secs>"
     parser = OptionParser(usage=usage)

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.