[csw-devel] SF.net SVN: opencsw:[574] twitter/sweets/sweets.py
skayser at users.sourceforge.net
skayser at users.sourceforge.net
Tue Oct 25 11:57:29 CEST 2011
Revision: 574
http://opencsw.svn.sourceforge.net/opencsw/?rev=574&view=rev
Author: skayser
Date: 2011-10-25 09:57:29 +0000 (Tue, 25 Oct 2011)
Log Message:
-----------
twitter/sweets: switch from pycurl to urllib, add state tracking, print exception msgs
Modified Paths:
--------------
twitter/sweets/sweets.py
Modified: twitter/sweets/sweets.py
===================================================================
--- twitter/sweets/sweets.py 2011-10-25 08:44:42 UTC (rev 573)
+++ twitter/sweets/sweets.py 2011-10-25 09:57:29 UTC (rev 574)
@@ -5,15 +5,14 @@
# can be used for further processing (e.g. to generate RSS)
#
# TODO:
-# * Implement twitter_search (http://search.twitter.com/search.json?q=opencsw)
-# * twitter_search: catch and log twitter API HTTP status codes <> 200
-# * run: write result JSON to output file
-# * get_results: use refresh_url from response for follow up queries
+# * Add logging for debug purposes (URL fetches, reponses received)
+# * run: write or append result JSON to output file
# * Rewrite file open/close blocks with "with" once python2.5+ on dev box
#
import json
-import pycurl
+import urllib
+import urllib2
import sys
import time
import types
@@ -27,29 +26,30 @@
f.close()
return data
-def twitter_search_real(query, since=""):
+def twitter_search_real(query, since_id=""):
"""Search twitter, returns a single-line, JSON search result."""
- output=StringIO.StringIO()
- c = pycurl.Curl()
- c.setopt(c.URL, 'http://search.twitter.com/search.json?q=' + query)
- c.setopt(c.WRITEFUNCTION, output.write)
- c.perform()
- c.close()
- return output.getvalue()
+ url_base = "http://search.twitter.com/search.json"
+ url_params = []
+ url_params.append(( "q" , query ))
+ if since_id: url_params.append(( "since_id" , since_id))
+ url = url_base + "?" + urllib.urlencode(url_params)
+ output = urllib2.urlopen(url).read()
+ return output
twitter_search = twitter_search_real
-def get_results(query):
+def get_results(query, state):
"""Trigger twitter search and extract results from JSON response."""
- json_data = twitter_search(query)
+ json_data = twitter_search(query, state.get('max_seen'))
response = json.read(json_data)
+ state['max_seen'] = response.get('max_id')
return response['results']
-def run(interval, query):
+def run(interval, query, state={}):
"""Run the main loop which queries Twitter and writes the output."""
while 1:
try:
- results = get_results(query)
+ results = get_results(query, state)
for result in results:
# Tweet URL #http://twitter.com/#!/<from_user>/status/<id_str>
msg = "%s - http://twitter.com/#!/%s/status/%s" % (
@@ -58,13 +58,14 @@
result['id_str']
)
print msg.encode(sys.stdout.encoding, 'replace')
- if hasattr(interval, "__call__"): interval()
- else: time.sleep(float(interval))
- except pycurl.error:
- print "Couldn't retrieve URL"
- except json.ReadException:
- print "Couldn't read JSON response"
+ except urllib2.URLError, e:
+ print "Couldn't retrieve URL:", e
+ except json.ReadException, e:
+ print "Couldn't read JSON response", e
+ if hasattr(interval, "__call__"): interval()
+ else: time.sleep(float(interval))
+
if __name__ == '__main__':
usage = "%prog -s <searchterm> -t <secs>"
parser = OptionParser(usage=usage)
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
More information about the devel
mailing list