[csw-devel] SF.net SVN: opencsw:[382] utilities/site_links.py

[wahwah at users.sourceforge.net]

Revision: 382
          http://opencsw.svn.sourceforge.net/opencsw/?rev=382&view=rev
Author:   wahwah
Date:     2011-07-12 13:24:05 +0000 (Tue, 12 Jul 2011)

Log Message:
-----------
site-links: Add a fetcher function

The function can be overridden in derived classes is necessary.

Modified Paths:
--------------
    utilities/site_links.py

Modified: utilities/site_links.py
===================================================================
--- utilities/site_links.py	2011-07-12 13:23:53 UTC (rev 381)
+++ utilities/site_links.py	2011-07-12 13:24:05 UTC (rev 382)
@@ -123,9 +123,8 @@
     def Executify(self):
         for site in self.sites:
             logging.debug("Fetching %s", site)
-            try:
-                url_d = urllib2.urlopen(site.url)
-                content = url_d.read()
+            content = self._GetUrl(site.url)
+            if content:
                 try:
                   soup = bs.BeautifulSoup(content)
                   for a_tag in soup('a'):
@@ -135,8 +134,6 @@
                 except bs.HTMLParseError, e:
                   logging.warning("Could not parse %s: %s"
                                   % (repr(site.url), e))
-            except urllib2.URLError, e:
-                logging.warning("Couldn't fetch %s: %s.", site, e)
         dot_code = self.EmitDot()
         fd = open("out.dot", "w")
         fd.write(dot_code)
@@ -193,6 +190,16 @@
             l.append("%s → %s" % (site_a, site_b))
         return l
 
+    def _GetUrl(self, url):
+        content = None
+        try:
+          url_d = urllib2.urlopen(url)
+          content = url_d.read()
+        except urllib2.URLError, e:
+          logging.warning("Couldn't fetch %s: %s.", url, e)
+        return content
+
+
 def main():
     cswlinks = OpenCswLinks(SITE_DATA)
     cswlinks.Executify()


This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.