[csw-devel] SF.net SVN: gar:[17900] csw/mgar/gar/v2/lib/python

[wahwah at users.sourceforge.net]

Revision: 17900
          http://gar.svn.sourceforge.net/gar/?rev=17900&view=rev
Author:   wahwah
Date:     2012-04-30 07:14:24 +0000 (Mon, 30 Apr 2012)
Log Message:
-----------
integrate-catalogs: Cache more data locally

Run faster! Faster! Faster!

Modified Paths:
--------------
    csw/mgar/gar/v2/lib/python/integrate_catalogs.py
    csw/mgar/gar/v2/lib/python/rest.py
    csw/mgar/gar/v2/lib/python/safe_remove_package.py

Modified: csw/mgar/gar/v2/lib/python/integrate_catalogs.py
===================================================================
--- csw/mgar/gar/v2/lib/python/integrate_catalogs.py	2012-04-30 07:08:51 UTC (rev 17899)
+++ csw/mgar/gar/v2/lib/python/integrate_catalogs.py	2012-04-30 07:14:24 UTC (rev 17900)
@@ -242,7 +242,7 @@
         options.include_version_changes)
     bundles_by_md5 = {}
     bundles_missing = set()
-    cp = rest.CachedPkgstats("pkgstats.db")
+    cp = rest.CachedPkgstats("pkgstats")
     for key in catalogs:
       for pkg in catalogs[key]:
         # logging.debug("%r", pkg)

Modified: csw/mgar/gar/v2/lib/python/rest.py
===================================================================
--- csw/mgar/gar/v2/lib/python/rest.py	2012-04-30 07:08:51 UTC (rev 17899)
+++ csw/mgar/gar/v2/lib/python/rest.py	2012-04-30 07:14:24 UTC (rev 17900)
@@ -95,9 +95,10 @@
 
   def __init__(self, filename):
     self.filename = filename
-    self.d = gdbm.open(filename, "c")
+    self.d = gdbm.open("%s.db" % self.filename, "c")
     self.rest_client = RestClient()
     self.local_cache = {}
+    self.deps = gdbm.open("%s-deps.db" % self.filename, "c")
 
   def GetPkgstats(self, md5):
     if md5 in self.local_cache:
@@ -109,3 +110,13 @@
       self.d[md5] = cjson.encode(pkgstats)
       self.local_cache[md5] = pkgstats
       return pkgstats
+
+  def GetDeps(self, md5):
+    if str(md5) in self.deps:
+      return cjson.decode(self.deps[md5])
+    else:
+      pkgstats = self.GetPkgstats(md5)
+      data = {"deps": pkgstats["depends"],
+              "pkgname": pkgstats["basic_stats"]["pkgname"]}
+      self.deps[md5] = cjson.encode(data)
+      return data

Modified: csw/mgar/gar/v2/lib/python/safe_remove_package.py
===================================================================
--- csw/mgar/gar/v2/lib/python/safe_remove_package.py	2012-04-30 07:08:51 UTC (rev 17899)
+++ csw/mgar/gar/v2/lib/python/safe_remove_package.py	2012-04-30 07:14:24 UTC (rev 17900)
@@ -21,12 +21,15 @@
 import subprocess
 
 
+class Error(Exception):
+  """A generic error."""
+
 class RevDeps(object):
 
   def __init__(self):
     self.cached_catalogs = {}
     self.rest_client = rest.RestClient()
-    self.cp = rest.CachedPkgstats("pkgstats.db")
+    self.cp = rest.CachedPkgstats("pkgstats")
 
   def MakeRevIndex(self, catrel, arch, osrel):
     key = (catrel, arch, osrel)
@@ -42,16 +45,12 @@
     for pkg_simple in catalog:
       # pprint.pprint(pkg_simple)
       md5 = pkg_simple["md5_sum"]
-      pkg = self.cp.GetPkgstats(md5)
-      if not pkg:
-        logging.warning("No package for %r", md5)
-        continue
-      for dep_pkgname, _ in pkg["depends"]:
+      # pkg = self.cp.GetPkgstats(md5)
+      short_data = self.cp.GetDeps(md5)
+      pkgname = short_data["pkgname"]
+      for dep_pkgname, _ in short_data["deps"]:
         rev_dep_set = rev_deps.setdefault(dep_pkgname, list())
-        rev_dep_set.append((md5, pkg["basic_stats"]["pkgname"]))
-      sys.stdout.write(".")
-      sys.stdout.flush()
-    sys.stdout.write("\n")
+        rev_dep_set.append((md5, pkgname))
     self.cached_catalogs[key] = rev_deps
     with open(fn, "w") as fd:
       fd.write(cjson.encode(self.cached_catalogs[key]))

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.