[csw-devel] SF.net SVN: gar:[17282] csw/mgar/gar/v2/lib/python/integrate_catalogs.py

[wahwah at users.sourceforge.net]

Revision: 17282
          http://gar.svn.sourceforge.net/gar/?rev=17282&view=rev
Author:   wahwah
Date:     2012-03-03 23:23:38 +0000 (Sat, 03 Mar 2012)
Log Message:
-----------
integrate-catalogs: Prep work for parallel fetch

The multiprocessing module is broken at the moment, a fix is underway.

Modified Paths:
--------------
    csw/mgar/gar/v2/lib/python/integrate_catalogs.py

Modified: csw/mgar/gar/v2/lib/python/integrate_catalogs.py
===================================================================
--- csw/mgar/gar/v2/lib/python/integrate_catalogs.py	2012-03-03 12:08:00 UTC (rev 17281)
+++ csw/mgar/gar/v2/lib/python/integrate_catalogs.py	2012-03-03 23:23:38 UTC (rev 17282)
@@ -101,12 +101,25 @@
 def GetDiffsByCatalogname(catrel_from, catrel_to, include_downgrades):
   rest_client = rest.RestClient()
   diffs_by_catalogname = {}
+  def GetCatalog(rest_client, r_catrel, r_arch, r_osrel):
+    catalog = rest_client.GetCatalog(r_catrel, r_arch, r_osrel)
+    return ((r_catrel, r_arch, r_osrel), catalog)
+  # TODO(maciej): Enable this once the multiprocessing module is fixed.
+  # https://www.opencsw.org/mantis/view.php?id=4894
+  # proc_pool = multiprocessing.Pool(20)
+  catalogs_to_fetch_args = []
   for arch in common_constants.PHYSICAL_ARCHITECTURES:
+    for osrel in common_constants.OS_RELS:
+      for catrel in (catrel_from, catrel_to):
+        catalogs_to_fetch_args.append((rest_client, catrel, arch, osrel))
+  # Convert this to pool.map when multiprocessing if fixed.
+  catalogs = dict(map(lambda x: GetCatalog(*x), catalogs_to_fetch_args))
+  for arch in common_constants.PHYSICAL_ARCHITECTURES:
     logging.debug("Architecture: %s", arch)
     for osrel in common_constants.OS_RELS:
       logging.debug("OS release: %s", osrel)
-      cat_from = rest_client.GetCatalog(catrel_from, arch, osrel)
-      cat_to = rest_client.GetCatalog(catrel_to, arch, osrel)
+      cat_from = catalogs[(catrel_from, arch, osrel)]
+      cat_to = catalogs[(catrel_to, arch, osrel)]
       # Should use catalog comparator, but the data format is different
       if cat_from is None:
         cat_from = []

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.