[csw-devel] SF.net SVN: gar:[10454] csw/mgar/gar/v2

wahwah at users.sourceforge.net wahwah at users.sourceforge.net
Tue Jul 6 10:25:10 CEST 2010


Revision: 10454
          http://gar.svn.sourceforge.net/gar/?rev=10454&view=rev
Author:   wahwah
Date:     2010-07-06 08:25:10 +0000 (Tue, 06 Jul 2010)

Log Message:
-----------
mGAR v2: checkpkg, terminal output tweaks: removed some debug messages and added progress bars for operations that take a long time during the whole catalog run.

Modified Paths:
--------------
    csw/mgar/gar/v2/bin/analyze_module_results.py
    csw/mgar/gar/v2/lib/python/checkpkg.py
    csw/mgar/gar/v2/lib/python/opencsw.py

Modified: csw/mgar/gar/v2/bin/analyze_module_results.py
===================================================================
--- csw/mgar/gar/v2/bin/analyze_module_results.py	2010-07-06 08:23:36 UTC (rev 10453)
+++ csw/mgar/gar/v2/bin/analyze_module_results.py	2010-07-06 08:25:10 UTC (rev 10454)
@@ -1,9 +1,11 @@
 #!/opt/csw/bin/python2.6
 # $Id$
 
+import itertools
 import operator
 import optparse
 import os
+import progressbar
 import sys
 
 # The following bit of code sets the correct path to Python libraries
@@ -27,9 +29,16 @@
   overrides_list = [pkg.GetSavedOverrides() for pkg in pkgstats]
   files = os.listdir(options.extractdir)
   error_tags = []
+  no_files = len(files)
+  bar = progressbar.ProgressBar()
+  bar.maxval = no_files
+  bar.start()
+  counter = itertools.count()
   for file_name in files:
     full_path = os.path.join(options.extractdir, file_name)
     error_tags.extend(checkpkg.ErrorTagsFromFile(full_path))
+    bar.update(counter.next())
+  bar.finish()
   override_list = reduce(operator.add, overrides_list)
   (tags_after_overrides,
    unapplied_overrides) = overrides.ApplyOverrides(error_tags, override_list)

Modified: csw/mgar/gar/v2/lib/python/checkpkg.py
===================================================================
--- csw/mgar/gar/v2/lib/python/checkpkg.py	2010-07-06 08:23:36 UTC (rev 10453)
+++ csw/mgar/gar/v2/lib/python/checkpkg.py	2010-07-06 08:25:10 UTC (rev 10454)
@@ -562,6 +562,8 @@
   return expanded_list
 
 def ExpandSymlink(symlink, target, input_path):
+  # A lot of time is spent here, e.g. 13841985 calls, 206s.
+  # TODO: Optimize this.  Make it a class and add a cache?
   symlink_re = re.compile(r"%s(/|$)" % symlink)
   if re.search(symlink_re, input_path):
     result = input_path.replace(symlink, target)
@@ -707,9 +709,13 @@
     return errors
 
   def GetOptimizedAllStats(self, stats_obj_list):
+    logging.info("Loading package statistics.")
     pkgs_data = []
     counter = itertools.count()
     length = len(stats_obj_list)
+    bar = progressbar.ProgressBar()
+    bar.maxval = length
+    bar.start()
     for stats_obj in stats_obj_list:
       # pkg_data = {}
       # This bit is tightly tied to the data structures returned by
@@ -718,11 +724,13 @@
       # Python strings are already implementing the flyweight pattern. What's
       # left is lists and dictionaries.
       i = counter.next()
-      logging.debug("Loading stats for %s (%s/%s)",
-                    stats_obj.md5sum, i, length)
+      # logging.debug("Loading stats for %s (%s/%s)",
+      #               stats_obj.md5sum, i, length)
       raw_pkg_data = stats_obj.GetAllStats()
       pkg_data = raw_pkg_data
       pkgs_data.append(pkg_data)
+      bar.update(i)
+    bar.finish()
     return pkgs_data
 
   def Run(self):
@@ -869,6 +877,11 @@
     logging.debug("All package statistics loaded.")
     messenger = CheckpkgMessenger()
     # Individual checks
+    count = itertools.count()
+    bar = progressbar.ProgressBar()
+    bar.maxval = len(pkgs_data) * len(self.individual_checks)
+    logging.info("Running checks.")
+    bar.start()
     for pkg_data in pkgs_data:
       pkgname = pkg_data["basic_stats"]["pkgname"]
       check_interface = IndividualCheckInterface(pkgname, pkgmap)
@@ -878,7 +891,10 @@
         function(pkg_data, check_interface, logger=logger, messenger=messenger)
         if check_interface.errors:
           errors[pkgname] = check_interface.errors
+        bar.update(count.next())
+    bar.finish()
     # Set checks
+    logging.info("Running set checks.")
     for function in self.set_checks:
       logger = logging.getLogger(function.__name__)
       check_interface = SetCheckInterface(pkgmap)
@@ -970,13 +986,12 @@
   def GetDbObject(self):
     if not self.db_pkg_stats:
       md5_sum = self.GetMd5sum()
-      logging.debug("GetDbObject() md5_sum=%s", md5_sum)
       res = m.Srv4FileStats.select(m.Srv4FileStats.q.md5_sum==md5_sum)
       if not res.count():
-        logging.debug("%s are not in the db", md5_sum)
+        # TODO: Change this bit to throw an exception if the object is not
+        # found.
         return None
       else:
-        logging.debug("%s are in the db", md5_sum)
         self.db_pkg_stats = res.getOne()
     return self.db_pkg_stats
 
@@ -1192,7 +1207,6 @@
     return pkg_stats
 
   def GetAllStats(self):
-    logging.debug("GetAllStats()")
     if not self.all_stats and self.StatsExist():
       self.all_stats = self.ReadSavedStats()
     elif not self.all_stats:

Modified: csw/mgar/gar/v2/lib/python/opencsw.py
===================================================================
--- csw/mgar/gar/v2/lib/python/opencsw.py	2010-07-06 08:23:36 UTC (rev 10453)
+++ csw/mgar/gar/v2/lib/python/opencsw.py	2010-07-06 08:25:10 UTC (rev 10454)
@@ -894,7 +894,6 @@
                              catalogname)
     # This might potentially cause a file descriptor leak, but I'm not going to
     # worry about that at this stage.
-    logging.debug("Trying to open %s", repr(file_path))
     if os.path.isfile(file_path):
       return open(file_path, "r")
     else:
@@ -980,7 +979,6 @@
       if strip:
         strip_re = re.compile(r"^%s" % strip)
         fields = [re.sub(strip_re, "", x) for x in fields]
-      # logging.debug(fields)
       line_to_add = None
       installed_path = None
       prototype_class = None
@@ -1173,7 +1171,7 @@
         parsed = self._ParseCatalogLine(line)
         catalog_data.append(parsed)
       except CatalogLineParseError, e:
-        logging.error("Could not parse %s, %s", repr(line), e)
+        logging.debug("Could not parse %s, %s", repr(line), e)
     return catalog_data
 
   def GetCatalogData(self):


This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.


More information about the devel mailing list