[csw-devel] SF.net SVN: gar:[10454] csw/mgar/gar/v2
wahwah at users.sourceforge.net
wahwah at users.sourceforge.net
Tue Jul 6 10:25:10 CEST 2010
Revision: 10454
http://gar.svn.sourceforge.net/gar/?rev=10454&view=rev
Author: wahwah
Date: 2010-07-06 08:25:10 +0000 (Tue, 06 Jul 2010)
Log Message:
-----------
mGAR v2: checkpkg, terminal output tweaks: removed some debug messages and added progress bars for operations that take a long time during the whole catalog run.
Modified Paths:
--------------
csw/mgar/gar/v2/bin/analyze_module_results.py
csw/mgar/gar/v2/lib/python/checkpkg.py
csw/mgar/gar/v2/lib/python/opencsw.py
Modified: csw/mgar/gar/v2/bin/analyze_module_results.py
===================================================================
--- csw/mgar/gar/v2/bin/analyze_module_results.py 2010-07-06 08:23:36 UTC (rev 10453)
+++ csw/mgar/gar/v2/bin/analyze_module_results.py 2010-07-06 08:25:10 UTC (rev 10454)
@@ -1,9 +1,11 @@
#!/opt/csw/bin/python2.6
# $Id$
+import itertools
import operator
import optparse
import os
+import progressbar
import sys
# The following bit of code sets the correct path to Python libraries
@@ -27,9 +29,16 @@
overrides_list = [pkg.GetSavedOverrides() for pkg in pkgstats]
files = os.listdir(options.extractdir)
error_tags = []
+ no_files = len(files)
+ bar = progressbar.ProgressBar()
+ bar.maxval = no_files
+ bar.start()
+ counter = itertools.count()
for file_name in files:
full_path = os.path.join(options.extractdir, file_name)
error_tags.extend(checkpkg.ErrorTagsFromFile(full_path))
+ bar.update(counter.next())
+ bar.finish()
override_list = reduce(operator.add, overrides_list)
(tags_after_overrides,
unapplied_overrides) = overrides.ApplyOverrides(error_tags, override_list)
Modified: csw/mgar/gar/v2/lib/python/checkpkg.py
===================================================================
--- csw/mgar/gar/v2/lib/python/checkpkg.py 2010-07-06 08:23:36 UTC (rev 10453)
+++ csw/mgar/gar/v2/lib/python/checkpkg.py 2010-07-06 08:25:10 UTC (rev 10454)
@@ -562,6 +562,8 @@
return expanded_list
def ExpandSymlink(symlink, target, input_path):
+ # A lot of time is spent here, e.g. 13841985 calls, 206s.
+ # TODO: Optimize this. Make it a class and add a cache?
symlink_re = re.compile(r"%s(/|$)" % symlink)
if re.search(symlink_re, input_path):
result = input_path.replace(symlink, target)
@@ -707,9 +709,13 @@
return errors
def GetOptimizedAllStats(self, stats_obj_list):
+ logging.info("Loading package statistics.")
pkgs_data = []
counter = itertools.count()
length = len(stats_obj_list)
+ bar = progressbar.ProgressBar()
+ bar.maxval = length
+ bar.start()
for stats_obj in stats_obj_list:
# pkg_data = {}
# This bit is tightly tied to the data structures returned by
@@ -718,11 +724,13 @@
# Python strings are already implementing the flyweight pattern. What's
# left is lists and dictionaries.
i = counter.next()
- logging.debug("Loading stats for %s (%s/%s)",
- stats_obj.md5sum, i, length)
+ # logging.debug("Loading stats for %s (%s/%s)",
+ # stats_obj.md5sum, i, length)
raw_pkg_data = stats_obj.GetAllStats()
pkg_data = raw_pkg_data
pkgs_data.append(pkg_data)
+ bar.update(i)
+ bar.finish()
return pkgs_data
def Run(self):
@@ -869,6 +877,11 @@
logging.debug("All package statistics loaded.")
messenger = CheckpkgMessenger()
# Individual checks
+ count = itertools.count()
+ bar = progressbar.ProgressBar()
+ bar.maxval = len(pkgs_data) * len(self.individual_checks)
+ logging.info("Running checks.")
+ bar.start()
for pkg_data in pkgs_data:
pkgname = pkg_data["basic_stats"]["pkgname"]
check_interface = IndividualCheckInterface(pkgname, pkgmap)
@@ -878,7 +891,10 @@
function(pkg_data, check_interface, logger=logger, messenger=messenger)
if check_interface.errors:
errors[pkgname] = check_interface.errors
+ bar.update(count.next())
+ bar.finish()
# Set checks
+ logging.info("Running set checks.")
for function in self.set_checks:
logger = logging.getLogger(function.__name__)
check_interface = SetCheckInterface(pkgmap)
@@ -970,13 +986,12 @@
def GetDbObject(self):
if not self.db_pkg_stats:
md5_sum = self.GetMd5sum()
- logging.debug("GetDbObject() md5_sum=%s", md5_sum)
res = m.Srv4FileStats.select(m.Srv4FileStats.q.md5_sum==md5_sum)
if not res.count():
- logging.debug("%s are not in the db", md5_sum)
+ # TODO: Change this bit to throw an exception if the object is not
+ # found.
return None
else:
- logging.debug("%s are in the db", md5_sum)
self.db_pkg_stats = res.getOne()
return self.db_pkg_stats
@@ -1192,7 +1207,6 @@
return pkg_stats
def GetAllStats(self):
- logging.debug("GetAllStats()")
if not self.all_stats and self.StatsExist():
self.all_stats = self.ReadSavedStats()
elif not self.all_stats:
Modified: csw/mgar/gar/v2/lib/python/opencsw.py
===================================================================
--- csw/mgar/gar/v2/lib/python/opencsw.py 2010-07-06 08:23:36 UTC (rev 10453)
+++ csw/mgar/gar/v2/lib/python/opencsw.py 2010-07-06 08:25:10 UTC (rev 10454)
@@ -894,7 +894,6 @@
catalogname)
# This might potentially cause a file descriptor leak, but I'm not going to
# worry about that at this stage.
- logging.debug("Trying to open %s", repr(file_path))
if os.path.isfile(file_path):
return open(file_path, "r")
else:
@@ -980,7 +979,6 @@
if strip:
strip_re = re.compile(r"^%s" % strip)
fields = [re.sub(strip_re, "", x) for x in fields]
- # logging.debug(fields)
line_to_add = None
installed_path = None
prototype_class = None
@@ -1173,7 +1171,7 @@
parsed = self._ParseCatalogLine(line)
catalog_data.append(parsed)
except CatalogLineParseError, e:
- logging.error("Could not parse %s, %s", repr(line), e)
+ logging.debug("Could not parse %s, %s", repr(line), e)
return catalog_data
def GetCatalogData(self):
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
More information about the devel
mailing list