[csw-devel] SF.net SVN: gar:[20796] csw/mgar/gar/v2/lib

wahwah at users.sourceforge.net wahwah at users.sourceforge.net
Thu Apr 18 17:17:17 CEST 2013


Revision: 20796
          http://gar.svn.sourceforge.net/gar/?rev=20796&view=rev
Author:   wahwah
Date:     2013-04-18 15:17:17 +0000 (Thu, 18 Apr 2013)
Log Message:
-----------
pkgdb: Various cleanup

Collected numerous changes that have no meritoric significance.

pkgdb: Sort imports in the rest.py file

Modified Paths:
--------------
    csw/mgar/gar/v2/lib/python/checkpkg_lib.py
    csw/mgar/gar/v2/lib/python/inspective_package.py
    csw/mgar/gar/v2/lib/python/package.py
    csw/mgar/gar/v2/lib/python/package_stats.py
    csw/mgar/gar/v2/lib/python/pkgdb.py
    csw/mgar/gar/v2/lib/python/rest.py
    csw/mgar/gar/v2/lib/python/system_pkgmap.py
    csw/mgar/gar/v2/lib/web/releases_web.py

Modified: csw/mgar/gar/v2/lib/python/checkpkg_lib.py
===================================================================
--- csw/mgar/gar/v2/lib/python/checkpkg_lib.py	2013-04-18 14:18:02 UTC (rev 20795)
+++ csw/mgar/gar/v2/lib/python/checkpkg_lib.py	2013-04-18 15:17:17 UTC (rev 20796)
@@ -1092,7 +1092,7 @@
     logging.debug("AddSrv4ToCatalog(%s, %s, %s, %s, %s)",
         sqo_srv4, osrel, arch, catrel, who)
     if not who:
-    	who = 'unknown'
+      who = 'unknown'
     # There are only i386 and sparc catalogs.
     if arch != 'i386' and arch != 'sparc':
       raise CatalogDatabaseError("Wrong architecture: %s" % arch)

Modified: csw/mgar/gar/v2/lib/python/inspective_package.py
===================================================================
--- csw/mgar/gar/v2/lib/python/inspective_package.py	2013-04-18 14:18:02 UTC (rev 20795)
+++ csw/mgar/gar/v2/lib/python/inspective_package.py	2013-04-18 15:17:17 UTC (rev 20796)
@@ -56,8 +56,8 @@
         "will probably finish successfully when do you that."
         % full_path)
     if "/opt/csw/share" in full_path:
-    	file_info["mime_type"] = "application/octet-stream; fallback"
-    	logging.error(msg)
+      file_info["mime_type"] = "application/octet-stream; fallback"
+      logging.error(msg)
     else:
       raise package.PackageError(msg)
   if sharedlib_utils.IsBinary(file_info, check_consistency=False):
@@ -214,7 +214,7 @@
       retcode, stdout, stderr = shell.ShellCommand(args)
       if retcode:
         logging.error("%s returned an error: %s", args, stderr)
-      	# Should it just skip over an error?
+        # Should it just skip over an error?
         continue
       nm_out = stdout.splitlines()
 
@@ -300,7 +300,6 @@
 
       cur_section = None
       for line in elfdump_out:
-
         try:
           elf_info, cur_section = self._ParseElfdumpLine(line, cur_section)
         except package.StdoutSyntaxError as e:
@@ -510,7 +509,7 @@
         elfdump_data = m.groupdict()
 
     if not m:
-      raise package.StdoutSyntaxError("Could not parse %s" % (repr(line)))
+      raise package.StdoutSyntaxError("Could not parse %r" % line)
 
     return elfdump_data, section
 

Modified: csw/mgar/gar/v2/lib/python/package.py
===================================================================
--- csw/mgar/gar/v2/lib/python/package.py	2013-04-18 14:18:02 UTC (rev 20795)
+++ csw/mgar/gar/v2/lib/python/package.py	2013-04-18 15:17:17 UTC (rev 20796)
@@ -35,19 +35,23 @@
 proxy=
 """
 
+
 class Error(Exception):
-  pass
+  """A generic error."""
 
 
 class SystemUtilityError(Error):
   """A problem occurred while running system utility, e.g. ldd."""
 
+
 class PackageError(Error):
-  pass
+  """A problem with the package itself that causes problems with examining it."""
 
+
 class StdoutSyntaxError(Error):
-  pass
+  """Problem with data returned by a process."""
 
+
 class CswSrv4File(shell.ShellMixin, object):
   """Represents a package in the srv4 format (pkg)."""
 

Modified: csw/mgar/gar/v2/lib/python/package_stats.py
===================================================================
--- csw/mgar/gar/v2/lib/python/package_stats.py	2013-04-18 14:18:02 UTC (rev 20795)
+++ csw/mgar/gar/v2/lib/python/package_stats.py	2013-04-18 15:17:17 UTC (rev 20796)
@@ -55,6 +55,8 @@
 
 class PackageStatsMixin(object):
   """Collects stats about a package and saves them.
+
+  Takes care of processing data from a SVR4 package and returing them as data structures.
   """
 
   def __init__(self, srv4_pkg, stats_basedir=None, md5sum=None, debug=False):
@@ -223,7 +225,7 @@
         "binaries_elf_info": dir_pkg.GetBinaryElfInfo(),
     }
     self.SaveStats(pkg_stats)
-    logging.debug("Statistics of %s have been collected and saved in the db.",
+    logging.debug("_CollectStats(): Stats of %s have been collected and saved in the db.",
                   repr(dir_pkg.pkgname))
     return pkg_stats
 

Modified: csw/mgar/gar/v2/lib/python/pkgdb.py
===================================================================
--- csw/mgar/gar/v2/lib/python/pkgdb.py	2013-04-18 14:18:02 UTC (rev 20795)
+++ csw/mgar/gar/v2/lib/python/pkgdb.py	2013-04-18 15:17:17 UTC (rev 20796)
@@ -202,6 +202,7 @@
     if catrel not in CATALOGS_ALLOWED_TO_BE_IMPORTED:
       raise UsageError("Catalogs that can be imported: %s"
                        % CATALOGS_ALLOWED_TO_BE_IMPORTED)
+
     catalog_dir = os.path.dirname(catalog_file)
     # The plan:
     # - read in the catalog file, and build a md5-filename correspondence
@@ -214,16 +215,20 @@
     for catalog_entry in catalog_data:
       cat_entry_by_md5[catalog_entry["md5sum"]] = catalog_entry
       cat_entry_by_basename[catalog_entry["file_basename"]] = catalog_entry
+
     # - import all srv4 files that were not in the database so far
     entries_to_import = []
+
     logging.debug("Checking which srv4 files are already in the db.")
     for md5 in cat_entry_by_md5:
       try:
         sqo_list = m.Srv4FileStats.selectBy(md5_sum=md5).getOne()
       except sqlobject.main.SQLObjectNotFound, e:
         entries_to_import.append(cat_entry_by_md5[md5])
+
     basenames = [x["file_basename"] for x in entries_to_import]
     file_list = []
+
     if entries_to_import:
       logging.info("Srv4 files to import:")
       for basename in sorted(basenames):
@@ -242,6 +247,7 @@
       for statdict in new_statdicts:
         new_statdicts_by_md5[statdict["basic_stats"]["md5_sum"]] = statdict
         package_stats.PackageStats.ImportPkg(statdict)
+
     # - sync the specific catalog
     #   - find the md5 sum list of the current catalog
     logging.debug("Retrieving current catalog assigments from the db.")
@@ -264,6 +270,7 @@
         # Since the srv4_in_cat object has lost its reference, there's no use
         # keeping it around.
         srv4_in_cat.destroySelf()
+
     disk_md5s = set(cat_entry_by_md5)
     db_md5s = set(db_srv4s_in_cat_by_md5)
     #   - match the md5 sum lists between db and disk
@@ -278,12 +285,14 @@
         logging.info(
             " - %s",
             db_srv4s_in_cat_by_md5[md5].srv4file.basename)
+
     if md5_sums_to_add:
       logging.info("To add to from %s %s %s:", osrel, arch, catrel)
       for md5 in md5_sums_to_add:
         logging.info(
             " + %s",
             cat_entry_by_md5[md5]["file_basename"])
+
     user = getpass.getuser()
     # Remove
     # We could use checkpkg_lib.Catalog.RemoveSrv4(), but it would redo
@@ -292,6 +301,7 @@
       logging.info("Removing assignments from the catalog.")
       for md5 in md5_sums_to_remove:
         db_srv4s_in_cat_by_md5[md5].destroySelf()
+
     # Add
     if md5_sums_to_add:
       logging.info("Adding srv4 files to the %s %s %s catalog.",
@@ -497,14 +507,16 @@
     c = checkpkg_lib.Catalog()
     md5_sums = args[3:]
     for md5_sum in md5_sums:
-      logging.debug("Adding %s to the catalog", md5_sum)
       try:
         sqo_srv4 = m.Srv4FileStats.select(
             m.Srv4FileStats.q.md5_sum==md5_sum).getOne()
+        logging.debug('Adding %s to the catalog', md5_sum)
         c.AddSrv4ToCatalog(sqo_srv4, osrel, arch, catrel, who=user)
-      except sqlobject.main.SQLObjectNotFound, e:
-        logging.warning("Srv4 file %s was not found in the database.",
-                        md5_sum)
+      except sqlobject.main.SQLObjectNotFound as e:
+        logging.warning('Srv4 file %r, osrel %r, arch %r or catrel %r '
+                        'was not found in the database, '
+                        'and the package was not added: %s',
+                        md5_sum, osrel, arch, catrel, e)
   elif command == 'del-from-cat':
     if len(args) < 4:
       raise UsageError("Not enough arguments, see usage.")

Modified: csw/mgar/gar/v2/lib/python/rest.py
===================================================================
--- csw/mgar/gar/v2/lib/python/rest.py	2013-04-18 14:18:02 UTC (rev 20795)
+++ csw/mgar/gar/v2/lib/python/rest.py	2013-04-18 15:17:17 UTC (rev 20796)
@@ -1,12 +1,12 @@
 #!/usr/bin/env python2.6
 
-import os
 from StringIO import StringIO
+import anydbm
 import cjson
-import anydbm
 import logging
+import os
+import pycurl
 import urllib2
-import pycurl
 
 DEFAULT_URL = "http://buildfarm.opencsw.org"
 RELEASES_APP = "/releases"

Modified: csw/mgar/gar/v2/lib/python/system_pkgmap.py
===================================================================
--- csw/mgar/gar/v2/lib/python/system_pkgmap.py	2013-04-18 14:18:02 UTC (rev 20795)
+++ csw/mgar/gar/v2/lib/python/system_pkgmap.py	2013-04-18 15:17:17 UTC (rev 20796)
@@ -552,20 +552,13 @@
         try:
           sqo_srv4 = self._GetFakeSrv4(pkgname, osrel, arch)
         except sqlobject.main.SQLObjectNotFound, e:
-          print d
+          logging.fatal(d)
           raise
         if sqo_srv4 not in cleaned_pkgs:
           sqo_srv4.RemoveAllCswFiles()
           cleaned_pkgs.add(sqo_srv4)
         sqo_pkginst = self._GetPkginst(pkgname)
         f_path, f_basename = os.path.split(d["path"])
-        # This is really slow (one run ~1h), but works.
-        # To speed it up, raw SQL + cursor.executemany() could be used, but
-        # there's a incompatibility between MySQL and sqlite drivers:
-        # MySQL:  INSERT ... VALUES (%s, %s, %s);
-        # sqlite: INSERT ... VALUES (?, ?, ?);
-        # For now, using the sqlobject ORM which is slow, but at least
-        # handles compatibility issues.
         csw_file = m.CswFile(pkginst=sqo_pkginst,
             line=d["line"], path=f_path, basename=f_basename,
             srv4_file=sqo_srv4)

Modified: csw/mgar/gar/v2/lib/web/releases_web.py
===================================================================
--- csw/mgar/gar/v2/lib/web/releases_web.py	2013-04-18 14:18:02 UTC (rev 20795)
+++ csw/mgar/gar/v2/lib/web/releases_web.py	2013-04-18 15:17:17 UTC (rev 20796)
@@ -21,6 +21,7 @@
 import hashlib
 import logging
 
+
 urls = (
   r'/', 'Index',
   r'/srv4/', 'Srv4List',
@@ -35,7 +36,12 @@
 
 OPENCSW_ROOT = "/home/mirror/opencsw-official"
 ALLPKGS_DIR = os.path.join(OPENCSW_ROOT, "allpkgs")
-CAN_UPLOAD_TO_CATALOGS = frozenset(["unstable", "kiel", "bratislava", "beanie"])
+CAN_UPLOAD_TO_CATALOGS = frozenset([
+    "unstable",
+    "kiel",
+    "bratislava",
+    "beanie",
+])
 
 def ConnectToDatabase():
   configuration.SetUpSqlobjectConnection()
@@ -50,14 +56,10 @@
     messages = []
     configuration.SetUpSqlobjectConnection()
     x = web.input(srv4_file={})
-    # x['srv4_file'].filename
-    # x['srv4_file'].value
-    # x['srv4_file'].file.read()
     web.header(
         'Content-type',
         'application/x-vnd.opencsw.pkg;type=upload-results')
     hash = hashlib.md5()
-    # hash.update(x['srv4_file'].file.read())
     hash.update(x['srv4_file'].value)
     data_md5_sum = hash.hexdigest()
     declared_md5_sum = x['md5_sum']

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.



More information about the devel mailing list