[csw-devel] SF.net SVN: gar:[13970] csw/mgar/gar/v2

wahwah at users.sourceforge.net wahwah at users.sourceforge.net
Mon Mar 28 03:10:06 CEST 2011


Revision: 13970
          http://gar.svn.sourceforge.net/gar/?rev=13970&view=rev
Author:   wahwah
Date:     2011-03-28 01:10:06 +0000 (Mon, 28 Mar 2011)

Log Message:
-----------
csw-upload-pkg: checkpkg integration

Before sending packages, csw-upload-pkg calls checkpkg (fork + exec) to
check packages against catalogs they are about to be inserted into.

It's a very rudimentary checkpkg integration.  It relies on operation on
client side, which is not the optimal place for checks.  However, it's
better than no checks.

Signed-off-by: Maciej Blizinski <maciej at opencsw.org>

Modified Paths:
--------------
    csw/mgar/gar/v2/lib/python/csw_upload_pkg.py
    csw/mgar/gar/v2/lib/python/csw_upload_pkg_test.py
    csw/mgar/gar/v2/tests/run_tests.py

Modified: csw/mgar/gar/v2/lib/python/csw_upload_pkg.py
===================================================================
--- csw/mgar/gar/v2/lib/python/csw_upload_pkg.py	2011-03-28 01:09:24 UTC (rev 13969)
+++ csw/mgar/gar/v2/lib/python/csw_upload_pkg.py	2011-03-28 01:10:06 UTC (rev 13970)
@@ -18,6 +18,7 @@
 import socket
 import rest
 import struct_util
+import subprocess
 
 
 BASE_URL = "http://buildfarm.opencsw.org"
@@ -72,14 +73,88 @@
     self._rest_client = rest.RestClient(self.rest_url)
 
   def Upload(self):
+    do_upload = True
+    planned_modifications = []
+    metadata_by_md5 = {}
     for filename in self.filenames:
-      parsed_basename = opencsw.ParsePackageFileName(
-          os.path.basename(filename))
-      if parsed_basename["vendortag"] != "CSW":
-        raise PackageCheckError(
-            "Package vendor tag is %s instead of CSW."
-            % parsed_basename["vendortag"])
-      self._UploadFile(filename)
+      md5_sum = self._GetFileMd5sum(filename)
+      file_in_allpkgs, file_metadata = self._GetSrv4FileMetadata(md5_sum)
+      if file_in_allpkgs:
+        logging.debug("File %s already uploaded.", filename)
+      else:
+        if do_upload:
+          logging.debug("Uploading %s.", filename)
+          self._PostFile(filename)
+          # Querying the database again, this time the data should be
+          # there
+          file_in_allpkgs, file_metadata = self._GetSrv4FileMetadata(md5_sum)
+      logging.debug("file_metadata %s", repr(file_metadata))
+      if not file_metadata:
+        logging.error(
+            "File metadata was not found in the database.  "
+            "This happens when the package you're trying to upload was never "
+            "unpacked and imported into the database.  "
+            "To fix the problem, run checkpkg against your package and try "
+            "importing again.")
+        raise DataError("file_metadata is empty: %s" % repr(file_metadata))
+      osrel = file_metadata['osrel']
+      arch = file_metadata['arch']
+      metadata_by_md5[md5_sum] = file_metadata
+      catalogs = self._MatchSrv4ToCatalogs(
+          filename, DEFAULT_CATREL, arch, osrel, md5_sum)
+      for unused_catrel, cat_arch, cat_osrel in catalogs:
+        planned_modifications.append(
+            (filename, md5_sum,
+             arch, osrel, cat_arch, cat_osrel))
+    # The plan: 
+    # - Create groups of files to be inserted into each of the catalogs
+    # - Invoke checkpkg to check every target catalog
+    checkpkg_sets = self._CheckpkgSets(planned_modifications)
+    bin_dir = os.path.dirname(__file__)
+    checkpkg_executable = os.path.join(bin_dir, "checkpkg")
+    checks_failed_for_catalogs = []
+    args_by_cat = {}
+    for arch, osrel in checkpkg_sets:
+      if "5.11" in osrel:
+        logging.debug("Skipping Solaris 11 checks")
+        continue
+      print ("Checking packages against catalog %s %s %s"
+             % (DEFAULT_CATREL, arch, osrel))
+      md5_sums = []
+      basenames = []
+      for filename, md5_sum in checkpkg_sets[(arch, osrel)]:
+        md5_sums.append(md5_sum)
+        basenames.append(os.path.basename(filename))
+      # Not using the checkpkg Python API.  The reason is that checkpkg
+      # requires the process calling its API to have an established
+      # MySQL connection, while csw-upload-pkg does not, and it's better
+      # if it stays that way.
+      args_by_cat[(arch, osrel)] = [
+          checkpkg_executable,
+          "--catalog-release", DEFAULT_CATREL,
+          "--os-release", osrel,
+          "--architecture", arch,
+      ] + md5_sums
+      ret = subprocess.call(args_by_cat[(arch, osrel)] + ["--quiet"])
+      if ret:
+        checks_failed_for_catalogs.append(
+            (arch, osrel, basenames)
+        )
+    if checks_failed_for_catalogs:
+      print "Checks failed for catalogs:"
+      for arch, osrel, basenames in checks_failed_for_catalogs:
+        print "  - %s %s" % (arch, osrel)
+        for basename in basenames:
+          print "    %s" % basename
+        print "To see errors, run:"
+        print " ", " ".join(args_by_cat[(arch, osrel)])
+      print ("Packages have not been submitted to the %s catalog."
+             % DEFAULT_CATREL)
+    else:
+      print "All checks successful. Proceeding."
+      for arch, osrel in checkpkg_sets:
+        for filename, md5_sum in checkpkg_sets[(arch, osrel)]:
+          self._InsertIntoCatalog(filename, arch, osrel, file_metadata)
 
   def Remove(self):
     for filename in self.filenames:
@@ -213,7 +288,7 @@
             catalogs.append(cat_key)
           else:
             logging.info(
-                "Not inserting %s %s package into %s containing a %s package",
+                "Not matching %s %s package with %s containing a %s package",
                 catalogname,
                 srv4_osrel, osrel, srv4_in_catalog["osrel"])
           logging.debug(
@@ -221,31 +296,6 @@
               catrel, arch, osrel, catalogname)
     return tuple(catalogs)
 
-  def _UploadFile(self, filename):
-    md5_sum = self._GetFileMd5sum(filename)
-    file_in_allpkgs, file_metadata = self._GetSrv4FileMetadata(md5_sum)
-    if file_in_allpkgs:
-      logging.debug("File %s already uploaded.", filename)
-    else:
-      logging.debug("Uploading %s.", filename)
-      self._PostFile(filename)
-    file_in_allpkgs, file_metadata = self._GetSrv4FileMetadata(md5_sum)
-    logging.debug("file_metadata %s", repr(file_metadata))
-    if not file_metadata:
-      logging.error(
-          "File metadata was not found in the database.  "
-          "This happens when the package you're trying to upload was never "
-          "unpacked and imported into the database.  "
-          "To fix the problem, run checkpkg against your package and try "
-          "importing again.")
-      raise DataError("file_metadata is empty: %s" % repr(file_metadata))
-    osrel = file_metadata['osrel']
-    arch = file_metadata['arch']
-    catalogs = self._MatchSrv4ToCatalogs(
-        filename, DEFAULT_CATREL, arch, osrel, md5_sum)
-    for unused_catrel, cat_arch, cat_osrel in catalogs:
-      self._InsertIntoCatalog(filename, cat_arch, cat_osrel, file_metadata)
-
   def _InsertIntoCatalog(self, filename, arch, osrel, file_metadata):
     logging.debug(
         "_InsertIntoCatalog(%s, %s, %s)",
@@ -360,7 +410,27 @@
     if http_code >= 400 and http_code <= 499:
       raise RestCommunicationError("%s - HTTP code: %s" % (url, http_code))
 
+  def _CheckpkgSets(self, planned_modifications):
+    """Groups packages according to catalogs.
+    
+    Used to determine groups of packages to check together, against
+    a specific catalog.
 
+    Args:
+      A list of tuples
+
+    Returns:
+      A dictionary of tuples, indexed by (arch, osrel) tuples.
+    """
+    by_catalog = {}
+    for fields in planned_modifications:
+      filename, md5_sum, pkg_arch, pkg_osrel, cat_arch, cat_osrel = fields
+      key = cat_arch, cat_osrel
+      by_catalog.setdefault(key, [])
+      by_catalog[key].append((filename, md5_sum))
+    return by_catalog
+
+
 if __name__ == '__main__':
   parser = optparse.OptionParser(USAGE)
   parser.add_option("-d", "--debug",

Modified: csw/mgar/gar/v2/lib/python/csw_upload_pkg_test.py
===================================================================
--- csw/mgar/gar/v2/lib/python/csw_upload_pkg_test.py	2011-03-28 01:09:24 UTC (rev 13969)
+++ csw/mgar/gar/v2/lib/python/csw_upload_pkg_test.py	2011-03-28 01:10:06 UTC (rev 13970)
@@ -51,6 +51,26 @@
     "size": 7617270,
     "version_string": "7.2,REV=2011.01.21",
 }
+TEST_PLANNED_MODIFICATIONS_1 = [
+ ('foo.pkg',
+  '58f564d11d6419592dcca3915bfabc55',
+  u'all',
+  u'SunOS5.9',
+  'sparc',
+  u'SunOS5.9'),
+ ('foo.pkg',
+  '58f564d11d6419592dcca3915bfabc55',
+  u'all',
+  u'SunOS5.9',
+  'sparc',
+  u'SunOS5.10'),
+ ('bar.pkg',
+  '84b409eb7c2faf87e22ee0423e55b888',
+  u'sparc',
+  u'SunOS5.9',
+  u'sparc',
+  u'SunOS5.9'),
+]
 
 
 class Srv4UploaderUnitTest(mox.MoxTestBase):
@@ -297,6 +317,16 @@
     self.mox.ResetAll()
     self.mox.UnsetStubs()
 
+  def test_CheckpkgSets(self):
+    su = csw_upload_pkg.Srv4Uploader(None, None)
+    expected = {
+        ('sparc', u'SunOS5.10'):
+          [('foo.pkg', '58f564d11d6419592dcca3915bfabc55')],
+        ('sparc', u'SunOS5.9'):
+          [('foo.pkg', '58f564d11d6419592dcca3915bfabc55'),
+           ('bar.pkg', '84b409eb7c2faf87e22ee0423e55b888')]}
+    self.assertEqual(expected, su._CheckpkgSets(TEST_PLANNED_MODIFICATIONS_1))
 
+
 if __name__ == '__main__':
   unittest.main()

Modified: csw/mgar/gar/v2/tests/run_tests.py
===================================================================
--- csw/mgar/gar/v2/tests/run_tests.py	2011-03-28 01:09:24 UTC (rev 13969)
+++ csw/mgar/gar/v2/tests/run_tests.py	2011-03-28 01:10:06 UTC (rev 13970)
@@ -11,10 +11,11 @@
 
 # To add more test files, create <name>.py file and add a corresponding line
 # here:
+from catalog_notifier_test   import *
 from catalog_test            import *
-from catalog_notifier_test   import *
 from checkpkg_lib_test       import *
 from checkpkg_test           import *
+from csw_upload_pkg_test     import *
 from dependency_checks_test  import *
 from inspective_package_test import *
 from ldd_emul_test           import *


This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.


More information about the devel mailing list