[csw-devel] SF.net SVN: gar:[13312] csw/mgar/gar/v2/lib/python
wahwah at users.sourceforge.net
wahwah at users.sourceforge.net
Mon Feb 14 21:31:43 CET 2011
Revision: 13312
http://gar.svn.sourceforge.net/gar/?rev=13312&view=rev
Author: wahwah
Date: 2011-02-14 20:31:43 +0000 (Mon, 14 Feb 2011)
Log Message:
-----------
integrate-catalogs: A new tool
integrate-catalogs allows to integrate one catalog into another, e.g. unstable
into testing.
It generates shell commands to execute, which can be reviewed and executed.
./integrate_catalogs.py --catrel-from unstable --catrel-to testing -o script.sh
vim script.sh
bash script.sh
To see usage information:
./integrate_catalogs.py --help
Modified Paths:
--------------
csw/mgar/gar/v2/lib/python/catalog.py
csw/mgar/gar/v2/lib/python/catalog_test.py
csw/mgar/gar/v2/lib/python/models.py
csw/mgar/gar/v2/lib/python/opencsw.py
csw/mgar/gar/v2/lib/python/opencsw_test.py
csw/mgar/gar/v2/lib/python/rest.py
Added Paths:
-----------
csw/mgar/gar/v2/lib/python/integrate_catalogs.py
Modified: csw/mgar/gar/v2/lib/python/catalog.py
===================================================================
--- csw/mgar/gar/v2/lib/python/catalog.py 2011-02-14 20:30:50 UTC (rev 13311)
+++ csw/mgar/gar/v2/lib/python/catalog.py 2011-02-14 20:31:43 UTC (rev 13312)
@@ -173,7 +173,11 @@
class CatalogComparator(object):
def GetCatalogDiff(self, cat_a, cat_b):
- """Returns a difference between two catalogs."""
+ """Returns a difference between two catalogs.
+
+ Catalogs need to be represented either as a OpencswCatalog() object, or as
+ a dictionary, as returned by OpencswCatalog.GetDataByCatalogname().
+ """
if type(cat_a) == dict:
bc_a = cat_a
else:
Modified: csw/mgar/gar/v2/lib/python/catalog_test.py
===================================================================
--- csw/mgar/gar/v2/lib/python/catalog_test.py 2011-02-14 20:30:50 UTC (rev 13311)
+++ csw/mgar/gar/v2/lib/python/catalog_test.py 2011-02-14 20:31:43 UTC (rev 13312)
@@ -25,21 +25,31 @@
'145351cf6186fdcadcd169b66387f72f 214091 '
'CSWcommon|CSWlibevent none none\n')
+# Package as returned by the catalog file parser
PKG_STRUCT_1 = {
+ 'catalogname': 'syslog_ng',
'category': 'none',
+ 'deps': (
+ 'CSWcommon',
+ 'CSWcswclassutils',
+ 'CSWeventlog',
+ 'CSWgcc4corert',
+ 'CSWggettextrt',
+ 'CSWglib2',
+ 'CSWosslrt',
+ 'CSWpcrert',
+ 'CSWtcpwrap',
+ 'CSWzlib',
+ ),
+ 'file_basename': 'syslog_ng-3.0.4,REV=2009.08.30-SunOS5.8-i386-CSW.pkg.gz',
'i_deps': (),
+ 'md5sum': 'cfe40c06e994f6e8d3b191396d0365cb',
'pkgname': 'CSWsyslogng',
- 'md5sum': 'cfe40c06e994f6e8d3b191396d0365cb',
+ 'size': '137550',
'version': '3.0.4,REV=2009.08.30',
- 'deps': ('CSWgcc4corert', 'CSWeventlog', 'CSWosslrt', 'CSWzlib',
- 'CSWpcrert', 'CSWggettextrt', 'CSWglib2', 'CSWtcpwrap',
- 'CSWcswclassutils', 'CSWcommon'),
- 'file_basename': 'syslog_ng-3.0.4,REV=2009.08.30-SunOS5.8-i386-CSW.pkg.gz',
- 'size': '137550',
- 'catalogname': 'syslog_ng'}
+}
-
class OpencswCatalogUnitTest(unittest.TestCase):
def test_ParseCatalogLine_1(self):
Added: csw/mgar/gar/v2/lib/python/integrate_catalogs.py
===================================================================
--- csw/mgar/gar/v2/lib/python/integrate_catalogs.py (rev 0)
+++ csw/mgar/gar/v2/lib/python/integrate_catalogs.py 2011-02-14 20:31:43 UTC (rev 13312)
@@ -0,0 +1,168 @@
+#!/usr/bin/env python2.6
+
+"""Allows to integrate catalogs, e.g. unstable into testing.
+
+The script generated shell commands that perform the catalog integration. It
+does not run them, because they need to be reviewed by a human before they can
+be executed.
+
+The script does not understand package versions. It only displays commands
+necessary to bring one catalog to the state of another catalog.
+"""
+
+from Cheetah import Template
+import cPickle
+import catalog
+import common_constants
+import logging
+import opencsw
+import optparse
+import pprint
+import rest
+import sys
+import urllib2
+
+
+CATALOG_MOD_TMPL = """#!/bin/bash
+# Catalog modification (not integration yet): $catrel_from -> $catrel_to
+# Generated by $prog
+
+set -x
+
+PKGDB=bin/pkgdb
+
+#for catalogname in $sorted($diffs_by_catalogname):
+#if "new_pkgs" in $diffs_by_catalogname[$catalogname]:
+#for arch, osrel, new_pkg in $diffs_by_catalogname[$catalogname]["new_pkgs"]:
+# adding $new_pkg["basename"]
+\${PKGDB} add-to-cat $osrel $arch $catrel_to $new_pkg["md5_sum"]
+#end for
+#end if
+#if "removed_pkgs" in $diffs_by_catalogname[$catalogname]:
+#for arch, osrel, rem_pkg in $diffs_by_catalogname[$catalogname]["removed_pkgs"]:
+# removing $rem_pkg["basename"]
+\${PKGDB} del-from-cat $osrel $arch $catrel_to $rem_pkg["md5_sum"]
+#end for
+#end if
+#if "updated_pkgs" in $diffs_by_catalogname[$catalogname]:
+#for arch, osrel, up_pkg_pair in $diffs_by_catalogname[$catalogname]["updated_pkgs"]:
+#if $up_pkg_pair["direction"] == "downgrade":
+# WARNING: DOWNGRADE
+#end if
+# $catalogname $up_pkg_pair["direction"] from $up_pkg_pair["from"]["version"] to $up_pkg_pair["to"]["version"]
+\${PKGDB} del-from-cat $osrel $arch $catrel_to $up_pkg_pair["from"]["md5_sum"]
+\${PKGDB} add-to-cat $osrel $arch $catrel_to $up_pkg_pair["to"]["md5_sum"]
+#end for
+#end if
+
+#end for
+"""
+
+
+class Error(Exception):
+ """Generic error."""
+
+class UsageError(Error):
+ """Wrong usage."""
+
+
+def IndexByCatalogname(catalog):
+ return dict((x["catalogname"], x) for x in catalog)
+
+
+def GetDiffsByCatalogname(catrel_from, catrel_to, include_downgrades):
+ rest_client = rest.RestClient()
+ diffs_by_catalogname = {}
+ for arch in common_constants.PHYSICAL_ARCHITECTURES:
+ logging.debug("Architecture: %s", arch)
+ for osrel in common_constants.OS_RELS:
+ logging.debug("OS release: %s", osrel)
+ cat_from = rest_client.GetCatalog(catrel_from, arch, osrel)
+ cat_to = rest_client.GetCatalog(catrel_to, arch, osrel)
+ # Should use catalog comparator, but the data format is different
+ if cat_from is None:
+ cat_from = []
+ if cat_to is None:
+ cat_to = []
+ cat_from_by_c = IndexByCatalogname(cat_from)
+ cat_to_by_c = IndexByCatalogname(cat_to)
+ comparator = catalog.CatalogComparator()
+ new_pkgs, removed_pkgs, updated_pkgs = comparator.GetCatalogDiff(
+ cat_to_by_c, cat_from_by_c)
+ # By passing the catalogs (as arguments) in reverse order, we get
+ # packages to be updated in new_pkgs, and so forth.
+ for pkg in new_pkgs:
+ catalogname_d = diffs_by_catalogname.setdefault(pkg["catalogname"], {})
+ catalogname_d.setdefault("new_pkgs", []).append((arch, osrel, pkg))
+ for pkg in removed_pkgs:
+ catalogname_d = diffs_by_catalogname.setdefault(pkg["catalogname"], {})
+ catalogname_d.setdefault("removed_pkgs", []).append((arch, osrel, pkg))
+ for pkg_pair in updated_pkgs:
+ # Upgrade or downgrade?
+ cmp_result = opencsw.CompareVersions(
+ pkg_pair["from"]["version"],
+ pkg_pair["to"]["version"])
+ if cmp_result < 0:
+ direction = "upgrade"
+ else:
+ direction = "downgrade"
+ pkg_pair["direction"] = direction
+ pkg = pkg_pair["from"]
+ if direction == "upgrade" or include_downgrades:
+ catalogname_d = diffs_by_catalogname.setdefault(pkg["catalogname"], {})
+ catalogname_d.setdefault("updated_pkgs", []).append((arch, osrel, pkg_pair))
+ return diffs_by_catalogname
+
+
+def main():
+ parser = optparse.OptionParser()
+ parser.add_option("--catrel-from",
+ dest="catrel_from",
+ default="unstable",
+ help="Catalog release to integrate from, e.g. 'unstable'.")
+ parser.add_option("--catrel-to",
+ dest="catrel_to",
+ default="testing",
+ help="Catalog release to integrate to, e.g. 'testing'.")
+ parser.add_option("--from-pickle", dest="from_pickle",
+ help=("If specified, loads data from a pickle file instead of polling "
+ "the database."))
+ parser.add_option("--save-pickle", dest="save_pickle",
+ help="If specified, saves pickled data to a file.")
+ parser.add_option("-o", "--output-file", dest="output_file",
+ help="Filename to save output to.")
+ parser.add_option("--no-include-downgrades", dest="include_downgrades",
+ default=True, action="store_false",
+ help="Skip package downgrades.")
+ options, args = parser.parse_args()
+ logging.basicConfig(level=logging.DEBUG)
+ if not options.output_file:
+ raise UsageError("Please specify the output file. See --help.")
+ catrel_from = options.catrel_from
+ catrel_to = options.catrel_to
+ if options.from_pickle:
+ with open("tmp.pickle", "rb") as fd:
+ diffs_by_catalogname = cPickle.load(fd)
+ else:
+ diffs_by_catalogname = GetDiffsByCatalogname(
+ catrel_from, catrel_to, options.include_downgrades)
+ namespace = {
+ "diffs_by_catalogname": diffs_by_catalogname,
+ "catrel_to": catrel_to,
+ "catrel_from": catrel_from,
+ "prog": sys.argv[0],
+ }
+ if options.save_pickle:
+ with open(options.save_pickle, "wb") as fd:
+ cPickle.dump(diffs_by_catalogname, fd)
+ t = Template.Template(CATALOG_MOD_TMPL, searchList=[namespace])
+ if options.output_file:
+ logging.info("Saving output to %s", options.output_file)
+ with open(options.output_file, "wb") as fd:
+ fd.write(unicode(t))
+ else:
+ sys.stdout.write(unicode(t))
+
+
+if __name__ == '__main__':
+ main()
Property changes on: csw/mgar/gar/v2/lib/python/integrate_catalogs.py
___________________________________________________________________
Added: svn:executable
+ *
Modified: csw/mgar/gar/v2/lib/python/models.py
===================================================================
--- csw/mgar/gar/v2/lib/python/models.py 2011-02-14 20:30:50 UTC (rev 13311)
+++ csw/mgar/gar/v2/lib/python/models.py 2011-02-14 20:31:43 UTC (rev 13312)
@@ -210,19 +210,23 @@
def GetRestRepr(self):
mimetype = "application/x-vnd.opencsw.pkg;type=srv4-detail"
data = {
+ 'arch': self.arch.name,
+ 'basename': self.basename,
+ # For compatibility with the catalog parser from catalog.py
+ 'file_basename': self.basename,
'catalogname': self.catalogname,
- 'basename': self.basename,
- 'md5_sum': self.md5_sum,
- 'size': self.size,
+ 'filename_arch': self.filename_arch.name,
'maintainer_email': self.maintainer.email,
'maintainer_full_name': self.maintainer.full_name,
- 'version_string': self.version_string,
- 'arch': self.arch.name,
- 'pkgname': self.pkginst.pkgname,
+ 'md5_sum': self.md5_sum,
'mtime': unicode(self.mtime),
'osrel': self.os_rel.short_name,
+ 'pkgname': self.pkginst.pkgname,
'rev': self.rev,
- 'filename_arch': self.filename_arch.name,
+ 'size': self.size,
+ 'version_string': self.version_string,
+ # For compatibility with the catalog parser from catalog.py
+ 'version': self.version_string,
# 'in_catalogs': unicode([unicode(x) for x in self.in_catalogs]),
}
return mimetype, data
Modified: csw/mgar/gar/v2/lib/python/opencsw.py
===================================================================
--- csw/mgar/gar/v2/lib/python/opencsw.py 2011-02-14 20:30:50 UTC (rev 13311)
+++ csw/mgar/gar/v2/lib/python/opencsw.py 2011-02-14 20:31:43 UTC (rev 13312)
@@ -165,6 +165,40 @@
return version_str, version_info, revision_info
+def ParseRevisionInfo(revinfo):
+ if "REV" in revinfo:
+ rev = revinfo["REV"]
+ m = re.match(r"(\d+)\.(\d+)\.(\d+)", rev)
+ if m:
+ return tuple([int(x) for x in m.groups()])
+ else:
+ return ()
+ else:
+ return ()
+
+
+def CompareVersions(v1, v2):
+ """Compares two package versions represented as strings.
+
+ This function should eventually converge with what pkgutil is doing.
+ """
+ # ('1.8.1', {'minor version': '8', 'patchlevel': '1', 'major version': '1'},
+ # {'REV': '2010.07.13'})
+ logging.debug("CompareVersions(%s, %s)", repr(v1), repr(v2))
+ bv1, sv1, ri1 = ParseVersionString(v1)
+ bv2, sv2, ri2 = ParseVersionString(v2)
+ vn1 = tuple([int(x) for x in re.findall(r"\d+", bv1)])
+ vn2 = tuple([int(x) for x in re.findall(r"\d+", bv2)])
+ pr1, pr2 = (), ()
+ if "REV" in ri1:
+ pr1 = ParseRevisionInfo(ri1)
+ if "REV" in ri2:
+ pr2 = ParseRevisionInfo(ri2)
+ key1 = pr1 + vn1
+ key2 = pr2 + vn2
+ return cmp(key1, key2)
+
+
class CatalogBasedOpencswPackage(object):
catalog_downloaded = False
Modified: csw/mgar/gar/v2/lib/python/opencsw_test.py
===================================================================
--- csw/mgar/gar/v2/lib/python/opencsw_test.py 2011-02-14 20:30:50 UTC (rev 13311)
+++ csw/mgar/gar/v2/lib/python/opencsw_test.py 2011-02-14 20:31:43 UTC (rev 13312)
@@ -225,6 +225,44 @@
self.assertEqual(expected, opencsw.ParseVersionString(data))
+class CompareVersionsTest(unittest.TestCase):
+
+ V1 = "1.8.1,REV=2010.07.13"
+ V2 = "1.8.2,REV=2011.01.17"
+
+ def testLessThan(self):
+ self.assertEqual(-1, opencsw.CompareVersions(self.V1, self.V2))
+
+ def testGreaterThan(self):
+ self.assertEqual(1, opencsw.CompareVersions(self.V2, self.V1))
+
+ def testEquals(self):
+ self.assertEqual(0, opencsw.CompareVersions(self.V1, self.V1))
+
+ def testRevNoRev(self):
+ self.assertEqual(-1, opencsw.CompareVersions("1.8.1", self.V1))
+
+ def testRevNoRevIgnoreVersion(self):
+ self.assertEqual(-1, opencsw.CompareVersions("1.8.2", self.V1))
+
+ def testNoRev(self):
+ self.assertEqual(-1, opencsw.CompareVersions("1.8.1", "1.8.2"))
+
+ def testNoRevLetter(self):
+ self.assertEqual(-1, opencsw.CompareVersions("1.8.1", "1.8.1b"))
+
+ def testNoEqual(self):
+ self.assertEqual(0, opencsw.CompareVersions("1.8.1", "1.8.1"))
+
+ def testParseRevisionInfo(self):
+ r1 = {"REV": "2010.07.13"}
+ self.assertEqual((2010, 07, 13), opencsw.ParseRevisionInfo(r1))
+
+ def testParseRevisionInfoEmpty(self):
+ r1 = {}
+ self.assertEqual((), opencsw.ParseRevisionInfo(r1))
+
+
class UpgradeTypeTest(unittest.TestCase):
def testUpgradeType_1(self):
Modified: csw/mgar/gar/v2/lib/python/rest.py
===================================================================
--- csw/mgar/gar/v2/lib/python/rest.py 2011-02-14 20:30:50 UTC (rev 13311)
+++ csw/mgar/gar/v2/lib/python/rest.py 2011-02-14 20:31:43 UTC (rev 13312)
@@ -6,6 +6,15 @@
BASE_URL = "http://buildfarm.opencsw.org/pkgdb/rest"
+
+class Error(Exception):
+ """Generic error."""
+
+
+class ArgumentError(Error):
+ """Wrong arguments passed."""
+
+
class RestClient(object):
def GetPkgByMd5(self, md5_sum):
@@ -27,10 +36,16 @@
}
def GetCatalog(self, catrel, arch, osrel):
+ if not catrel:
+ raise ArgumentError("Missing catalog release.")
url = BASE_URL + "/catalogs/%s/%s/%s/" % (catrel, arch, osrel)
logging.debug("GetCatalog(): GET %s", url)
- data = urllib2.urlopen(url).read()
- return json.loads(data)
+ try:
+ data = urllib2.urlopen(url).read()
+ return json.loads(data)
+ except urllib2.HTTPError, e:
+ logging.warning(e)
+ return None
def Srv4ByCatalogAndCatalogname(self, catrel, arch, osrel, catalogname):
"""Returns a srv4 data structure or None if not found."""
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
More information about the devel
mailing list