[csw-devel] SF.net SVN: gar:[13307] csw/mgar/gar/v2/lib
wahwah at users.sourceforge.net
wahwah at users.sourceforge.net
Mon Feb 14 21:26:36 CET 2011
Revision: 13307
http://gar.svn.sourceforge.net/gar/?rev=13307&view=rev
Author: wahwah
Date: 2011-02-14 20:26:36 +0000 (Mon, 14 Feb 2011)
Log Message:
-----------
csw-upload-pkg: OS-release-specific uploads
Update to csw-upload-pkg based on a suggestion from Peter Felecan. When
uploading a 5.9 file, for example, it'll look at the current state of the
catalog, and if a 5.10-specific package is present, csw-upload-pkg will only
upload the 5.9 package to the 5.9 catalog. If, on the other hand, there is
the same version of the package in 5.10 (and 5.11), csw-upload-pkg will upload
the package there as well.
There's a new flag, --os-release, which makes csw-upload-pkg upload a package
to one, specified, catalog only.
Modified Paths:
--------------
csw/mgar/gar/v2/lib/python/csw_upload_pkg.py
csw/mgar/gar/v2/lib/python/pkgdb.py
csw/mgar/gar/v2/lib/python/rest.py
csw/mgar/gar/v2/lib/python/struct_util.py
csw/mgar/gar/v2/lib/python/struct_util_test.py
csw/mgar/gar/v2/lib/web/pkgdb_web.py
Added Paths:
-----------
csw/mgar/gar/v2/lib/python/csw_upload_pkg_test.py
Modified: csw/mgar/gar/v2/lib/python/csw_upload_pkg.py
===================================================================
--- csw/mgar/gar/v2/lib/python/csw_upload_pkg.py 2011-02-14 18:49:29 UTC (rev 13306)
+++ csw/mgar/gar/v2/lib/python/csw_upload_pkg.py 2011-02-14 20:26:36 UTC (rev 13307)
@@ -16,22 +16,28 @@
import json
import common_constants
import socket
+import rest
BASE_URL = "http://buildfarm.opencsw.org/releases/"
+DEFAULT_CATREL = "unstable"
USAGE = """%prog [ options ] <pkg1> [ <pkg2> [ ... ] ]
Uploads a set of packages to the unstable catalog in opencsw-future.
- When an ARCH=all package is sent, it's added to both sparc and i386 catalogs
- When a SunOS5.x package is sent, it's added to catalogs SunOS5.x,
- SunOS5.(x+1), up to SunOS5.11.
+ SunOS5.(x+1), up to SunOS5.11, but only if there are no packages specific to
+ 5.10 (and/or 5.11).
- If a package update is sent, the tool uses catalogname to identify the
- package it's supposed to replace
+ package it's supposed to replace
The --remove option affects the same catalogs as the regular use, except that
it removes assignments of a given package to catalogs, instead of adding them.
+The --os-release flag makes %prog only insert the package to catalog with the
+given OS release.
+
For more information, see:
http://wiki.opencsw.org/automated-release-process#toc0
"""
@@ -54,10 +60,12 @@
class Srv4Uploader(object):
- def __init__(self, filenames, debug=False):
+ def __init__(self, filenames, os_release=None, debug=False):
self.filenames = filenames
self.md5_by_filename = {}
self.debug = debug
+ self._rest_client = rest.RestClient()
+ self.os_release = os_release
def Upload(self):
for filename in self.filenames:
@@ -78,14 +86,16 @@
file_in_allpkgs, file_metadata = self._GetSrv4FileMetadata(md5_sum)
osrel = file_metadata['osrel']
arch = file_metadata['arch']
- self._IterateOverCatalogs(
- filename, file_metadata,
- arch, osrel, self._RemoveFromCatalog)
+ catalogs = self._MatchSrv4ToCatalogs(
+ filename, DEFAULT_CATREL, arch, osrel, md5_sum)
+ for unused_catrel, cat_arch, cat_osrel in catalogs:
+ self._RemoveFromCatalog(filename, file_metadata, cat_arch, cat_osrel)
def _RemoveFromCatalog(self, filename, arch, osrel, file_metadata):
md5_sum = self._GetFileMd5sum(filename)
basename = os.path.basename(filename)
parsed_basename = opencsw.ParsePackageFileName(basename)
+ # TODO: Move this bit to a separate class (RestClient)
url = (
"%scatalogs/unstable/%s/%s/%s/"
% (BASE_URL, arch, osrel, md5_sum))
@@ -121,19 +131,47 @@
self.md5_by_filename[filename] = md5_sum
return self.md5_by_filename[filename]
- def _IterateOverCatalogs(self, filename, file_metadata, arch, osrel, callback):
- # Implementing backward compatibility. A package for SunOS5.x is also
- # inserted into SunOS5.(x+n) for n=(0, 1, ...)
+ def _MatchSrv4ToCatalogs(self, filename,
+ catrel, srv4_arch, srv4_osrel,
+ md5_sum):
+ """Compile a list of catalogs affected by the given file.
+
+ If it's a 5.9 package, it can be matched to 5.9, 5.10 and 5.11. However,
+ if there already are specific 5.10 and/or 5.11 versions of the package,
+ don't overwrite them.
+ """
+ basename = os.path.basename(filename)
+ parsed_basename = opencsw.ParsePackageFileName(basename)
+ osrels = None
for idx, known_osrel in enumerate(common_constants.OS_RELS):
- if osrel == known_osrel:
+ if srv4_osrel == known_osrel:
osrels = common_constants.OS_RELS[idx:]
- if arch == 'all':
+ assert osrels, "OS releases not found"
+ if srv4_arch == 'all':
archs = ('sparc', 'i386')
else:
- archs = (arch,)
+ archs = (srv4_arch,)
+ catalogname = parsed_basename["catalogname"]
+ catalogs = []
for arch in archs:
for osrel in osrels:
- callback(filename, arch, osrel, file_metadata)
+ srv4_in_catalog = self._rest_client.Srv4ByCatalogAndCatalogname(
+ catrel, arch, osrel, catalogname)
+ if not srv4_in_catalog:
+ continue
+ if srv4_in_catalog["osrel"] == srv4_osrel:
+ # The same architecture as our package, meaning that we can insert
+ # the same architecture into the catalog.
+ if (not self.os_release
+ or (self.os_release and osrel == self.os_release)):
+ catalogs.append((catrel, arch, osrel))
+ else:
+ if self.os_release and osrel == self.os_release:
+ catalogs.append((catrel, arch, osrel))
+ logging.debug(
+ "Catalog %s %s %s has another version of %s.",
+ catrel, arch, osrel, catalogname)
+ return tuple(catalogs)
def _UploadFile(self, filename):
md5_sum = self._GetFileMd5sum(filename)
@@ -149,9 +187,10 @@
raise DataError("file_metadata is empty: %s" % repr(file_metadata))
osrel = file_metadata['osrel']
arch = file_metadata['arch']
- self._IterateOverCatalogs(
- filename, file_metadata,
- arch, osrel, self._InsertIntoCatalog)
+ catalogs = self._MatchSrv4ToCatalogs(
+ filename, DEFAULT_CATREL, arch, osrel, md5_sum)
+ for unused_catrel, cat_arch, cat_osrel in catalogs:
+ self._InsertIntoCatalog(filename, file_metadata, cat_arch, cat_osrel)
def _InsertIntoCatalog(self, filename, arch, osrel, file_metadata):
logging.info(
@@ -272,6 +311,9 @@
dest="remove",
default=False, action="store_true",
help="Remove packages from catalogs instead of adding them")
+ parser.add_option("--os-release",
+ dest="os_release",
+ help="If specified, only uploads to the specified OS release.")
options, args = parser.parse_args()
if options.debug:
logging.basicConfig(level=logging.DEBUG)
@@ -281,7 +323,12 @@
hostname = socket.gethostname()
if not hostname.startswith('login'):
logging.warning("This script is meant to be run on the login host.")
- uploader = Srv4Uploader(args, debug=options.debug)
+ os_release = options.os_release
+ if os_release:
+ os_release = struct_util.OsReleaseToLong(os_release)
+ uploader = Srv4Uploader(args,
+ os_release=os_release,
+ debug=options.debug)
if options.remove:
uploader.Remove()
else:
Added: csw/mgar/gar/v2/lib/python/csw_upload_pkg_test.py
===================================================================
--- csw/mgar/gar/v2/lib/python/csw_upload_pkg_test.py (rev 0)
+++ csw/mgar/gar/v2/lib/python/csw_upload_pkg_test.py 2011-02-14 20:26:36 UTC (rev 13307)
@@ -0,0 +1,151 @@
+#!/usr/bin/env python2.6
+
+import unittest
+import csw_upload_pkg
+import mox
+import rest
+import copy
+
+GDB_STRUCT_9 = {
+ "arch": "sparc",
+ "basename": "gdb-7.2,REV=2011.01.21-SunOS5.9-sparc-CSW.pkg.gz",
+ "catalogname": "gdb",
+ "filename_arch": "sparc",
+ "maintainer_email": "pfele... at opencsw.org",
+ "maintainer_full_name": None,
+ "md5_sum": "7971e31461b53638d7813407fab4765b",
+ "mtime": "2011-01-24 03:09:54",
+ "osrel": "SunOS5.9",
+ "pkgname": "CSWgdb",
+ "rev": "2011.01.21",
+ "size": 7616184,
+ "version_string": "7.2,REV=2011.01.21",
+}
+GDB_STRUCT_10 = {
+ "arch": "sparc",
+ "basename": "gdb-7.2,REV=2011.01.21-SunOS5.10-sparc-CSW.pkg.gz",
+ "catalogname": "gdb",
+ "filename_arch": "sparc",
+ "maintainer_email": "pfele... at opencsw.org",
+ "maintainer_full_name": None,
+ "md5_sum": "09cccf8097e982dadbd717910963e378",
+ "mtime": "2011-01-24 03:10:05",
+ "osrel": "SunOS5.10",
+ "pkgname": "CSWgdb",
+ "rev": "2011.01.21",
+ "size": 7617270,
+ "version_string": "7.2,REV=2011.01.21",
+}
+
+class Srv4UploaderUnitTest(mox.MoxTestBase):
+
+ def test_MatchSrv4ToCatalogsSame(self):
+ rest_client_mock = self.mox.CreateMock(rest.RestClient)
+ self.mox.StubOutWithMock(rest, "RestClient")
+ rest.RestClient().AndReturn(rest_client_mock)
+ rest_client_mock.Srv4ByCatalogAndCatalogname(
+ 'unstable', 'sparc', u'SunOS5.9', 'gdb').AndReturn(GDB_STRUCT_9)
+ rest_client_mock.Srv4ByCatalogAndCatalogname(
+ 'unstable', 'sparc', u'SunOS5.10', 'gdb').AndReturn(GDB_STRUCT_9)
+ rest_client_mock.Srv4ByCatalogAndCatalogname(
+ 'unstable', 'sparc', u'SunOS5.11', 'gdb').AndReturn(GDB_STRUCT_9)
+ self.mox.ReplayAll()
+ su = csw_upload_pkg.Srv4Uploader(None)
+ result = su._MatchSrv4ToCatalogs(
+ "gdb-7.2,REV=2011.01.21-SunOS5.9-sparc-CSW.pkg.gz",
+ "unstable", "sparc", "SunOS5.9",
+ "deadbeef61b53638d7813407fab4765b")
+ expected = (
+ ("unstable", "sparc", "SunOS5.9"),
+ ("unstable", "sparc", "SunOS5.10"),
+ ("unstable", "sparc", "SunOS5.11"),
+ )
+ self.assertEquals(expected, result)
+
+ def test_MatchSrv4ToCatalogsDifferent(self):
+ rest_client_mock = self.mox.CreateMock(rest.RestClient)
+ self.mox.StubOutWithMock(rest, "RestClient")
+ rest.RestClient().AndReturn(rest_client_mock)
+ rest_client_mock.Srv4ByCatalogAndCatalogname(
+ 'unstable', 'sparc', u'SunOS5.9', 'gdb').AndReturn(GDB_STRUCT_9)
+ rest_client_mock.Srv4ByCatalogAndCatalogname(
+ 'unstable', 'sparc', u'SunOS5.10', 'gdb').AndReturn(GDB_STRUCT_10)
+ rest_client_mock.Srv4ByCatalogAndCatalogname(
+ 'unstable', 'sparc', u'SunOS5.11', 'gdb').AndReturn(GDB_STRUCT_10)
+ self.mox.ReplayAll()
+ su = csw_upload_pkg.Srv4Uploader(None)
+ result = su._MatchSrv4ToCatalogs(
+ "gdb-7.2,REV=2011.01.21-SunOS5.9-sparc-CSW.pkg.gz",
+ "unstable", "sparc", "SunOS5.9",
+ "deadbeef61b53638d7813407fab4765b")
+ expected = (
+ ("unstable", "sparc", "SunOS5.9"),
+ )
+ self.assertEquals(expected, result)
+
+ def test_MatchSrv4ToCatalogsSameSpecificOsrel(self):
+ rest_client_mock = self.mox.CreateMock(rest.RestClient)
+ self.mox.StubOutWithMock(rest, "RestClient")
+ rest.RestClient().AndReturn(rest_client_mock)
+ rest_client_mock.Srv4ByCatalogAndCatalogname(
+ 'unstable', 'sparc', u'SunOS5.9', 'gdb').AndReturn(GDB_STRUCT_9)
+ rest_client_mock.Srv4ByCatalogAndCatalogname(
+ 'unstable', 'sparc', u'SunOS5.10', 'gdb').AndReturn(GDB_STRUCT_9)
+ rest_client_mock.Srv4ByCatalogAndCatalogname(
+ 'unstable', 'sparc', u'SunOS5.11', 'gdb').AndReturn(GDB_STRUCT_9)
+ self.mox.ReplayAll()
+ su = csw_upload_pkg.Srv4Uploader(None, os_release="SunOS5.10")
+ result = su._MatchSrv4ToCatalogs(
+ "gdb-7.2,REV=2011.01.21-SunOS5.9-sparc-CSW.pkg.gz",
+ "unstable", "sparc", "SunOS5.9",
+ "deadbeef61b53638d7813407fab4765b")
+ expected = (
+ ("unstable", "sparc", "SunOS5.10"),
+ )
+ self.assertEquals(expected, result)
+
+ def test_MatchSrv4ToCatalogsSameSpecificOsrelAlreadyPresent(self):
+ rest_client_mock = self.mox.CreateMock(rest.RestClient)
+ self.mox.StubOutWithMock(rest, "RestClient")
+ rest.RestClient().AndReturn(rest_client_mock)
+ rest_client_mock.Srv4ByCatalogAndCatalogname(
+ 'unstable', 'sparc', u'SunOS5.9', 'gdb').AndReturn(GDB_STRUCT_9)
+ rest_client_mock.Srv4ByCatalogAndCatalogname(
+ 'unstable', 'sparc', u'SunOS5.10', 'gdb').AndReturn(GDB_STRUCT_10)
+ rest_client_mock.Srv4ByCatalogAndCatalogname(
+ 'unstable', 'sparc', u'SunOS5.11', 'gdb').AndReturn(GDB_STRUCT_10)
+ self.mox.ReplayAll()
+ su = csw_upload_pkg.Srv4Uploader(None, os_release="SunOS5.10")
+ result = su._MatchSrv4ToCatalogs(
+ "gdb-7.2,REV=2011.01.21-SunOS5.9-sparc-CSW.pkg.gz",
+ "unstable", "sparc", "SunOS5.9",
+ "deadbeef61b53638d7813407fab4765b")
+ expected = (
+ ("unstable", "sparc", "SunOS5.10"),
+ )
+ self.assertEquals(expected, result)
+
+ def test_MatchSrv4ToCatalogsNotPresent(self):
+ rest_client_mock = self.mox.CreateMock(rest.RestClient)
+ self.mox.StubOutWithMock(rest, "RestClient")
+ rest.RestClient().AndReturn(rest_client_mock)
+ rest_client_mock.Srv4ByCatalogAndCatalogname(
+ 'unstable', 'sparc', u'SunOS5.9', 'gdb').AndReturn(GDB_STRUCT_9)
+ rest_client_mock.Srv4ByCatalogAndCatalogname(
+ 'unstable', 'sparc', u'SunOS5.10', 'gdb').AndReturn(GDB_STRUCT_9)
+ rest_client_mock.Srv4ByCatalogAndCatalogname(
+ 'unstable', 'sparc', u'SunOS5.11', 'gdb').AndReturn(None)
+ self.mox.ReplayAll()
+ su = csw_upload_pkg.Srv4Uploader(None, os_release="SunOS5.10")
+ result = su._MatchSrv4ToCatalogs(
+ "gdb-7.2,REV=2011.01.21-SunOS5.9-sparc-CSW.pkg.gz",
+ "unstable", "sparc", "SunOS5.9",
+ "deadbeef61b53638d7813407fab4765b")
+ expected = (
+ ("unstable", "sparc", "SunOS5.10"),
+ )
+ self.assertEquals(expected, result)
+
+
+if __name__ == '__main__':
+ unittest.main()
Property changes on: csw/mgar/gar/v2/lib/python/csw_upload_pkg_test.py
___________________________________________________________________
Added: svn:executable
+ *
Modified: csw/mgar/gar/v2/lib/python/pkgdb.py
===================================================================
--- csw/mgar/gar/v2/lib/python/pkgdb.py 2011-02-14 18:49:29 UTC (rev 13306)
+++ csw/mgar/gar/v2/lib/python/pkgdb.py 2011-02-14 20:26:36 UTC (rev 13307)
@@ -507,8 +507,8 @@
m.Srv4FileInCatalog.q.catrel==sqo_catrel,
m.Srv4FileStats.q.catalogname.contains(catalogname),
m.Srv4FileStats.q.use_to_generate_catalogs==True),
- join=join,
- ).orderBy("catalogname")
+ join=join,
+ ).orderBy("catalogname")
for sqo_srv4 in res:
print "%s %s" % (sqo_srv4.basename, sqo_srv4.md5_sum)
elif command == 'sync-cat-from-file':
Modified: csw/mgar/gar/v2/lib/python/rest.py
===================================================================
--- csw/mgar/gar/v2/lib/python/rest.py 2011-02-14 18:49:29 UTC (rev 13306)
+++ csw/mgar/gar/v2/lib/python/rest.py 2011-02-14 20:26:36 UTC (rev 13307)
@@ -31,3 +31,16 @@
logging.debug("GetCatalog(): GET %s", url)
data = urllib2.urlopen(url).read()
return json.loads(data)
+
+ def Srv4ByCatalogAndCatalogname(self, catrel, arch, osrel, catalogname):
+ """Returns a srv4 data structure or None if not found."""
+ url = BASE_URL + (
+ "/catalogs/%s/%s/%s/%s/"
+ % (catrel, arch, osrel, catalogname))
+ logging.debug("Srv4ByCatalogAndCatalogname(): GET %s", url)
+ try:
+ data = urllib2.urlopen(url).read()
+ return json.loads(data)
+ except urllib2.HTTPError, e:
+ logging.warning(e)
+ return None
Modified: csw/mgar/gar/v2/lib/python/struct_util.py
===================================================================
--- csw/mgar/gar/v2/lib/python/struct_util.py 2011-02-14 18:49:29 UTC (rev 13306)
+++ csw/mgar/gar/v2/lib/python/struct_util.py 2011-02-14 20:26:36 UTC (rev 13307)
@@ -18,6 +18,13 @@
return index
+def OsReleaseToLong(osrel):
+ if osrel.startswith("SunOS"):
+ return osrel
+ else:
+ return "SunOS%s" % osrel
+
+
def ResolveSymlink(link_from, link_to):
target = os.path.normpath(
os.path.join(os.path.dirname(link_from), link_to))
Modified: csw/mgar/gar/v2/lib/python/struct_util_test.py
===================================================================
--- csw/mgar/gar/v2/lib/python/struct_util_test.py 2011-02-14 18:49:29 UTC (rev 13306)
+++ csw/mgar/gar/v2/lib/python/struct_util_test.py 2011-02-14 20:26:36 UTC (rev 13307)
@@ -46,6 +46,14 @@
"/libexec/foo-exec",
struct_util.ResolveSymlink("/opt/csw/bin/foo", "/libexec/foo-exec"))
+class OsReleaseToLongTest(unittest.TestCase):
+ def testLong(self):
+ self.assertEqual("SunOS5.9", struct_util.OsReleaseToLong("SunOS5.9"))
+
+ def testShort(self):
+ self.assertEqual("SunOS5.9", struct_util.OsReleaseToLong("5.9"))
+
+
if __name__ == '__main__':
unittest.main()
Modified: csw/mgar/gar/v2/lib/web/pkgdb_web.py
===================================================================
--- csw/mgar/gar/v2/lib/web/pkgdb_web.py 2011-02-14 18:49:29 UTC (rev 13306)
+++ csw/mgar/gar/v2/lib/web/pkgdb_web.py 2011-02-14 20:26:36 UTC (rev 13307)
@@ -30,6 +30,8 @@
r'/rest/catalogs/([^/]+)/([^/]+)/([^/]+)/', 'Catalogs',
r'/rest/catalogs/([^/]+)/([^/]+)/([^/]+)/pkgname-by-filename',
'PkgnameByFilename',
+ # Query by catalog release, arch, OS release and catalogname
+ r'/rest/catalogs/([^/]+)/([^/]+)/([^/]+)/catalognames/([^/]+)/', 'Srv4ByCatAndCatalogname',
r'/rest/srv4/([0-9a-f]{32})/', 'RestSrv4Detail',
)
@@ -268,6 +270,38 @@
raise web.notfound()
+class Srv4ByCatAndCatalogname(object):
+
+ def GET(self, catrel_name, arch_name, osrel_name, catalogname):
+ """Get a srv4 reference by catalog ane catalogname."""
+ configuration.SetUpSqlobjectConnection()
+ sqo_osrel, sqo_arch, sqo_catrel = pkgdb.GetSqoTriad(
+ osrel_name, arch_name, catrel_name)
+ join = [
+ sqlbuilder.INNERJOINOn(None,
+ models.Srv4FileInCatalog,
+ models.Srv4FileInCatalog.q.srv4file==models.Srv4FileStats.q.id),
+ ]
+ res = models.Srv4FileStats.select(
+ sqlobject.AND(
+ models.Srv4FileInCatalog.q.osrel==sqo_osrel,
+ models.Srv4FileInCatalog.q.arch==sqo_arch,
+ models.Srv4FileInCatalog.q.catrel==sqo_catrel,
+ models.Srv4FileStats.q.catalogname==catalogname,
+ models.Srv4FileStats.q.use_to_generate_catalogs==True),
+ join=join,
+ )
+ try:
+ srv4 = res.getOne()
+ mimetype, data = srv4.GetRestRepr()
+ web.header('Content-type', mimetype)
+ return json.dumps(data)
+ except (
+ sqlobject.main.SQLObjectNotFound,
+ sqlobject.dberrors.OperationalError), e:
+ raise web.notfound()
+
+
web.webapi.internalerror = web.debugerror
# app = web.application(urls, globals())
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
More information about the devel
mailing list