[csw-devel] SF.net SVN: gar:[19649] csw/mgar/gar/v2/lib/python
wahwah at users.sourceforge.net
wahwah at users.sourceforge.net
Sun Nov 11 11:35:34 CET 2012
Revision: 19649
http://gar.svn.sourceforge.net/gar/?rev=19649&view=rev
Author: wahwah
Date: 2012-11-11 10:35:34 +0000 (Sun, 11 Nov 2012)
Log Message:
-----------
safe_remove_package.py: Revive
The safe_remove_package utility was not working, because it was shelling out
to csw-upload-pkg which had the --remove option removed some 4 months ago.
Instead of calling csw-upload-pkg, it's not using the RestClient class from
rest.py. It's now much faster, because it's avoiding the csw-upload-pkg
startup time.
Happy removing!
Modified Paths:
--------------
csw/mgar/gar/v2/lib/python/csw_upload_pkg.py
csw/mgar/gar/v2/lib/python/rest.py
csw/mgar/gar/v2/lib/python/safe_remove_package.py
Modified: csw/mgar/gar/v2/lib/python/csw_upload_pkg.py
===================================================================
--- csw/mgar/gar/v2/lib/python/csw_upload_pkg.py 2012-11-11 10:20:24 UTC (rev 19648)
+++ csw/mgar/gar/v2/lib/python/csw_upload_pkg.py 2012-11-11 10:35:34 UTC (rev 19649)
@@ -529,16 +529,7 @@
else:
print "Continuing anyway."
- username = os.environ["LOGNAME"]
- authfile = os.path.join('/etc/opt/csw/releases/auth', username)
-
- try:
- with open(authfile, 'r') as af:
- password = af.read().strip()
- except IOError, e:
- logging.warning("Error reading %s: %s", authfile, e)
- password = getpass.getpass("{0}'s pkg release password> ".format(username))
-
+ username, password = rest.GetUsernameAndPassword()
uploader = Srv4Uploader(args,
options.rest_url,
os_release=os_release,
Modified: csw/mgar/gar/v2/lib/python/rest.py
===================================================================
--- csw/mgar/gar/v2/lib/python/rest.py 2012-11-11 10:20:24 UTC (rev 19648)
+++ csw/mgar/gar/v2/lib/python/rest.py 2012-11-11 10:35:34 UTC (rev 19649)
@@ -1,13 +1,16 @@
#!/usr/bin/env python2.6
+import os
+from StringIO import StringIO
import cjson
import gdbm
import logging
import urllib2
+import pycurl
DEFAULT_URL = "http://buildfarm.opencsw.org"
+RELEASES_APP = "/releases"
-
class Error(Exception):
"""Generic error."""
@@ -16,12 +19,20 @@
"""Wrong arguments passed."""
+class RestCommunicationError(Error):
+ """An error during REST request processing."""
+
+
class RestClient(object):
PKGDB_APP = "/pkgdb/rest"
- def __init__(self, rest_url=DEFAULT_URL):
+ def __init__(self, rest_url=DEFAULT_URL, username=None, password=None,
+ debug=False):
self.rest_url = rest_url
+ self.username = username
+ self.password = password
+ self.debug = debug
def GetPkgByMd5(self, md5_sum):
url = self.rest_url + self.PKGDB_APP + "/srv4/%s/" % md5_sum
@@ -86,7 +97,60 @@
data = urllib2.urlopen(url).read()
return cjson.decode(data)
+ def Srv4ByCatalogAndPkgname(self, catrel, arch, osrel, pkgname):
+ """Returns a srv4 data structure or None if not found."""
+ url = self.rest_url + self.PKGDB_APP + (
+ "/catalogs/%s/%s/%s/pkgnames/%s/"
+ % (catrel, arch, osrel, pkgname))
+ logging.debug("Srv4ByCatalogAndPkgname(): GET %s", url)
+ # The server is no longer returning 404 when the package is absent. If
+ # a HTTP error code is returned, we're letting the application fail.
+ data = urllib2.urlopen(url).read()
+ return cjson.decode(data)
+ def _SetAuth(self, c):
+ """Set basic HTTP auth options on given Curl object."""
+ if self.username:
+ logging.debug("Using basic AUTH for user %s", self.username)
+ c.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_ANY)
+ c.setopt(pycurl.USERPWD, "%s:%s" % (self.username, self.password))
+ else:
+ logging.debug("User and password not set, not using HTTP AUTH")
+ return c
+
+ def RemoveSvr4FromCatalog(self, catrel, arch, osrel, md5_sum):
+ url = (
+ "%s%s/catalogs/%s/%s/%s/%s/"
+ % (self.rest_url,
+ RELEASES_APP,
+ catrel, arch, osrel,
+ md5_sum))
+ logging.debug("DELETE @ URL: %s %s", type(url), url)
+ c = pycurl.Curl()
+ d = StringIO()
+ h = StringIO()
+ c.setopt(pycurl.URL, str(url))
+ c.setopt(pycurl.CUSTOMREQUEST, "DELETE")
+ c.setopt(pycurl.WRITEFUNCTION, d.write)
+ c.setopt(pycurl.HEADERFUNCTION, h.write)
+ c.setopt(pycurl.HTTPHEADER, ["Expect:"]) # Fixes the HTTP 417 error
+ c = self._SetAuth(c)
+ if self.debug:
+ c.setopt(c.VERBOSE, 1)
+ c.perform()
+ http_code = c.getinfo(pycurl.HTTP_CODE)
+ logging.debug(
+ "DELETE curl getinfo: %s %s %s",
+ type(http_code),
+ http_code,
+ c.getinfo(pycurl.EFFECTIVE_URL))
+ c.close()
+ if not (http_code >= 200 and http_code <= 299):
+ raise RestCommunicationError(
+ "%s - HTTP code: %s, content: %s"
+ % (url, http_code, d.getvalue()))
+
+
class CachedPkgstats(object):
"""Class responsible for holding and caching package stats.
@@ -120,3 +184,15 @@
"pkgname": pkgstats["basic_stats"]["pkgname"]}
self.deps[md5] = cjson.encode(data)
return data
+
+def GetUsernameAndPassword():
+ username = os.environ["LOGNAME"]
+ password = None
+ authfile = os.path.join('/etc/opt/csw/releases/auth', username)
+ try:
+ with open(authfile, 'r') as af:
+ password = af.read().strip()
+ except IOError, e:
+ logging.warning("Error reading %s: %s", authfile, e)
+ password = getpass.getpass("{0}'s pkg release password> ".format(username))
+ return username, password
Modified: csw/mgar/gar/v2/lib/python/safe_remove_package.py
===================================================================
--- csw/mgar/gar/v2/lib/python/safe_remove_package.py 2012-11-11 10:20:24 UTC (rev 19648)
+++ csw/mgar/gar/v2/lib/python/safe_remove_package.py 2012-11-11 10:35:34 UTC (rev 19649)
@@ -6,7 +6,8 @@
- checks all the catalogs and gets the md5 sums for each catalog
- checks for reverse dependencies; if there are any, stops
- - when there are no rev deps, prints a csw-upload-pkg --remove call
+ - when there are no rev deps, makes a REST call to remove the package
+
"""
import optparse
@@ -18,12 +19,34 @@
import sys
import os
import cjson
-import subprocess
+USAGE = """%prog --os-releases=SunOS5.10,SunOS5.11 -c <catalogname>
+A practical usage example - let's say we have a list of packages to remove in
+a file named 'pkg-list.txt'. We'll also have a cache of packages already
+removed in packages_dropped_cache.txt. The following call will remove the
+listed packages:
+
+for p in $(cat pkg-list.txt)
+do
+ if ! ggrep "^$p\$" packages_dropped_cache.txt > /dev/null
+ then
+ ./safe_remove_package.py \\
+ --os-releases=SunOS5.10,SunOS5.11 \\
+ -c "$p"
+ fi
+done
+"""
+
+
+UNSTABLE = "unstable"
+
class Error(Exception):
"""A generic error."""
+class DataError(Exception):
+ """Wrong data encountered."""
+
class RevDeps(object):
def __init__(self):
@@ -43,7 +66,6 @@
catalog = self.rest_client.GetCatalog(*key)
rev_deps = {}
for pkg_simple in catalog:
- # pprint.pprint(pkg_simple)
md5 = pkg_simple["md5_sum"]
# pkg = self.cp.GetPkgstats(md5)
short_data = self.cp.GetDeps(md5)
@@ -68,14 +90,20 @@
class PackageRemover(object):
+ def CachePackageIsGone(self, catalogname):
+ with open("packages_dropped_cache.txt", "ab") as fd:
+ fd.write("{0}\n".format(catalogname))
+
def RemovePackage(self, catalogname, execute=False, os_releases=None):
if not os_releases:
os_releases = common_constants.OS_RELS
- # Get md5 sums
- rest_client = rest.RestClient()
+ username, password = rest.GetUsernameAndPassword()
+ rest_client = rest.RestClient(username=username, password=password)
rd = RevDeps()
rev_deps = {}
- to_remove = {}
+ # md5 sums to remove
+ to_remove = []
+ found_anywhere = False
for osrel in os_releases:
if osrel not in common_constants.OS_RELS:
logging.warning(
@@ -86,41 +114,52 @@
logging.info("%s is an obsolete OS release. Skipping.", osrel)
continue
for arch in common_constants.PHYSICAL_ARCHITECTURES:
- pkg_simple = rest_client.Srv4ByCatalogAndCatalogname("unstable", arch, osrel, catalogname)
+ pkg_simple = rest_client.Srv4ByCatalogAndCatalogname(UNSTABLE, arch, osrel, catalogname)
+ if not pkg_simple:
+ # Maybe we were given a pkgname instead of a catalogname? We can try
+ # that before failing.
+ pkg_simple = rest_client.Srv4ByCatalogAndPkgname(
+ UNSTABLE, arch, osrel, catalogname)
+ if not pkg_simple:
+ msg = "{0} was not in the unstable {1} {2} catalog."
+ logging.debug(msg.format(repr(catalogname), arch, osrel))
+ continue
+ if pkg_simple:
+ found_anywhere = True
md5 = pkg_simple["md5_sum"]
pkg = rd.cp.GetPkgstats(md5)
- key = "unstable", arch, osrel
- cat_rev_deps = rd.RevDeps("unstable", arch, osrel, md5)
+ key = UNSTABLE, arch, osrel
+ cat_rev_deps = rd.RevDeps(UNSTABLE, arch, osrel, md5)
if cat_rev_deps:
rev_deps[key] = cat_rev_deps
- f = (
- "/home/mirror/opencsw/unstable/%s/%s/%s"
- % (arch, osrel.replace("SunOS", ""), pkg["basic_stats"]["pkg_basename"]))
- files = to_remove.setdefault(osrel, [])
- files.append(f)
+ to_remove.append((UNSTABLE, arch, osrel, md5))
+ if not found_anywhere:
+ self.CachePackageIsGone(catalogname)
if rev_deps:
- print "Reverse dependencies found. Bailing out."
- pprint.pprint(rev_deps)
+ print "Not removing, rev-deps present: ",
+ print pkg_simple["catalogname"], ":", " ; ".join(
+ ["%s %s %s %s"
+ % (x[0], x[1], x[2], ",".join(y[1] for y in rev_deps[x]))
+ for x in rev_deps])
else:
- for osrel in to_remove:
- args = ["csw-upload-pkg", "--remove", "--os-release",
- osrel] + to_remove[osrel]
- print " ".join(args)
+ for catrel, arch, osrel, md5_sum in to_remove:
+ print "# [%s]" % pkg_simple["catalogname"], catrel, arch, osrel, md5_sum
if execute:
- subprocess.call(args)
+ rest_client.RemoveSvr4FromCatalog(catrel, arch, osrel, md5_sum)
+ if found_anywhere:
+ self.CachePackageIsGone(catalogname)
-
def main():
- parser = optparse.OptionParser()
+ parser = optparse.OptionParser(USAGE)
parser.add_option("-c", "--catalogname", dest="catalogname")
parser.add_option("--os-releases", dest="os_releases",
help=("Comma separated OS releases, e.g. "
"SunOS5.9,SunOS5.10"))
parser.add_option("--debug", dest="debug", action="store_true")
- parser.add_option("--execute", dest="execute", action="store_true",
- help=("Don't just display, but execute and remove the "
- "packages."))
+ parser.add_option("--dry-run", dest="dry_run",
+ default=False, action="store_true",
+ help=("Don't remove the packages packages."))
options, args = parser.parse_args()
debug_level = logging.INFO
if options.debug:
@@ -130,7 +169,7 @@
if options.os_releases:
os_releases = options.os_releases.split(",")
pr = PackageRemover()
- pr.RemovePackage(options.catalogname, options.execute, os_releases)
+ pr.RemovePackage(options.catalogname, not options.dry_run, os_releases)
if __name__ == '__main__':
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
More information about the devel
mailing list