[csw-devel] SF.net SVN: gar:[17688] csw/mgar/gar/v2/lib/python
wahwah at users.sourceforge.net
wahwah at users.sourceforge.net
Sun Apr 15 10:31:21 CEST 2012
Revision: 17688
http://gar.svn.sourceforge.net/gar/?rev=17688&view=rev
Author: wahwah
Date: 2012-04-15 08:31:21 +0000 (Sun, 15 Apr 2012)
Log Message:
-----------
safe-remove-package: Initial commit
A script to help safely remove packages. Checks reverse dependencies of
a package, and prints out what to do (csw-upload-pkg --remove or bailout with
a reason).
Modified Paths:
--------------
csw/mgar/gar/v2/lib/python/integrate_catalogs.py
csw/mgar/gar/v2/lib/python/rest.py
Added Paths:
-----------
csw/mgar/gar/v2/lib/python/safe_remove_package.py
Modified: csw/mgar/gar/v2/lib/python/integrate_catalogs.py
===================================================================
--- csw/mgar/gar/v2/lib/python/integrate_catalogs.py 2012-04-15 08:30:00 UTC (rev 17687)
+++ csw/mgar/gar/v2/lib/python/integrate_catalogs.py 2012-04-15 08:31:21 UTC (rev 17688)
@@ -109,30 +109,6 @@
return dict((x[field], x) for x in d)
-class CachedPkgstats(object):
- """Class responsible for holding and caching package stats.
-
- Wraps RestClient and provides a caching layer.
- """
-
- def __init__(self, filename):
- self.filename = filename
- self.d = gdbm.open(filename, "c")
- self.rest_client = rest.RestClient()
- self.local_cache = {}
-
- def GetPkgstats(self, md5):
- if md5 in self.local_cache:
- return self.local_cache[md5]
- elif str(md5) in self.d:
- return cjson.decode(self.d[md5])
- else:
- pkgstats = self.rest_client.GetPkgstatsByMd5(md5)
- self.d[md5] = cjson.encode(pkgstats)
- self.local_cache[md5] = pkgstats
- return pkgstats
-
-
def GetDiffsByCatalogname(catrel_from, catrel_to, include_downgrades,
include_version_changes):
rest_client = rest.RestClient()
@@ -265,7 +241,7 @@
options.include_version_changes)
bundles_by_md5 = {}
bundles_missing = set()
- cp = CachedPkgstats("pkgstats.db")
+ cp = rest.CachedPkgstats("pkgstats.db")
for key in catalogs:
for pkg in catalogs[key]:
# logging.debug("%r", pkg)
Modified: csw/mgar/gar/v2/lib/python/rest.py
===================================================================
--- csw/mgar/gar/v2/lib/python/rest.py 2012-04-15 08:30:00 UTC (rev 17687)
+++ csw/mgar/gar/v2/lib/python/rest.py 2012-04-15 08:31:21 UTC (rev 17688)
@@ -1,8 +1,9 @@
#!/usr/bin/env python2.6
+import cjson
+import gdbm
import logging
import urllib2
-import cjson
DEFAULT_URL = "http://buildfarm.opencsw.org"
@@ -84,3 +85,27 @@
# a HTTP error code is returned, we're letting the application fail.
data = urllib2.urlopen(url).read()
return cjson.decode(data)
+
+
+class CachedPkgstats(object):
+ """Class responsible for holding and caching package stats.
+
+ Wraps RestClient and provides a caching layer.
+ """
+
+ def __init__(self, filename):
+ self.filename = filename
+ self.d = gdbm.open(filename, "c")
+ self.rest_client = RestClient()
+ self.local_cache = {}
+
+ def GetPkgstats(self, md5):
+ if md5 in self.local_cache:
+ return self.local_cache[md5]
+ elif str(md5) in self.d:
+ return cjson.decode(self.d[md5])
+ else:
+ pkgstats = self.rest_client.GetPkgstatsByMd5(md5)
+ self.d[md5] = cjson.encode(pkgstats)
+ self.local_cache[md5] = pkgstats
+ return pkgstats
Added: csw/mgar/gar/v2/lib/python/safe_remove_package.py
===================================================================
--- csw/mgar/gar/v2/lib/python/safe_remove_package.py (rev 0)
+++ csw/mgar/gar/v2/lib/python/safe_remove_package.py 2012-04-15 08:31:21 UTC (rev 17688)
@@ -0,0 +1,116 @@
+#!/usr/bin/env python2.6
+
+"""Helps to safely remove a package (by catalogname) from the catalog.
+
+Takes a catalog name and:
+
+ - checks all the catalogs and gets the md5 sums for each catalog
+ - checks for reverse dependencies; if there are any, stops
+ - when there are no rev deps, prints a csw-upload-pkg --remove call
+"""
+
+import optparse
+import rest
+import common_constants
+import pprint
+import gdbm
+import logging
+import sys
+import os
+import cjson
+
+
+class RevDeps(object):
+
+ def __init__(self):
+ self.cached_catalogs = {}
+ self.rest_client = rest.RestClient()
+ self.cp = rest.CachedPkgstats("pkgstats.db")
+
+ def MakeRevIndex(self, catrel, arch, osrel):
+ key = (catrel, arch, osrel)
+ if key in self.cached_catalogs:
+ return
+ fn = "revdeps-%s-%s-%s.json" % key
+ if os.path.exists(fn):
+ with open(fn, "r") as fd:
+ self.cached_catalogs[key] = cjson.decode(fd.read())
+ return
+ catalog = self.rest_client.GetCatalog(*key)
+ rev_deps = {}
+ for pkg_simple in catalog:
+ # pprint.pprint(pkg_simple)
+ md5 = pkg_simple["md5_sum"]
+ pkg = self.cp.GetPkgstats(md5)
+ if not pkg:
+ logging.warning("No package for %r", md5)
+ continue
+ for dep_pkgname, _ in pkg["depends"]:
+ rev_dep_set = rev_deps.setdefault(dep_pkgname, list())
+ rev_dep_set.append((md5, pkg["basic_stats"]["pkgname"]))
+ sys.stdout.write(".")
+ sys.stdout.flush()
+ sys.stdout.write("\n")
+ self.cached_catalogs[key] = rev_deps
+ with open(fn, "w") as fd:
+ fd.write(cjson.encode(self.cached_catalogs[key]))
+
+ def RevDeps(self, catrel, arch, osrel, md5_sum):
+ self.MakeRevIndex(catrel, arch, osrel)
+ pkg = self.cp.GetPkgstats(md5_sum)
+ pkgname = pkg["basic_stats"]["pkgname"]
+ key = (catrel, arch, osrel)
+ if pkgname in self.cached_catalogs[key]:
+ return self.cached_catalogs[key][pkgname]
+ else:
+ return []
+
+
+class PackageRemover(object):
+
+ def RemovePackage(self, catalogname):
+ # Get md5 sums
+ rest_client = rest.RestClient()
+ rd = RevDeps()
+ rev_deps = {}
+ to_remove = {}
+ for osrel in common_constants.OS_RELS:
+ if osrel in common_constants.OBSOLETE_OS_RELS:
+ continue
+ for arch in common_constants.PHYSICAL_ARCHITECTURES:
+ pkg_simple = rest_client.Srv4ByCatalogAndCatalogname("unstable", arch, osrel, catalogname)
+ md5 = pkg_simple["md5_sum"]
+ pkg = rd.cp.GetPkgstats(md5)
+ key = "unstable", arch, osrel
+ cat_rev_deps = rd.RevDeps("unstable", arch, osrel, md5)
+ if cat_rev_deps:
+ rev_deps[key] = cat_rev_deps
+ f = (
+ "/home/mirror/opencsw/unstable/%s/%s/%s"
+ % (arch, osrel.replace("SunOS", ""), pkg["basic_stats"]["pkg_basename"]))
+ files = to_remove.setdefault(osrel, [])
+ files.append(f)
+ if rev_deps:
+ print "Reverse dependencies found. Bailing out."
+ pprint.pprint(rev_deps)
+ else:
+ for osrel in to_remove:
+ print (
+ "csw-upload-pkg --remove --os-release %s %s"
+ % (osrel, " ".join(to_remove[osrel])))
+
+
+
+def main():
+ parser = optparse.OptionParser()
+ parser.add_option("-c", "--catalogname", dest="catalogname")
+ parser.add_option("--debug", dest="debug", action="store_true")
+ options, args = parser.parse_args()
+ if options.debug:
+ logging.basicConfig(level=logging.DEBUG)
+ pr = PackageRemover()
+ pr.RemovePackage(options.catalogname)
+
+
+if __name__ == '__main__':
+ main()
Property changes on: csw/mgar/gar/v2/lib/python/safe_remove_package.py
___________________________________________________________________
Added: svn:executable
+ *
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
More information about the devel
mailing list