SF.net SVN: gar:[22212] csw/mgar/gar/v2/lib/python
cgrzemba at users.sourceforge.net
cgrzemba at users.sourceforge.net
Mon Oct 14 16:44:48 CEST 2013
Revision: 22212
http://gar.svn.sourceforge.net/gar/?rev=22212&view=rev
Author: cgrzemba
Date: 2013-10-14 14:44:48 +0000 (Mon, 14 Oct 2013)
Log Message:
-----------
add find_obsolete_pkgs.py, extend interface of safe_remove_package.py
Modified Paths:
--------------
csw/mgar/gar/v2/lib/python/safe_remove_package.py
Added Paths:
-----------
csw/mgar/gar/v2/lib/python/find_obsolete_pkgs.py
Added: csw/mgar/gar/v2/lib/python/find_obsolete_pkgs.py
===================================================================
--- csw/mgar/gar/v2/lib/python/find_obsolete_pkgs.py (rev 0)
+++ csw/mgar/gar/v2/lib/python/find_obsolete_pkgs.py 2013-10-14 14:44:48 UTC (rev 22212)
@@ -0,0 +1,185 @@
+#!/opt/csw/bin/python -t -O
+
+''' Compare two different catalog releases and show:
+ - Which packages need rebuilding (so they don't depend on the _stub any more)
+ - Which _stub packages can be removed
+ - Which packages can declare incompatibility on the old packages, so that the old packages can be removed
+
+ set PYTHONPATH=/path/to/.buildsys/v2
+ alternatively can used: sys.path.append('/path/to/.buildsys/v2')
+
+ ToDO:
+ - read the sub hint from catalog
+ * if stub has consumer -> rebuild
+ * if stub has no consumer and the stub is present in the old/"from" catalog -> remove
+ * if stub has no consumer and the stub does not yet present in the old/"from" catalog -> keep
+'''
+
+import optparse
+import pprint
+import gdbm
+import logging
+import sys
+import os
+import subprocess
+from collections import namedtuple
+import re
+import cjson
+
+from lib.python import rest
+from lib.python import common_constants
+from lib.python import configuration
+from lib.python.safe_remove_package import RevDeps
+
+logging.basicConfig(format='%(asctime)s %(levelname)s:%(message)s')
+logger = logging.getLogger(__name__)
+
+oldcatrel = ''
+newcatrel = ''
+datadir=configuration.CHECKPKG_DIR % os.environ
+# fn_revdep = os.path.join(datadir,'RevDeps_%s_%s_%s.json')
+fn_cat = os.path.join(datadir,'catalog_%s_%s_%s.json')
+fn_removelst = 'PkgsToRemoveFrom_%s_%s_%s.lst'
+fn_rebuildlst = 'PkgsToRebuildFrom_%s_%s_%s.lst'
+revdeplst = {}
+
+CatSubSet = namedtuple('CatSubSet','catalogname md5_sum version dependlist')
+
+class CompCatalog(object):
+
+ def __init__(self,name,arch,osrel):
+ self.rest_client = rest.RestClient()
+ self.catrel = name
+ self.arch = arch
+ self.osrel = osrel
+
+ def __getCat(self, name,arch,osrel):
+ ''' get dependcy list from catalog, read cached list if available '''
+ catlst = {}
+ if os.path.exists(fn_cat % (name,osrel,arch)):
+ logger.info('CompCatalog::getCat: use cached data: %s' % (fn_cat % (name,osrel,arch)))
+ with open(fn_cat % (name,osrel,arch), "r") as fd:
+ catlst = cjson.decode(fd.read())
+ else:
+ cat = self.rest_client.GetCatalog(name,arch,osrel)
+ for pkg in cat:
+ try:
+ pkgitems = self.rest_client.GetPkgstatsByMd5(pkg['md5_sum'])
+ pkgdeplst = [ i[0] for i in pkgitems['depends']]
+ catlst[pkgitems['basic_stats']['pkgname']] = CatSubSet(pkg['catalogname'],pkg['md5_sum'],pkg['version'],pkgdeplst)
+ except Exception as inst:
+ logger.error("CompCatalog::getPkgStat: %s %s %s" , type(inst),pkg['catalogname'],pkg['md5_sum'])
+ with open(fn_cat % (name,osrel,arch), "w") as fd:
+ fd.write(cjson.encode(catlst))
+ logger.info('CompCatalog::getCat: write cache file: %s' % (fn_cat % (name,osrel,arch)))
+ return catlst
+
+ def getCatalog(self):
+ return self.__getCat(self.catrel,self.arch,self.osrel)
+
+def processCat(catrel,arch,osrel):
+ revdeplst = {}
+
+ logger.info("processCat: -> %s %s %s" % (catrel, arch, osrel))
+ cc = CompCatalog(catrel,arch,osrel)
+ catlst = cc.getCatalog()
+ logger.info("processCat: iterate on %s" % (catrel))
+
+ ''' build reverse dependency list '''
+ rd = RevDeps()
+ for p in catlst.keys():
+ revdeplst[p] = rd.RevDepsByPkg(catrel,arch,osrel,p)
+
+ logger.info("processCat: <- %s %s %s" % (catrel, arch, osrel))
+ return catlst, revdeplst
+
+def main():
+ parser = optparse.OptionParser()
+ parser.add_option("--debug", dest="debug", action="store_true")
+ parser.add_option("--verbose", dest="verbose", action="store_true")
+ parser.add_option("--to-catalog-release", dest="newcatalog", default='kiel',
+ help='set name of catalog to fetch', metavar = 'catalogname')
+ parser.add_option("--from-catalog-release", dest="oldcatalog", default='dublin',
+ help='set name of previous (older) catalog to fetch', metavar = 'old catalogname')
+ parser.add_option("--os-arch", dest="arch", default='i386',
+ help='set name of architecture (sparc|i386) to fetch', metavar = 'OS Architecture')
+ parser.add_option("--os-release", dest="osrel", default='SunOS5.10',
+ help='set os release to fetch (SunOS5.10|SunOS5.11)', metavar = 'OS Release')
+ options, args = parser.parse_args()
+ opterror = False
+ if options.verbose:
+ logger.setLevel(logging.INFO)
+ if options.debug:
+ logger.setLevel(logging.DEBUG)
+ if options.newcatalog in common_constants.DEFAULT_CATALOG_RELEASES:
+ newcatrel = options.newcatalog
+ else:
+ logger.error('unknown catalog: %s',options.newcatalog)
+ opterror = True
+ if options.oldcatalog in common_constants.DEFAULT_CATALOG_RELEASES:
+ oldcatrel = options.oldcatalog
+ else:
+ logger.error('unknown catalog: %s',options.newcatalog)
+ opterror = True
+ if options.arch in common_constants.PHYSICAL_ARCHITECTURES:
+ arch = options.arch
+ else:
+ logger.error('unknown architecture: %s',options.arch)
+ opterror = True
+ if options.osrel in common_constants.OS_RELS:
+ osrel = options.osrel
+ else:
+ logger.error('unknown OS Release: %s',options.osrel)
+ opterror = True
+ if opterror:
+ sys.exit(1)
+
+ newcatlst, newrevdeplst = processCat(newcatrel,arch,osrel)
+ oldcatlst, oldrevdeplst = processCat(oldcatrel,arch,osrel)
+
+ to_remove_candidates = []
+ rebuildlst = []
+ logger.debug(' process dependecies in %s' % newcatrel)
+ for p in newrevdeplst.keys():
+ ''' check stub packages '''
+ catalogname = CatSubSet(*newcatlst[p]).catalogname
+ if catalogname.endswith("_stub"):
+ if not newrevdeplst[p]:
+ to_remove_candidates.append(p)
+ logger.debug("{0}({1}) has no consumer".format(p,catalogname))
+ else:
+ for dp in newrevdeplst[p]:
+ dpkg = dp[1]
+ if dpkg not in rebuildlst and not CatSubSet(*newcatlst[dpkg]).catalogname.endswith("_stub"):
+ rebuildlst.append(dpkg)
+ logger.info(" REBUILD: {3}\n\t\t\tthese still use {0} ({1}) in {2}\n"
+ .format(p,CatSubSet(*newcatlst[p]).catalogname,newcatrel,
+ [ dp[1] for dp in newrevdeplst[p]]))
+ reallyremovelst = []
+ logger.debug(' process dependecies in %s' % newcatrel)
+
+ for p in to_remove_candidates:
+ if p in oldrevdeplst: # this package is already a _stub in oldcatalog -> drop
+ reallyremovelst.append(p)
+ logger.info(" DROP : %s from %s" % (p,newcatrel))
+ else:
+ logger.info(" KEEP : {0} not a _stub package in {1}".format(p,oldcatrel))
+
+ print ('write %s' % (fn_removelst % (newcatrel,osrel,arch)))
+ rmcnt = 0
+ with open(fn_removelst % (newcatrel,osrel,arch), "w") as fd:
+ for rp in reallyremovelst:
+ fd.write(CatSubSet(*newcatlst[rp]).catalogname+'\n')
+ rmcnt = rmcnt + 1
+ logger.info("packages to remove: %d" % rmcnt)
+ print ('write %s' % (fn_rebuildlst % (newcatrel,osrel,arch)))
+ rbcnt = 0
+ with open(fn_rebuildlst % (newcatrel,osrel,arch), "w") as fd:
+ for rp in rebuildlst:
+ fd.write(CatSubSet(*newcatlst[rp]).catalogname+'\n')
+ rbcnt = rbcnt + 1
+ logger.info("packages to rebuild: %d" % rbcnt)
+
+
+if __name__ == '__main__':
+ main()
Property changes on: csw/mgar/gar/v2/lib/python/find_obsolete_pkgs.py
___________________________________________________________________
Added: svn:executable
+ *
Modified: csw/mgar/gar/v2/lib/python/safe_remove_package.py
===================================================================
--- csw/mgar/gar/v2/lib/python/safe_remove_package.py 2013-10-14 14:42:06 UTC (rev 22211)
+++ csw/mgar/gar/v2/lib/python/safe_remove_package.py 2013-10-14 14:44:48 UTC (rev 22212)
@@ -21,6 +21,8 @@
import cjson
import urllib2
+from lib.python import configuration
+
USAGE = """%prog --os-releases=SunOS5.10,SunOS5.11 -c <catalogname>
A practical usage example - let's say we have a list of packages to remove in
@@ -42,6 +44,9 @@
UNSTABLE = "unstable"
EVERY_N_DOTS = 100
+datadir = configuration.CHECKPKG_DIR % os.environ
+fn_revdeps = os.path.join(datadir,'revdeps-%s-%s-%s.json')
+fn_pkgstatsdb = os.path.join(datadir,"pkgstats")
class Error(Exception):
"""A generic error."""
@@ -49,18 +54,24 @@
class DataError(Exception):
"""Wrong data encountered."""
+
class RevDeps(object):
+ '''
+ returns a list of [md5_sum,pkgname]
+ in the moment not used, perhaps later usefull:
+ RevDepsSet = namedtuple('RevDepsSet','md5_sum pkgname')
+ '''
def __init__(self):
self.cached_catalogs = {}
self.rest_client = rest.RestClient()
- self.cp = rest.CachedPkgstats("pkgstats")
+ self.cp = rest.CachedPkgstats(fn_pkgstatsdb)
def MakeRevIndex(self, catrel, arch, osrel, quiet=False):
key = (catrel, arch, osrel)
if key in self.cached_catalogs:
return
- fn = "revdeps-%s-%s-%s.json" % key
+ fn = fn_revdeps % key
if os.path.exists(fn):
with open(fn, "r") as fd:
self.cached_catalogs[key] = cjson.decode(fd.read())
@@ -88,7 +99,7 @@
with open(fn, "w") as fd:
fd.write(cjson.encode(self.cached_catalogs[key]))
- def RevDeps(self, catrel, arch, osrel, md5_sum):
+ def RevDepsByMD5(self, catrel, arch, osrel, md5_sum):
self.MakeRevIndex(catrel, arch, osrel)
pkg = self.cp.GetPkgstats(md5_sum)
pkgname = pkg["basic_stats"]["pkgname"]
@@ -98,6 +109,13 @@
else:
return []
+ def RevDepsByPkg(self, catrel, arch, osrel, pkgname):
+ self.MakeRevIndex(catrel, arch, osrel)
+ key = (catrel, arch, osrel)
+ if pkgname in self.cached_catalogs[key]:
+ return self.cached_catalogs[key][pkgname]
+ else:
+ return []
class PackageRemover(object):
@@ -143,9 +161,9 @@
if pkg_simple:
found_anywhere = True
md5 = pkg_simple["md5_sum"]
- pkg = rd.cp.GetPkgstats(md5)
+ # pkg = rd.cp.GetPkgstats(md5)
key = UNSTABLE, arch, osrel
- cat_rev_deps = rd.RevDeps(UNSTABLE, arch, osrel, md5)
+ cat_rev_deps = rd.RevDepsByMD5(UNSTABLE, arch, osrel, md5)
if cat_rev_deps:
rev_deps[key] = cat_rev_deps
to_remove.append((UNSTABLE, arch, osrel, md5))
@@ -168,7 +186,7 @@
def main():
parser = optparse.OptionParser(USAGE)
- parser.add_option("-c", "--catalogname", dest="catalogname")
+ parser.add_option("-c", "--catalogname", dest="catalogname", help='the name of the package in catalog')
parser.add_option("--os-releases", dest="os_releases",
help=("Comma separated OS releases, e.g. "
"SunOS5.9,SunOS5.10"))
@@ -181,7 +199,10 @@
if options.debug:
debug_level = logging.DEBUG
logging.basicConfig(level=debug_level)
- os_relases = None
+ if not options.catalogname:
+ logging.error('option catalogname required \n%s',USAGE)
+ sys.exit(1)
+ os_releases = common_constants.OS_RELS
if options.os_releases:
os_releases = options.os_releases.split(",")
pr = PackageRemover()
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
More information about the devel
mailing list