SF.net SVN: gar:[22212] csw/mgar/gar/v2/lib/python

cgrzemba at users.sourceforge.net cgrzemba at users.sourceforge.net
Mon Oct 14 16:44:48 CEST 2013


Revision: 22212
          http://gar.svn.sourceforge.net/gar/?rev=22212&view=rev
Author:   cgrzemba
Date:     2013-10-14 14:44:48 +0000 (Mon, 14 Oct 2013)
Log Message:
-----------
add find_obsolete_pkgs.py, extend interface of safe_remove_package.py

Modified Paths:
--------------
    csw/mgar/gar/v2/lib/python/safe_remove_package.py

Added Paths:
-----------
    csw/mgar/gar/v2/lib/python/find_obsolete_pkgs.py

Added: csw/mgar/gar/v2/lib/python/find_obsolete_pkgs.py
===================================================================
--- csw/mgar/gar/v2/lib/python/find_obsolete_pkgs.py	                        (rev 0)
+++ csw/mgar/gar/v2/lib/python/find_obsolete_pkgs.py	2013-10-14 14:44:48 UTC (rev 22212)
@@ -0,0 +1,185 @@
+#!/opt/csw/bin/python -t -O
+
+''' Compare two different catalog releases and show:
+    - Which packages need rebuilding (so they don't depend on the _stub any more)
+    - Which _stub packages can be removed
+    - Which packages can declare incompatibility on the old packages, so that the old packages can be removed
+    
+    set PYTHONPATH=/path/to/.buildsys/v2
+    alternatively can used: sys.path.append('/path/to/.buildsys/v2')
+
+    ToDO:
+    - read the sub hint from catalog
+      * if stub has consumer -> rebuild
+      * if stub has no consumer and the stub is present in the old/"from" catalog -> remove
+      * if stub has no consumer and the stub does not yet present in the old/"from" catalog -> keep      
+'''
+
+import optparse
+import pprint
+import gdbm
+import logging
+import sys
+import os
+import subprocess
+from collections import namedtuple
+import re
+import cjson
+
+from lib.python import rest
+from lib.python import common_constants
+from lib.python import configuration
+from lib.python.safe_remove_package import RevDeps
+
+logging.basicConfig(format='%(asctime)s %(levelname)s:%(message)s')
+logger = logging.getLogger(__name__)
+
+oldcatrel = ''
+newcatrel = ''
+datadir=configuration.CHECKPKG_DIR % os.environ
+# fn_revdep = os.path.join(datadir,'RevDeps_%s_%s_%s.json')
+fn_cat = os.path.join(datadir,'catalog_%s_%s_%s.json')
+fn_removelst = 'PkgsToRemoveFrom_%s_%s_%s.lst'
+fn_rebuildlst = 'PkgsToRebuildFrom_%s_%s_%s.lst'
+revdeplst = {}
+    
+CatSubSet = namedtuple('CatSubSet','catalogname md5_sum version dependlist')
+
+class CompCatalog(object):
+
+    def __init__(self,name,arch,osrel):
+        self.rest_client = rest.RestClient()
+        self.catrel = name
+        self.arch = arch
+        self.osrel = osrel
+      
+    def __getCat(self, name,arch,osrel):
+        ''' get dependcy list from catalog, read cached list if available '''
+        catlst = {}
+        if os.path.exists(fn_cat % (name,osrel,arch)):
+            logger.info('CompCatalog::getCat: use cached data: %s' % (fn_cat % (name,osrel,arch)))
+            with open(fn_cat % (name,osrel,arch), "r") as fd:
+                catlst = cjson.decode(fd.read())
+        else:
+            cat = self.rest_client.GetCatalog(name,arch,osrel)
+            for pkg in cat:
+                try:
+                    pkgitems = self.rest_client.GetPkgstatsByMd5(pkg['md5_sum'])
+                    pkgdeplst = [ i[0] for i in pkgitems['depends']]
+                    catlst[pkgitems['basic_stats']['pkgname']] = CatSubSet(pkg['catalogname'],pkg['md5_sum'],pkg['version'],pkgdeplst)
+                except Exception as inst:
+                    logger.error("CompCatalog::getPkgStat: %s %s %s" , type(inst),pkg['catalogname'],pkg['md5_sum'])
+            with open(fn_cat % (name,osrel,arch), "w") as fd:
+                fd.write(cjson.encode(catlst))
+                logger.info('CompCatalog::getCat: write cache file: %s' % (fn_cat % (name,osrel,arch)))
+        return catlst
+    
+    def getCatalog(self):
+        return self.__getCat(self.catrel,self.arch,self.osrel)
+
+def processCat(catrel,arch,osrel):
+    revdeplst = {}
+    
+    logger.info("processCat: -> %s %s %s" % (catrel, arch, osrel))
+    cc = CompCatalog(catrel,arch,osrel)
+    catlst = cc.getCatalog()
+    logger.info("processCat: iterate on %s" % (catrel))
+    
+    ''' build reverse dependency list '''
+    rd = RevDeps()
+    for p in catlst.keys():
+        revdeplst[p] = rd.RevDepsByPkg(catrel,arch,osrel,p)
+    
+    logger.info("processCat: <- %s %s %s" % (catrel, arch, osrel))
+    return catlst, revdeplst
+  
+def main():
+    parser = optparse.OptionParser()
+    parser.add_option("--debug", dest="debug", action="store_true")
+    parser.add_option("--verbose", dest="verbose", action="store_true")
+    parser.add_option("--to-catalog-release", dest="newcatalog", default='kiel', 
+                    help='set name of catalog to fetch', metavar = 'catalogname')
+    parser.add_option("--from-catalog-release", dest="oldcatalog", default='dublin', 
+                    help='set name of previous (older) catalog to fetch', metavar = 'old catalogname')
+    parser.add_option("--os-arch", dest="arch", default='i386', 
+                    help='set name of architecture (sparc|i386) to fetch', metavar = 'OS Architecture')
+    parser.add_option("--os-release", dest="osrel", default='SunOS5.10', 
+                    help='set os release to fetch (SunOS5.10|SunOS5.11)', metavar = 'OS Release')
+    options, args = parser.parse_args()
+    opterror = False
+    if options.verbose:
+        logger.setLevel(logging.INFO)
+    if options.debug:
+        logger.setLevel(logging.DEBUG)
+    if options.newcatalog in common_constants.DEFAULT_CATALOG_RELEASES:
+        newcatrel = options.newcatalog
+    else:
+        logger.error('unknown catalog: %s',options.newcatalog)
+        opterror = True
+    if options.oldcatalog in common_constants.DEFAULT_CATALOG_RELEASES:
+        oldcatrel = options.oldcatalog
+    else:
+        logger.error('unknown catalog: %s',options.newcatalog)
+        opterror = True
+    if options.arch in common_constants.PHYSICAL_ARCHITECTURES:
+        arch      = options.arch
+    else:
+        logger.error('unknown architecture: %s',options.arch)
+        opterror = True
+    if options.osrel in common_constants.OS_RELS: 
+        osrel     = options.osrel
+    else:
+        logger.error('unknown OS Release: %s',options.osrel)
+        opterror = True
+    if opterror:
+        sys.exit(1)
+    
+    newcatlst, newrevdeplst = processCat(newcatrel,arch,osrel)
+    oldcatlst, oldrevdeplst = processCat(oldcatrel,arch,osrel)
+    
+    to_remove_candidates = []
+    rebuildlst = []
+    logger.debug(' process dependecies in %s' % newcatrel)
+    for p in newrevdeplst.keys():
+        ''' check stub packages '''
+        catalogname = CatSubSet(*newcatlst[p]).catalogname
+        if catalogname.endswith("_stub"):
+            if not newrevdeplst[p]:
+                to_remove_candidates.append(p)
+                logger.debug("{0}({1}) has no consumer".format(p,catalogname))
+            else:
+                for dp in newrevdeplst[p]:
+                    dpkg = dp[1]
+                    if dpkg not in rebuildlst and not CatSubSet(*newcatlst[dpkg]).catalogname.endswith("_stub"):
+                          rebuildlst.append(dpkg)
+                          logger.info(" REBUILD: {3}\n\t\t\tthese still use {0} ({1}) in {2}\n"
+                                .format(p,CatSubSet(*newcatlst[p]).catalogname,newcatrel,
+                                        [ dp[1] for dp in newrevdeplst[p]]))
+    reallyremovelst = []
+    logger.debug(' process dependecies in %s' % newcatrel)
+
+    for p in to_remove_candidates:
+        if p in oldrevdeplst: # this package is already a _stub in oldcatalog -> drop
+            reallyremovelst.append(p)
+            logger.info(" DROP   : %s from %s" % (p,newcatrel))
+        else:
+            logger.info(" KEEP   : {0} not a _stub package in {1}".format(p,oldcatrel))
+    
+    print ('write %s' % (fn_removelst % (newcatrel,osrel,arch))) 
+    rmcnt = 0
+    with open(fn_removelst % (newcatrel,osrel,arch), "w") as fd:
+        for rp in reallyremovelst:
+            fd.write(CatSubSet(*newcatlst[rp]).catalogname+'\n')
+            rmcnt = rmcnt + 1
+    logger.info("packages to remove: %d" % rmcnt)
+    print ('write %s' % (fn_rebuildlst % (newcatrel,osrel,arch))) 
+    rbcnt = 0
+    with open(fn_rebuildlst % (newcatrel,osrel,arch), "w") as fd:
+        for rp in rebuildlst:
+            fd.write(CatSubSet(*newcatlst[rp]).catalogname+'\n')
+            rbcnt = rbcnt + 1
+    logger.info("packages to rebuild: %d" % rbcnt)
+
+
+if __name__ == '__main__':
+    main()


Property changes on: csw/mgar/gar/v2/lib/python/find_obsolete_pkgs.py
___________________________________________________________________
Added: svn:executable
   + *

Modified: csw/mgar/gar/v2/lib/python/safe_remove_package.py
===================================================================
--- csw/mgar/gar/v2/lib/python/safe_remove_package.py	2013-10-14 14:42:06 UTC (rev 22211)
+++ csw/mgar/gar/v2/lib/python/safe_remove_package.py	2013-10-14 14:44:48 UTC (rev 22212)
@@ -21,6 +21,8 @@
 import cjson
 import urllib2
 
+from lib.python import configuration
+
 USAGE = """%prog --os-releases=SunOS5.10,SunOS5.11 -c <catalogname>
 
 A practical usage example - let's say we have a list of packages to remove in
@@ -42,6 +44,9 @@
 
 UNSTABLE = "unstable"
 EVERY_N_DOTS = 100
+datadir = configuration.CHECKPKG_DIR % os.environ
+fn_revdeps = os.path.join(datadir,'revdeps-%s-%s-%s.json')
+fn_pkgstatsdb = os.path.join(datadir,"pkgstats")
 
 class Error(Exception):
   """A generic error."""
@@ -49,18 +54,24 @@
 class DataError(Exception):
   """Wrong data encountered."""
 
+
 class RevDeps(object):
+  ''' 
+  returns a list of [md5_sum,pkgname]
+  in the moment not used, perhaps later usefull:
+    RevDepsSet = namedtuple('RevDepsSet','md5_sum pkgname')
+  '''
 
   def __init__(self):
     self.cached_catalogs = {}
     self.rest_client = rest.RestClient()
-    self.cp = rest.CachedPkgstats("pkgstats")
+    self.cp = rest.CachedPkgstats(fn_pkgstatsdb)
 
   def MakeRevIndex(self, catrel, arch, osrel, quiet=False):
     key = (catrel, arch, osrel)
     if key in self.cached_catalogs:
       return
-    fn = "revdeps-%s-%s-%s.json" % key
+    fn = fn_revdeps % key
     if os.path.exists(fn):
       with open(fn, "r") as fd:
         self.cached_catalogs[key] = cjson.decode(fd.read())
@@ -88,7 +99,7 @@
     with open(fn, "w") as fd:
       fd.write(cjson.encode(self.cached_catalogs[key]))
 
-  def RevDeps(self, catrel, arch, osrel, md5_sum):
+  def RevDepsByMD5(self, catrel, arch, osrel, md5_sum):
     self.MakeRevIndex(catrel, arch, osrel)
     pkg = self.cp.GetPkgstats(md5_sum)
     pkgname = pkg["basic_stats"]["pkgname"]
@@ -98,6 +109,13 @@
     else:
       return []
 
+  def RevDepsByPkg(self, catrel, arch, osrel, pkgname):
+    self.MakeRevIndex(catrel, arch, osrel)
+    key = (catrel, arch, osrel)
+    if pkgname in self.cached_catalogs[key]:
+      return self.cached_catalogs[key][pkgname]
+    else:
+      return []
 
 class PackageRemover(object):
 
@@ -143,9 +161,9 @@
         if pkg_simple:
           found_anywhere = True
         md5 = pkg_simple["md5_sum"]
-        pkg = rd.cp.GetPkgstats(md5)
+        # pkg = rd.cp.GetPkgstats(md5)
         key = UNSTABLE, arch, osrel
-        cat_rev_deps = rd.RevDeps(UNSTABLE, arch, osrel, md5)
+        cat_rev_deps = rd.RevDepsByMD5(UNSTABLE, arch, osrel, md5)
         if cat_rev_deps:
           rev_deps[key] = cat_rev_deps
         to_remove.append((UNSTABLE, arch, osrel, md5))
@@ -168,7 +186,7 @@
 
 def main():
   parser = optparse.OptionParser(USAGE)
-  parser.add_option("-c", "--catalogname", dest="catalogname")
+  parser.add_option("-c", "--catalogname", dest="catalogname", help='the name of the package in catalog')
   parser.add_option("--os-releases", dest="os_releases",
                     help=("Comma separated OS releases, e.g. "
                           "SunOS5.9,SunOS5.10"))
@@ -181,7 +199,10 @@
   if options.debug:
     debug_level = logging.DEBUG
   logging.basicConfig(level=debug_level)
-  os_relases = None
+  if not options.catalogname:
+    logging.error('option catalogname required \n%s',USAGE)
+    sys.exit(1)
+  os_releases = common_constants.OS_RELS
   if options.os_releases:
     os_releases = options.os_releases.split(",")
   pr = PackageRemover()

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.



More information about the devel mailing list