SF.net SVN: gar:[23107] csw/mgar/gar/v2/lib/python
wahwah at users.sourceforge.net
wahwah at users.sourceforge.net
Sun Mar 2 18:52:23 CET 2014
Revision: 23107
http://sourceforge.net/p/gar/code/23107
Author: wahwah
Date: 2014-03-02 17:52:23 +0000 (Sun, 02 Mar 2014)
Log Message:
-----------
find-obsolete-pkgs: Update for the new code
Also fixes for safe_remove_package.py
Modified Paths:
--------------
csw/mgar/gar/v2/lib/python/find_obsolete_pkgs.py
csw/mgar/gar/v2/lib/python/integrate_catalogs.py
csw/mgar/gar/v2/lib/python/rest.py
csw/mgar/gar/v2/lib/python/safe_remove_package.py
Modified: csw/mgar/gar/v2/lib/python/find_obsolete_pkgs.py
===================================================================
--- csw/mgar/gar/v2/lib/python/find_obsolete_pkgs.py 2014-03-02 17:52:11 UTC (rev 23106)
+++ csw/mgar/gar/v2/lib/python/find_obsolete_pkgs.py 2014-03-02 17:52:23 UTC (rev 23107)
@@ -44,8 +44,8 @@
datadir=configuration.CHECKPKG_DIR % os.environ
# fn_revdep = os.path.join(datadir,'RevDeps_%s_%s_%s.json')
fn_cat = os.path.join(datadir,'catalog_%s_%s_%s.json')
-fn_removelst = 'PkgsToRemoveFrom_%s_%s_%s.lst'
-fn_rebuildlst = 'PkgsToRebuildFrom_%s_%s_%s.lst'
+fn_pkgs_to_remove = 'PkgsToRemoveFrom_%s_%s_%s.lst'
+fn_pkgs_to_rebuild = 'PkgsToRebuildFrom_%s_%s_%s.lst'
CatSubSet = namedtuple('CatSubSet',
'pkgname, catalogname, md5_sum, version, dependlist')
@@ -171,8 +171,8 @@
def WriteToTextFiles(pkgs_to_drop, pkgs_to_rebuild, newcatrel, arch, osrel):
- print ('write %s' % (fn_removelst % (newcatrel,osrel,arch)))
- with open(fn_removelst % (newcatrel, osrel, arch), "w") as fd:
+ print ('write %s' % (fn_pkgs_to_remove % (newcatrel,osrel,arch)))
+ with open(fn_pkgs_to_remove % (newcatrel, osrel, arch), "w") as fd:
for pkg in sorted(pkgs_to_drop, key=lambda p: p.catalogname):
fd.write(pkg.catalogname + '\n')
logger.info("number of packages to remove: %d" % len(pkgs_to_drop))
Modified: csw/mgar/gar/v2/lib/python/integrate_catalogs.py
===================================================================
--- csw/mgar/gar/v2/lib/python/integrate_catalogs.py 2014-03-02 17:52:11 UTC (rev 23106)
+++ csw/mgar/gar/v2/lib/python/integrate_catalogs.py 2014-03-02 17:52:23 UTC (rev 23107)
@@ -149,11 +149,7 @@
def GetCatalogs(catrel_from, catrel_to,
include_version_changes,
- include_downgrades):
- config = configuration.GetConfig()
- rest_client = rest.RestClient(
- pkgdb_url=config.get('rest', 'pkgdb'),
- releases_url=config.get('rest', 'releases'))
+ include_downgrades, rest_client):
def GetCatalog(rest_client, r_catrel, r_arch, r_osrel):
key = r_catrel, r_arch, r_osrel
catalog = rest_client.GetCatalog(*key)
@@ -277,6 +273,14 @@
fmt = '%(levelname)s %(asctime)s %(filename)s:%(lineno)d %(message)s'
logging.basicConfig(format=fmt, level=logging_level)
+ config = configuration.GetConfig()
+ username, password = rest.GetUsernameAndPassword()
+ rest_client = rest.RestClient(
+ pkgdb_url=config.get('rest', 'pkgdb'),
+ releases_url=config.get('rest', 'releases'),
+ username=username,
+ password=password)
+
if not options.output_file:
raise UsageError("Please specify the output file. See --help.")
catrel_from = options.catrel_from
@@ -294,18 +298,17 @@
catalogs = GetCatalogs(
catrel_from, catrel_to,
options.include_version_changes,
- options.include_downgrades)
+ options.include_downgrades, rest_client)
diffs_by_catalogname = ComposeDiffsByCatalogname(
catalogs, catrel_from, catrel_to,
options.include_version_changes,
options.include_downgrades)
bundles_by_md5 = {}
bundles_missing = set()
- cp = rest.CachedPkgstats("pkgstats")
+ cp = rest.CachedPkgstats("pkgstats", rest_client)
for key in catalogs:
if catalogs[key]: # could be None
for pkg in catalogs[key]:
- # logging.debug("%r", pkg)
md5 = pkg["md5_sum"]
if md5 not in bundles_by_md5 and md5 not in bundles_missing:
stats = cp.GetPkgstats(md5)
Modified: csw/mgar/gar/v2/lib/python/rest.py
===================================================================
--- csw/mgar/gar/v2/lib/python/rest.py 2014-03-02 17:52:11 UTC (rev 23106)
+++ csw/mgar/gar/v2/lib/python/rest.py 2014-03-02 17:52:23 UTC (rev 23107)
@@ -143,7 +143,7 @@
def RemoveSvr4FromCatalog(self, catrel, arch, osrel, md5_sum):
url = (
- "%s%s/catalogs/%s/%s/%s/%s/"
+ "%s/catalogs/%s/%s/%s/%s/"
% (self.releases_url, catrel, arch, osrel, md5_sum))
logging.debug("DELETE @ URL: %s %s", type(url), url)
c = pycurl.Curl()
@@ -490,13 +490,11 @@
Wraps RestClient and provides a caching layer.
"""
- def __init__(self, filename):
+ def __init__(self, filename, rest_client):
self.filename = filename
self.d = anydbm.open("%s.db" % self.filename, "c")
config = configuration.GetConfig()
- self.rest_client = RestClient(
- pkgdb_url=config.get('rest', 'pkgdb'),
- releases_url=config.get('rest', 'releases'))
+ self.rest_client = rest_client
self.deps = anydbm.open("%s-deps.db" % self.filename, "c")
def __del__(self):
Modified: csw/mgar/gar/v2/lib/python/safe_remove_package.py
===================================================================
--- csw/mgar/gar/v2/lib/python/safe_remove_package.py 2014-03-02 17:52:11 UTC (rev 23106)
+++ csw/mgar/gar/v2/lib/python/safe_remove_package.py 2014-03-02 17:52:23 UTC (rev 23107)
@@ -10,18 +10,18 @@
"""
+import cjson
+import gdbm
+import logging
import optparse
-import rest
-import common_constants
+import os
import pprint
-import gdbm
-import logging
import sys
-import os
-import cjson
import urllib2
from lib.python import configuration
+from lib.python import rest
+from lib.python import common_constants
USAGE = """%prog --os-releases=SunOS5.10,SunOS5.11 -c <catalogname>
@@ -42,7 +42,7 @@
"""
-UNSTABLE = "unstable"
+UNSTABLE = 'unstable'
EVERY_N_DOTS = 100
datadir = configuration.CHECKPKG_DIR % os.environ
fn_revdeps = os.path.join(datadir, 'revdeps-%s-%s-%s.json')
@@ -57,24 +57,15 @@
class RevDeps(object):
- '''
- returns a list of [md5_sum,pkgname]
+ """returns a list of [md5_sum,pkgname]
in the moment not used, perhaps later usefull:
RevDepsSet = namedtuple('RevDepsSet','md5_sum pkgname')
- '''
+ """
- def __init__(self, rest_client=None):
+ def __init__(self, rest_client):
self.cached_catalogs = {}
- self.cp = rest.CachedPkgstats(fn_pkgstatsdb)
+ self.cp = rest.CachedPkgstats(fn_pkgstatsdb, rest_client)
self.rest_client = rest_client
- if self.rest_client is None:
- config = configuration.GetConfig()
- username, password = rest.GetUsernameAndPassword()
- self.rest_client = rest.RestClient(
- pkgdb_url=config.get('rest', 'pkgdb'),
- releases_url=config.get('rest', 'releases'),
- username=username,
- password=password)
def MakeRevIndex(self, catrel, arch, osrel, quiet=False):
key = (catrel, arch, osrel)
@@ -85,6 +76,7 @@
with open(fn, "r") as fd:
self.cached_catalogs[key] = cjson.decode(fd.read())
return
+ # This should be rewritten to use RestClient.GetCatalogForGeneration
logging.info(
"Building a database of reverse dependencies. "
"This can take up to multiple hours.")
@@ -93,7 +85,6 @@
counter = 0
for pkg_simple in catalog:
md5 = pkg_simple["md5_sum"]
- # pkg = self.cp.GetPkgstats(md5)
short_data = self.cp.GetDeps(md5)
pkgname = short_data["pkgname"]
for dep_pkgname, _ in short_data["deps"]:
@@ -126,8 +117,12 @@
else:
return []
+
class PackageRemover(object):
+ def __init__(self, rest_client):
+ self.rest_client = rest_client
+
def CachePackageIsGone(self, catalogname):
with open("packages_dropped_cache.txt", "ab") as fd:
fd.write("{0}\n".format(catalogname))
@@ -136,8 +131,7 @@
if not os_releases:
os_releases = common_constants.OS_RELS
username, password = rest.GetUsernameAndPassword()
- rest_client = rest.RestClient(username=username, password=password)
- rd = RevDeps()
+ rd = RevDeps(self.rest_client)
rev_deps = {}
# md5 sums to remove
to_remove = []
@@ -153,7 +147,8 @@
continue
for arch in common_constants.PHYSICAL_ARCHITECTURES:
try:
- pkg_simple = rest_client.Srv4ByCatalogAndCatalogname(UNSTABLE, arch, osrel, catalogname)
+ pkg_simple = self.rest_client.Srv4ByCatalogAndCatalogname(
+ UNSTABLE, arch, osrel, catalogname)
except urllib2.HTTPError, e:
logging.warning("could not fetch %r from %s/%s: %s",
catalogname, arch, osrel, e)
@@ -161,7 +156,7 @@
if not pkg_simple:
# Maybe we were given a pkgname instead of a catalogname? We can try
# that before failing.
- pkg_simple = rest_client.Srv4ByCatalogAndPkgname(
+ pkg_simple = self.rest_client.Srv4ByCatalogAndPkgname(
UNSTABLE, arch, osrel, catalogname)
if not pkg_simple:
msg = "{0} was not in the unstable {1} {2} catalog."
@@ -170,7 +165,6 @@
if pkg_simple:
found_anywhere = True
md5 = pkg_simple["md5_sum"]
- # pkg = rd.cp.GetPkgstats(md5)
key = UNSTABLE, arch, osrel
cat_rev_deps = rd.RevDepsByMD5(UNSTABLE, arch, osrel, md5)
if cat_rev_deps:
@@ -188,7 +182,7 @@
for catrel, arch, osrel, md5_sum in to_remove:
print "# [%s]" % pkg_simple["catalogname"], catrel, arch, osrel, md5_sum
if execute:
- rest_client.RemoveSvr4FromCatalog(catrel, arch, osrel, md5_sum)
+ self.rest_client.RemoveSvr4FromCatalog(catrel, arch, osrel, md5_sum)
if found_anywhere:
self.CachePackageIsGone(catalogname)
@@ -215,7 +209,16 @@
os_releases = common_constants.OS_RELS
if options.os_releases:
os_releases = options.os_releases.split(",")
- pr = PackageRemover()
+
+ config = configuration.GetConfig()
+ username, password = rest.GetUsernameAndPassword()
+ rest_client = rest.RestClient(
+ pkgdb_url=config.get('rest', 'pkgdb'),
+ releases_url=config.get('rest', 'releases'),
+ username=username,
+ password=password)
+
+ pr = PackageRemover(rest_client)
pr.RemovePackage(options.catalogname, not options.dry_run, os_releases)
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
More information about the devel
mailing list