[csw-devel] SF.net SVN: gar:[7593] csw/mgar/pkg/pkgutil/trunk
bonivart at users.sourceforge.net
bonivart at users.sourceforge.net
Thu Dec 10 21:02:51 CET 2009
Revision: 7593
http://gar.svn.sourceforge.net/gar/?rev=7593&view=rev
Author: bonivart
Date: 2009-12-10 20:02:50 +0000 (Thu, 10 Dec 2009)
Log Message:
-----------
pkgutil: source update, update Makefile
Modified Paths:
--------------
csw/mgar/pkg/pkgutil/trunk/Makefile
csw/mgar/pkg/pkgutil/trunk/checksums
Added Paths:
-----------
csw/mgar/pkg/pkgutil/trunk/files/build_sun_catalog.py
csw/mgar/pkg/pkgutil/trunk/files/opencsw.py
Modified: csw/mgar/pkg/pkgutil/trunk/Makefile
===================================================================
--- csw/mgar/pkg/pkgutil/trunk/Makefile 2009-12-10 19:20:17 UTC (rev 7592)
+++ csw/mgar/pkg/pkgutil/trunk/Makefile 2009-12-10 20:02:50 UTC (rev 7593)
@@ -14,13 +14,12 @@
COMMON_PKG_DEPENDS =
MASTER_SITES = $(SF_MIRRORS)
-MASTER_SITES += svn-http://gar.svn.sourceforge.net/svnroot/opencsw/
DISTFILES = $(GARNAME)-$(GARVERSION).zip
DISTFILES += $(call admfiles,CSWpkgutil,prototype)
DISTFILES += i.cswpkgutil
DISTFILES += r.cswpkgutil
-DISTFILES += utilities
-NOCHECKSUM = utilities
+DISTFILES += build_sun_catalog.py
+DISTFILES += opencsw.py
PACKAGES = CSWpkgutil CSWpkgutilplus
@@ -57,19 +56,18 @@
@echo " ==> Installing $(GARNAME) (custom)"
@rm -rf $(DESTDIR)
@ginstall -m 755 -d $(DESTDIR)/etc/opt/csw/pkg-hooks/preargproc.d
- @ginstall $(WORKSRC)/$(GARNAME).conf $(DESTDIR)/etc/opt/csw/$(GARNAME).conf.CSW
+ @ginstall -m 644 $(WORKSRC)/$(GARNAME).conf $(DESTDIR)/etc/opt/csw/$(GARNAME).conf.CSW
@ginstall -m 755 $(WORKSRC)/pkgutillog $(DESTDIR)/etc/opt/csw/pkg-hooks/preargproc.d/01-CSW$(GARNAME)plus-log
@ginstall -m 755 -d $(DESTDIR)$(bindir)
- @ginstall $(WORKSRC)/$(GARNAME) $(WORKSRC)/bldcat $(WORKSRC)/chkcat $(DESTDIR)$(bindir)
- @ginstall -m 755 -d $(DESTDIR)$(sysconfdir)
- @ginstall $(WORKSRC)/$(GARNAME).conf $(DESTDIR)$(sysconfdir)/$(GARNAME).conf.CSW
+ @ginstall -m 755 $(WORKSRC)/$(GARNAME) $(WORKSRC)/bldcat $(WORKSRC)/chkcat $(DESTDIR)$(bindir)
+ @ginstall -m 755 -d $(DESTDIR)/opt/csw/etc
+ @ginstall -m 644 $(WORKSRC)/$(GARNAME).conf $(DESTDIR)/opt/csw/etc/$(GARNAME).conf.CSW
@ginstall -m 755 -d $(DESTDIR)/opt/csw/libexec/$(GARNAME)
- @ginstall $(WORKSRC)/wget-`uname -p` $(DESTDIR)/opt/csw/libexec/$(GARNAME)/wget
- @ginstall $(WORKSRC)/utilities/build_sun_catalog.py $(DESTDIR)/opt/csw/libexec/$(GARNAME)
- @ginstall $(WORKSRC)/utilities/opencsw.py $(DESTDIR)/opt/csw/libexec/$(GARNAME)
+ @ginstall -m 755 $(WORKSRC)/wget-`uname -p` $(DESTDIR)/opt/csw/libexec/$(GARNAME)/wget
+ @ginstall -m 755 $(FILEDIR)/build_sun_catalog.py $(FILEDIR)/opencsw.py $(DESTDIR)/opt/csw/libexec/$(GARNAME)
@ln -s ../libexec/pkgutil/build_sun_catalog.py $(DESTDIR)$(bindir)/build_sun_catalog
@ginstall -m 755 -d $(DESTDIR)$(docdir)/$(GARNAME)
- @ginstall $(WORKSRC)/readme $(WORKSRC)/license $(DESTDIR)$(docdir)/$(GARNAME)/
+ @ginstall -m 444 $(WORKSRC)/readme $(WORKSRC)/license $(DESTDIR)$(docdir)/$(GARNAME)/
@ginstall -m 755 -d $(DESTDIR)$(docdir)/$(GARNAME)plus
@ginstall -m 444 $(WORKSRC)/license $(DESTDIR)$(docdir)/$(GARNAME)plus/
@ginstall -m 444 $(WORKSRC)/readme.pkgutilplus $(DESTDIR)$(docdir)/$(GARNAME)plus/readme
@@ -80,5 +78,5 @@
@chmod 444 $(DESTDIR)$(mandir)/man1/*
@ginstall -m 755 -d $(DESTDIR)/var/opt/csw/$(GARNAME)/packages
@ginstall -m 755 -d $(DESTDIR)/var/opt/csw/$(GARNAME)/pkgask
- @ginstall $(WORKSRC)/admin $(DESTDIR)/var/opt/csw/$(GARNAME)/admin.CSW
+ @ginstall -m 644 $(WORKSRC)/admin $(DESTDIR)/var/opt/csw/$(GARNAME)/admin.CSW
@$(MAKECOOKIE)
Modified: csw/mgar/pkg/pkgutil/trunk/checksums
===================================================================
--- csw/mgar/pkg/pkgutil/trunk/checksums 2009-12-10 19:20:17 UTC (rev 7592)
+++ csw/mgar/pkg/pkgutil/trunk/checksums 2009-12-10 20:02:50 UTC (rev 7593)
@@ -1,5 +1,7 @@
a16c6f81bc537d6172fd65d8da1aeecc CSWpkgutil.gspec
a751314ee495d7477888522435b769e4 CSWpkgutil.prototype
+b5fffa0be100ddbb7262ce3a0a4e3bb7 build_sun_catalog.py
33f82561e1fcf3aaf22f66bac22afba1 i.cswpkgutil
-a52020816f3dda83d32621f85321632a pkgutil-1.9.1.zip
+3a6b789b3d5e05f41d2363dd26a92acf opencsw.py
+d65a85a929f3901d4a613006e7c2c7bd pkgutil-1.9.1.zip
6d472d94ea850b600fd29aba7a6fc4b3 r.cswpkgutil
Added: csw/mgar/pkg/pkgutil/trunk/files/build_sun_catalog.py
===================================================================
--- csw/mgar/pkg/pkgutil/trunk/files/build_sun_catalog.py (rev 0)
+++ csw/mgar/pkg/pkgutil/trunk/files/build_sun_catalog.py 2009-12-10 20:02:50 UTC (rev 7593)
@@ -0,0 +1,42 @@
+#!/opt/csw/bin/python2.6
+# coding=utf-8
+"""Builds a SUNW catalog in the OpenCSW format.
+
+$Id: build_sun_catalog.py 107 2009-12-08 18:32:37Z wahwah $
+"""
+
+__author__ = "Maciej Bliziński <blizinski at google.com>"
+
+import os
+import os.path
+import logging
+import opencsw
+import optparse
+import shutil
+import subprocess
+import tempfile
+
+
+def main():
+ parser = optparse.OptionParser()
+ parser.add_option("-p", "--product", dest="product_dir",
+ help="Product dir, e.g. .../Solaris_10/Product")
+ parser.add_option("-c", "--catalog-dir", dest="catalog_dir",
+ help="Target catalog dir")
+ options, args = parser.parse_args()
+ logging.basicConfig(level=logging.DEBUG)
+ options_ok = True
+ if not options.product_dir:
+ logging.error("--product directory is required. See --help.")
+ options_ok = False
+ if not options.catalog_dir:
+ logging.error("--catalog-dir is required. See --help.")
+ options_ok = False
+ if options_ok:
+ b = opencsw.OpencswCatalogBuilder(options.product_dir,
+ options.catalog_dir)
+ b.Run()
+
+
+if __name__ == "__main__":
+ main()
Property changes on: csw/mgar/pkg/pkgutil/trunk/files/build_sun_catalog.py
___________________________________________________________________
Added: svn:executable
+ *
Added: csw/mgar/pkg/pkgutil/trunk/files/opencsw.py
===================================================================
--- csw/mgar/pkg/pkgutil/trunk/files/opencsw.py (rev 0)
+++ csw/mgar/pkg/pkgutil/trunk/files/opencsw.py 2009-12-10 20:02:50 UTC (rev 7593)
@@ -0,0 +1,680 @@
+#!/opt/csw/bin/python2.6
+# coding=utf-8
+#
+# $Id: opencsw.py 107 2009-12-08 18:32:37Z wahwah $
+#
+# vim:set sw=2 ts=2 sts=2 expandtab:
+#
+# Copyright (c) 2009 OpenCSW
+# Author: Maciej Bliziński
+#
+# $Id: opencsw.py 107 2009-12-08 18:32:37Z wahwah $
+#
+# This program is free software; you can redistribute it and/or modify it under
+# the terms of the GNU General Public License version 2 as published by the
+# Free Software Foundation.
+
+import datetime
+import difflib
+import logging
+import os
+import os.path
+import re
+import shutil
+import subprocess
+import tempfile
+import urllib2
+
+ARCHITECTURES = ["i386", "sparc", "all"]
+MAJOR_VERSION = "major version"
+MINOR_VERSION = "minor version"
+PATCHLEVEL = "patchlevel"
+REVISION = "revision"
+OTHER_VERSION_INFO = "other version info"
+NEW_PACKAGE = "new package"
+NO_VERSION_CHANGE = "no version change"
+PKG_URL_TMPL = "http://www.opencsw.org/packages/%s"
+CATALOG_URL = "http://ftp.heanet.ie/pub/opencsw/current/i386/5.10/catalog"
+ADMIN_FILE_CONTENT = """
+basedir=default
+runlevel=nocheck
+conflict=nocheck
+setuid=nocheck
+action=nocheck
+partial=nocheck
+instance=unique
+idepend=quit
+rdepend=quit
+space=quit
+authentication=nocheck
+networktimeout=10
+networkretries=5
+keystore=/var/sadm/security
+proxy=
+"""
+EMAIL_TMPL = """From: %(from)s
+To: %(to)s
+Cc: %(cc)s
+Date: %(date)s
+Subject: newpkgs %(pkgnames)s
+
+%(body)s
+
+--
+$Id: opencsw.py 107 2009-12-08 18:32:37Z wahwah $
+"""
+
+
+class Error(Exception):
+ pass
+
+
+class PackageError(Error):
+ pass
+
+
+def ParsePackageFileName(p):
+ bits = p.split("-")
+ catalogname = bits[0]
+ version, version_info, revision_info = ParseVersionString(bits[1])
+ data = {
+ 'catalogname': catalogname,
+ 'full_version_string': bits[1],
+ 'version': version,
+ 'version_info': version_info,
+ 'revision_info': revision_info,
+ }
+ return data
+
+def ParseVersionString(s):
+ version_bits = re.split("_|,", s)
+ version_str = version_bits[0]
+ revision_bits = version_bits[1:]
+ revision_info = {}
+ version_info = {}
+ version_number_bits = version_str.split(".")
+ version_info[MAJOR_VERSION] = version_number_bits[0]
+ if len(version_number_bits) >= 2:
+ version_info[MINOR_VERSION] = version_number_bits[1]
+ if len(version_number_bits) >= 3:
+ version_info[PATCHLEVEL] = version_number_bits[2]
+ for version_bit in revision_bits:
+ if "=" in version_bit:
+ (var_name, var_value) = version_bit.split("=")
+ revision_info[var_name] = var_value
+ else:
+ if not "extra_strings" in revision_info:
+ revision_info["extra_strings"] = []
+ revision_info["extra_strings"].append(version_bit)
+ return version_str, version_info, revision_info
+
+
+class OpencswPackage(object):
+
+ catalog_downloaded = False
+
+ def __init__(self, catalogname):
+ self.catalogname = catalogname
+
+ def IsOnTheWeb(self):
+ url = PKG_URL_TMPL % self.catalogname
+ logging.debug("Opening %s", repr(url))
+ package_page = urllib2.urlopen(url)
+ html = package_page.read()
+ # Since beautifulsoup is not installed on the buildfarm yet, a quirk in
+ # package page display is going to be used.
+ package_not_in_mantis_pattern = u"cannot find maintainer"
+ return html.find(package_not_in_mantis_pattern) >= 0
+
+ def IsNew(self):
+ return not self.IsOnTheWeb()
+
+ def UpgradeType(self, new_version_string):
+ """What kind of upgrade would it be if the new version was X."""
+ (new_version,
+ new_version_info,
+ new_revision_info) = ParseVersionString(new_version_string)
+ catalog_data = self.GetCatalogPkgData()
+ if not catalog_data:
+ return (NEW_PACKAGE,
+ "/dev/null -> %s" % new_version_string,
+ (None, new_version_string))
+ cat_version_info = catalog_data["version_info"]
+ levels = (MAJOR_VERSION, MINOR_VERSION, PATCHLEVEL)
+ for level in levels:
+ if level in cat_version_info and level in new_version_info:
+ if (cat_version_info[level] != new_version_info[level]):
+ versions = (catalog_data["full_version_string"], new_version_string)
+ msg = "%s --> %s" % versions
+ return level, msg, versions
+ cat_rev_info = catalog_data["revision_info"]
+ for rev_kw in cat_rev_info:
+ if rev_kw in new_revision_info:
+ if cat_rev_info[rev_kw] != new_revision_info[rev_kw]:
+ msg = "%s: %s --> %s" % (rev_kw,
+ cat_rev_info[rev_kw],
+ new_revision_info[rev_kw])
+ versions = cat_rev_info[rev_kw], new_revision_info[rev_kw]
+ return REVISION, msg, versions
+ if (catalog_data["version"] == new_version
+ and
+ catalog_data["revision_info"] == new_revision_info):
+ return NO_VERSION_CHANGE, "no", (new_version, new_version)
+ return OTHER_VERSION_INFO, "other", (None, None)
+
+ @classmethod
+ def LazyDownloadCatalogData(cls, catalog_source=None):
+ if not cls.catalog_downloaded:
+ cls.DownloadCatalogData(catalog_source)
+ cls.catalog_downloaded = True
+
+ @classmethod
+ def DownloadCatalogData(cls, catalog_source):
+ """Downloads catalog data."""
+ logging.debug("Downloading catalog data from %s.", repr(CATALOG_URL))
+ if not catalog_source:
+ catalog_source = urllib2.urlopen(CATALOG_URL)
+ cls.catalog = {}
+ for line in catalog_source:
+ # Working around the GPG signature
+ if line.startswith("#"): continue
+ if line.startswith("-----BEGIN PGP SIGNED MESSAGE-----"): continue
+ if line.startswith("Hash:"): continue
+ if len(line.strip()) <= 0: continue
+ if line.startswith("-----BEGIN PGP SIGNATURE-----"): break
+ fields = re.split(r"\s+", line)
+ try:
+ cls.catalog[fields[0]] = ParsePackageFileName(fields[3])
+ except IndexError, e:
+ print repr(line)
+ print fields
+ print e
+ raise
+
+ @classmethod
+ def _GetCatalogPkgData(cls, catalogname):
+ cls.LazyDownloadCatalogData()
+ if catalogname in cls.catalog:
+ return cls.catalog[catalogname]
+ else:
+ return None
+
+ def GetCatalogPkgData(self):
+ """A wrapper for the classmethod _GetCatalogPkgData, supplying the catalogname."""
+ return self._GetCatalogPkgData(self.catalogname)
+
+
+class StagingDir(object):
+ """Represents a staging directory."""
+
+ def __init__(self, dir_path):
+ self.dir_path = dir_path
+
+ def GetLatest(self, software, architectures=ARCHITECTURES):
+ files = os.listdir(self.dir_path)
+ package_files = []
+ for a in architectures:
+ relevant_pkgs = sorted(shutil.fnmatch.filter(files,
+ "*-%s-*.pkg.gz" % a))
+ relevant_pkgs = sorted(shutil.fnmatch.filter(relevant_pkgs,
+ "%s-*.pkg.gz" % software))
+ if relevant_pkgs:
+ package_files.append(relevant_pkgs[-1])
+ if not package_files:
+ raise PackageError("Could not find %s in %s" % (software, self.dir_path))
+ logging.debug("The latest packages %s in %s are %s",
+ repr(software),
+ repr(self.dir_path),
+ repr(package_files))
+ return [os.path.join(self.dir_path, x) for x in package_files]
+
+
+class NewpkgMailer(object):
+
+ def __init__(self, pkgnames, paths,
+ release_mgr_name,
+ release_mgr_email,
+ sender_name,
+ sender_email,
+ release_cc):
+ self.sender = u"%s <%s>" % (sender_name, sender_email)
+ self.pkgnames = pkgnames
+ self.paths = paths
+ self.release_mgr = u"%s <%s>" % (release_mgr_name, release_mgr_email)
+ self.release_cc = u"%s" % release_cc
+
+ def FormatPackageAnnouncement(self, catalogname, new_pkgpath):
+ """TODO: Group common version upgrades."""
+ pkg = OpencswPackage(catalogname)
+ new_pkg_basename = os.path.split(new_pkgpath)[1]
+ new_data = ParsePackageFileName(new_pkg_basename)
+ new_version_str = new_data["full_version_string"]
+ upgrade_type, upgrade_msg = pkg.UpgradeType(new_version_str)
+ msg = []
+ if upgrade_type == NEW_PACKAGE:
+ msg.append("* new package")
+ else:
+ msg.append("* %s upgrade" % upgrade_type)
+ msg.append(" - %s" % upgrade_msg)
+ msg.append(" - %s" % new_pkgpath)
+ return msg
+
+ def FormatMail(self):
+ body_list = ["The following package files are ready to be released:"]
+ body_list.append("")
+ # Gathering package information, grouping packages that are upgraded
+ # together.
+ pkgs_data = {}
+ for p in self.paths:
+ base_file_name = os.path.split(p)[1]
+ catalogname = base_file_name.split("-")[0]
+ pkg = OpencswPackage(catalogname)
+ new_data = ParsePackageFileName(base_file_name)
+ new_version_str = new_data["full_version_string"]
+ catalog_data = pkg.GetCatalogPkgData()
+ if catalog_data:
+ catalog_version_str = catalog_data["full_version_string"]
+ else:
+ catalog_version_str = "package not in the catalog"
+ upgrade_type, upgrade_msg, versions = pkg.UpgradeType(new_version_str)
+ pkgs_data_key = (upgrade_type, upgrade_msg, versions)
+ if pkgs_data_key not in pkgs_data:
+ pkgs_data[pkgs_data_key] = []
+ pkg.srv4path = p
+ pkg.cat_version_str = catalog_version_str
+ pkg.new_version_str = new_version_str
+ pkgs_data[pkgs_data_key].append(pkg)
+ # Formatting grouped packages:
+ for upgrade_type, upgrade_msg, versions in pkgs_data:
+ msg = []
+ if upgrade_type == NEW_PACKAGE:
+ msg.append("* %s: %s" % (upgrade_type, upgrade_msg))
+ elif upgrade_type == NO_VERSION_CHANGE:
+ msg.append("* WARNING: no version change")
+ else:
+ msg.append("* %s upgrade" % (upgrade_type))
+ msg.append(" - from: %s" % versions[0])
+ msg.append(" - to: %s" % versions[1])
+ for pkg in pkgs_data[(upgrade_type, upgrade_msg, versions)]:
+ msg.append(" + %s" % pkg.srv4path)
+ body_list.extend(msg)
+ body_list.append("")
+ body = "\n".join(body_list)
+ d = {
+ 'from': self.sender,
+ 'to': self.release_mgr,
+ 'cc': self.release_cc,
+ 'pkgnames': ", ".join(self.pkgnames),
+ 'body': body,
+ 'date': datetime.datetime.now(),
+ }
+ mail_text = EMAIL_TMPL % d
+ return mail_text
+
+ def GetEditorName(self, env):
+ editor = "/opt/csw/bin/vim"
+ if "EDITOR" in env:
+ editor = env["EDITOR"]
+ if "VISUAL" in env:
+ editor = env["VISUAL"]
+ return editor
+
+
+class ShellMixin(object):
+
+ def ShellCommand(self, args, quiet=False):
+ logging.debug("Calling: %s", repr(args))
+ if quiet:
+ sub_stdout = subprocess.PIPE
+ sub_stderr = subprocess.PIPE
+ else:
+ sub_stdout = None
+ sub_stderr = None
+ retcode = subprocess.call(args,
+ stdout=sub_stdout,
+ stderr=sub_stderr)
+ if retcode:
+ raise Error("Running %s has failed." % repr(args))
+ return retcode
+
+
+class CswSrv4File(ShellMixin, object):
+ """Represents a package in the srv4 format (pkg)."""
+
+ def __init__(self, pkg_path):
+ self.pkg_path = pkg_path
+ self.workdir = None
+ self.gunzipped_path = None
+ self.transformed = False
+ self.dir_format_pkg = None
+
+ def GetWorkDir(self):
+ if not self.workdir:
+ self.workdir = tempfile.mkdtemp(prefix="pkg_")
+ fd = open(os.path.join(self.workdir, "admin"), "w")
+ fd.write(ADMIN_FILE_CONTENT)
+ fd.close()
+ return self.workdir
+
+ def GetAdminFilePath(self):
+ return os.path.join(self.GetWorkDir(), "admin")
+
+ def GetGunzippedPath(self):
+ gzip_suffix = ".gz"
+ pkg_suffix = ".pkg"
+ if not self.gunzipped_path:
+ if self.pkg_path.endswith("%s%s" % (pkg_suffix, gzip_suffix)):
+ base_name_gz = os.path.split(self.pkg_path)[1]
+ shutil.copy(self.pkg_path, self.GetWorkDir())
+ self.pkg_path = os.path.join(self.GetWorkDir(), base_name_gz)
+ args = ["gunzip", "-f", self.pkg_path]
+ retcode = self.ShellCommand(args)
+ self.gunzipped_path = self.pkg_path[:(-len(gzip_suffix))]
+ elif self.pkg_path.endswith(pkg_suffix):
+ self.gunzipped_path = self.pkg_path
+ else:
+ raise Error("The file name should end in either "
+ "%s or %s." % (gzip_suffix, pkg_suffix))
+ return self.gunzipped_path
+
+ def TransformToDir(self):
+ if not self.transformed:
+ args = ["pkgtrans", "-a", self.GetAdminFilePath(),
+ self.GetGunzippedPath(), self.GetWorkDir(), "all"]
+ retcode = self.ShellCommand(args, quiet=True)
+ dirs = self.GetDirs()
+ if len(dirs) != 1:
+ raise Error("Need exactly one package in the package stream: "
+ "%s." % (dirs))
+ self.dir_format_pkg = DirectoryFormatPackage(dirs[0])
+ self.transformed = True
+
+ def GetDirFormatPkg(self):
+ self.TransformToDir(self)
+ return self.dir_format_pkg
+
+ def GetDirs(self):
+ paths = os.listdir(self.GetWorkDir())
+ dirs = []
+ for p in paths:
+ abspath = os.path.join(self.GetWorkDir(), p)
+ if os.path.isdir(abspath):
+ dirs.append(abspath)
+ return dirs
+
+ def GetPkgmap(self, analyze_permissions, strip=None):
+ dir_format_pkg = self.GetDirFormatPkg()
+ return dir_format_pkg.GetPkgmap(analyze_permissions, strip)
+
+ def __del__(self):
+ if self.workdir:
+ logging.debug("Removing %s", repr(self.workdir))
+ shutil.rmtree(self.workdir)
+
+
+def ParsePkginfo(lines):
+ """Parses a pkginfo data."""
+ d = {}
+ for line in lines:
+ try:
+ # Can't use split, because there might be multiple '=' characters.
+ line = line.strip()
+ # Skip empty and commented lines
+ if not line: continue
+ if line.startswith("#"): continue
+ var_name, var_value = line.split("=", 1)
+ d[var_name] = var_value
+ except ValueError, e:
+ raise PackageError("Can't parse %s: %s" % (repr(line), e))
+ return d
+
+
+def PkgnameToCatName(pkgname):
+ """Creates a catalog name based on the pkgname.
+
+ SUNWbash --> sunw_bash
+ SUNWbashS --> sunw_bash_s
+ SUNWPython --> sunw_python
+
+ Incomprehensible, but unit tested!
+ """
+ known_prefixes = ["SUNW", "FJSV", "CSW"]
+ for prefix in known_prefixes:
+ if pkgname.startswith(prefix):
+ unused, tmp_prefix, the_rest = pkgname.partition(prefix)
+ pkgname = tmp_prefix + "_" + the_rest
+ return "_".join(SplitByCase(pkgname))
+
+
+def SplitByCase(s):
+ def CharType(c):
+ if c.isalnum():
+ if c.isupper():
+ return 1
+ else:
+ return 2
+ else:
+ return 3
+ chartype_list = [CharType(x) for x in s]
+ neighbors = zip(chartype_list, chartype_list[1:])
+ casechange = [False] + [x != y for x, y in neighbors]
+ str_list = [(cc * "_") + l.lower() for l, cc in zip(s, casechange)]
+ s2 = "".join(str_list)
+ return re.findall(r"[a-z]+", s2)
+
+
+def PkginfoToSrv4Name(pkginfo_dict):
+ SRV4_FN_TMPL = "%(catalog_name)s-%(version)s-%(osver)s-%(arch)s-%(tag)s.pkg"
+ fn_data = {}
+ fn_data["catalog_name"] = PkgnameToCatName(pkginfo_dict["PKG"])
+ fn_data["version"] = pkginfo_dict["VERSION"]
+ # os_version = pkginfo_dict["SUNW_PRODVERS"].split("/", 1)[0]
+ # fn_data["osver"] = pkginfo_dict["SUNW_PRODNAME"] + os_version
+ fn_data["osver"] = "SunOS5.10" # Hardcoded, because the original data contains
+ # trash.
+ fn_data["arch"] = pkginfo_dict["ARCH"]
+ fn_data["tag"] = "SUNW"
+ return SRV4_FN_TMPL % fn_data
+
+
+class DirectoryFormatPackage(ShellMixin, object):
+
+ def __init__(self, directory):
+ self.directory = directory
+ self.pkginfo_dict = None
+
+ def GetParsedPkginfo(self):
+ if not self.pkginfo_dict:
+ pkginfo_fd = open(self.GetPkginfoFilename(), "r")
+ self.pkginfo_dict = ParsePkginfo(pkginfo_fd)
+ pkginfo_fd.close()
+ return self.pkginfo_dict
+
+ def GetSrv4FileName(self):
+ """Guesses the Srv4FileName based on the package directory contents."""
+ return PkginfoToSrv4Name(self.GetParsedPkginfo())
+
+ def ToSrv4(self, target_dir):
+ target_file_name = self.GetSrv4FileName()
+ target_path = os.path.join(target_dir, target_file_name)
+ if os.path.exists(target_path):
+ return target_path
+ pkg_container_dir, pkg_dir = os.path.split(self.directory)
+ if not os.path.isdir(target_dir):
+ os.makedirs(target_dir)
+ args = ["pkgtrans", "-s", pkg_container_dir, target_path, pkg_dir]
+ self.ShellCommand(args, quiet=True)
+ args = ["gzip", "-f", target_path]
+ self.ShellCommand(args, quiet=True)
+ return target_path
+
+ def GetPkgmap(self, analyze_permissions=False, strip=None):
+ fd = open(os.path.join(self.directory, "pkgmap"), "r")
+ return Pkgmap(fd, analyze_permissions, strip)
+
+ def SetPkginfoEntry(self, key, value):
+ pkginfo = self.GetParsedPkginfo()
+ logging.debug("Setting %s to %s", repr(key), repr(value))
+ pkginfo[key] = value
+ self.WritePkginfo(pkginfo)
+ pkgmap_path = os.path.join(self.directory, "pkgmap")
+ pkgmap_fd = open(pkgmap_path, "r")
+ new_pkgmap_lines = []
+ pkginfo_re = re.compile("1 i pkginfo")
+ ws_re = re.compile(r"\s+")
+ for line in pkgmap_fd:
+ if pkginfo_re.search(line):
+ fields = ws_re.split(line)
+ # 3: size
+ # 4: sum
+ pkginfo_path = os.path.join(self.directory, "pkginfo")
+ args = ["cksum", pkginfo_path]
+ cksum_process = subprocess.Popen(args, stdout=subprocess.PIPE)
+ stdout, stderr = cksum_process.communicate()
+ cksum_process.wait()
+ size = ws_re.split(stdout)[1]
+ args = ["sum", pkginfo_path]
+ sum_process = subprocess.Popen(args, stdout=subprocess.PIPE)
+ stdout, stderr = sum_process.communicate()
+ sum_process.wait()
+ sum = ws_re.split(stdout)[0]
+ fields[3] = size
+ fields[4] = sum
+ logging.debug("New pkgmap line: %s", fields)
+ line = " ".join(fields)
+ new_pkgmap_lines.append(line.strip())
+ pkgmap_fd.close()
+ # Write that back
+ pkgmap_path_new = pkgmap_path + ".new"
+ logging.debug("Writing back to %s", pkgmap_path_new)
+ pkgmap_fd = open(pkgmap_path_new, "w")
+ pkgmap_fd.write("\n".join(new_pkgmap_lines))
+ pkgmap_fd.close()
+ shutil.move(pkgmap_path_new, pkgmap_path)
+
+ # TODO(maciej): Need to update the relevant line on pkgmap too
+
+ def GetPkginfoFilename(self):
+ return os.path.join(self.directory, "pkginfo")
+
+ def WritePkginfo(self, pkginfo_dict):
+ # Some packages extract read-only. To be sure, change them to be
+ # user-writable.
+ args = ["chmod", "-R", "u+w", self.directory]
+ self.ShellCommand(args)
+ pkginfo_filename = self.GetPkginfoFilename()
+ os.chmod(pkginfo_filename, 0644)
+ pkginfo_fd = open(pkginfo_filename, "w")
+ pkginfo_dict = self.GetParsedPkginfo()
+ for k, v in pkginfo_dict.items():
+ pkginfo_fd.write("%s=%s\n" % (k, pkginfo_dict[k]))
+ pkginfo_fd.close()
+
+ def ResetNameProperty(self):
+ """Sometimes, NAME= contains useless data. This method resets them."""
+ pkginfo_dict = self.GetParsedPkginfo()
+ catalog_name = PkgnameToCatName(pkginfo_dict["PKG"])
+ description = pkginfo_dict["DESC"]
+ pkginfo_name = "%s - %s" % (catalog_name, description)
+ self.SetPkginfoEntry("NAME", pkginfo_name)
+
+class Pkgmap(object):
+
+ def __init__(self, input, permissions=False,
+ strip=None):
+ self.paths = set()
+ self.analyze_permissions = permissions
+ for line in input:
+ fields = re.split(r'\s+', line)
+ if strip:
+ strip_re = re.compile(r"^%s" % strip)
+ fields = [re.sub(strip_re, "", x) for x in fields]
+ logging.debug(fields)
+ line_to_add = None
+ if len(fields) < 2:
+ continue
+ elif fields[1] in ('f', 'd'):
+ line_to_add = fields[3]
+ if self.analyze_permissions:
+ line_to_add += " %s" % fields[4]
+ elif fields[1] in ('s', 'l'):
+ link_from, link_to = fields[3].split("=")
+ line_to_add = "%s --> %s" % (link_from, link_to)
+ if line_to_add:
+ self.paths.add(line_to_add)
+
+
+class PackageComparator(object):
+
+ def __init__(self, file_name_a, file_name_b,
+ permissions=False,
+ strip_a=None,
+ strip_b=None):
+ self.analyze_permissions = permissions
+ self.pkg_a = CswSrv4File(file_name_a)
+ self.pkg_b = CswSrv4File(file_name_b)
+ self.strip_a = strip_a
+ self.strip_b = strip_b
+
+ def Run(self):
+ pkgmap_a = self.pkg_a.GetPkgmap(self.analyze_permissions, strip=self.strip_a)
+ pkgmap_b = self.pkg_b.GetPkgmap(self.analyze_permissions, strip=self.strip_b)
+ diff_ab = difflib.unified_diff(sorted(pkgmap_a.paths),
+ sorted(pkgmap_b.paths),
+ fromfile=self.pkg_a.pkg_path,
+ tofile=self.pkg_b.pkg_path)
+ diff_text = "\n".join(diff_ab)
+ if diff_text:
+ less_proc = subprocess.Popen(["less"], stdin=subprocess.PIPE)
+ less_stdout, less_stderr = less_proc.communicate(input=diff_text)
+ less_proc.wait()
+ else:
+ print "No differences found."
+
+
+class OpencswCatalogBuilder(object):
+
+ def __init__(self, product_dir, catalog_dir):
+ self.product_dir = product_dir
+ self.catalog_dir = catalog_dir
+
+ def Run(self):
+ pkg_dirs = os.listdir(self.product_dir)
+ for pkg_dir in pkg_dirs:
+ pkg_path = os.path.join(self.product_dir, pkg_dir)
+ pkginfo_path = os.path.join(pkg_path, "pkginfo")
+ if (os.path.isdir(pkg_path)
+ and
+ os.path.exists(pkginfo_path)):
+ if not self.Srv4Exists(pkg_path):
+ pkg = None
+ tmpdir = None
+ try:
+ tmpdir = tempfile.mkdtemp(prefix="sunw-pkg-")
+ logging.debug("Copying %s to %s", repr(pkg_path), repr(tmpdir))
+ tmp_pkg_dir = os.path.join(tmpdir, pkg_dir)
+ shutil.copytree(pkg_path, tmp_pkg_dir, symlinks=True)
+ pkg = DirectoryFormatPackage(tmp_pkg_dir)
+ # Replacing NAME= in the pkginfo, setting it to the catalog name.
+ pkg.ResetNameProperty()
+ pkg.ToSrv4(self.catalog_dir)
+ except IOError, e:
+ logging.warn("%s has failed: %s", pkg_path, e)
+ finally:
+ if pkg:
+ del(pkg)
+ if os.path.exists(tmpdir):
+ shutil.rmtree(tmpdir)
+ else:
+ logging.warn("srv4 file for %s already exists, skipping", pkg_path)
+ else:
+ logging.warn("%s is not a directory.", pkg_path)
+
+ def Srv4Exists(self, pkg_dir):
+ pkg = DirectoryFormatPackage(pkg_dir)
+ srv4_name = pkg.GetSrv4FileName()
+ srv4_name += ".gz"
+ srv4_path = os.path.join(self.catalog_dir, srv4_name)
+ result = os.path.exists(srv4_path)
+ logging.debug("Srv4Exists(%s) => %s, %s", pkg_dir, repr(srv4_path), result)
+ return result
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
More information about the devel
mailing list