[csw-devel] SF.net SVN: gar:[11643] csw/mgar/gar/v2/lib/python
wahwah at users.sourceforge.net
wahwah at users.sourceforge.net
Wed Nov 17 11:52:52 CET 2010
Revision: 11643
http://gar.svn.sourceforge.net/gar/?rev=11643&view=rev
Author: wahwah
Date: 2010-11-17 10:52:51 +0000 (Wed, 17 Nov 2010)
Log Message:
-----------
mGAR checkpkg: Moved hachoir_parser dependent classes to a separate file.
Modified Paths:
--------------
csw/mgar/gar/v2/lib/python/opencsw.py
csw/mgar/gar/v2/lib/python/package.py
csw/mgar/gar/v2/lib/python/package_checks.py
csw/mgar/gar/v2/lib/python/package_stats.py
csw/mgar/gar/v2/lib/python/sharedlib_utils.py
Added Paths:
-----------
csw/mgar/gar/v2/lib/python/inspective_package.py
Added: csw/mgar/gar/v2/lib/python/inspective_package.py
===================================================================
--- csw/mgar/gar/v2/lib/python/inspective_package.py (rev 0)
+++ csw/mgar/gar/v2/lib/python/inspective_package.py 2010-11-17 10:52:51 UTC (rev 11643)
@@ -0,0 +1,146 @@
+import package
+import os
+import re
+import logging
+import hachoir_parser as hp
+import sharedlib_utils
+import magic
+
+"""This file isolates code dependent on hachoir parser.
+
+hachoir parser takes quite a while to import.
+"""
+
+# Suppress unhelpful warnings
+# http://bitbucket.org/haypo/hachoir/issue/23
+import hachoir_core.config
+hachoir_core.config.quiet = True
+
+class InspectivePackage(package.DirectoryFormatPackage):
+ """Extends DirectoryFormatPackage to allow package inspection."""
+
+ def GetFilesMetadata(self):
+ """Returns a data structure with all the files plus their metadata.
+
+ [
+ {
+ "path": ...,
+ "mime_type": ...,
+ },
+ ]
+ """
+ if not self.files_metadata:
+ self.CheckPkgpathExists()
+ self.files_metadata = []
+ files_root = os.path.join(self.directory, "root")
+ if not os.path.exists(files_root):
+ return self.files_metadata
+ all_files = self.GetAllFilePaths()
+ def StripRe(x, strip_re):
+ return re.sub(strip_re, "", x)
+ root_re = re.compile(r"^root/")
+ file_magic = FileMagic()
+ for file_path in all_files:
+ full_path = unicode(self.MakeAbsolutePath(file_path))
+ file_info = {
+ "path": StripRe(file_path, root_re),
+ "mime_type": file_magic.GetFileMimeType(full_path)
+ }
+ if not file_info["mime_type"]:
+ logging.error("Could not establish the mime type of %s",
+ full_path)
+ # We really don't want that, as it misses binaries.
+ raise package.PackageError("Could not establish the mime type of %s"
+ % full_path)
+ if sharedlib_utils.IsBinary(file_info):
+ parser = hp.createParser(full_path)
+ if not parser:
+ logging.warning("Can't parse file %s", file_path)
+ else:
+ file_info["mime_type_by_hachoir"] = parser.mime_type
+ machine_id = parser["/header/machine"].value
+ file_info["machine_id"] = machine_id
+ file_info["endian"] = parser["/header/endian"].display
+ self.files_metadata.append(file_info)
+ return self.files_metadata
+
+ def ListBinaries(self):
+ """Lists all the binaries from a given package.
+
+ Original checkpkg code:
+
+ #########################################
+ # find all executables and dynamic libs,and list their filenames.
+ listbinaries() {
+ if [ ! -d $1 ] ; then
+ print errmsg $1 not a directory
+ rm -rf $EXTRACTDIR
+ exit 1
+ fi
+ find $1 -print | xargs file |grep ELF |nawk -F: '{print $1}'
+ }
+
+ Returns a list of absolute paths.
+
+ Now that there are files_metadata, this function can safely go away, once
+ all its callers are modified to use files_metadata instead.
+ """
+ if self.binaries is None:
+ self.CheckPkgpathExists()
+ files_metadata = self.GetFilesMetadata()
+ self.binaries = []
+ # The nested for-loop looks inefficient.
+ for file_info in files_metadata:
+ if sharedlib_utils.IsBinary(file_info):
+ self.binaries.append(file_info["path"])
+ self.binaries.sort()
+ return self.binaries
+
+
+class FileMagic(object):
+ """Libmagic sometimes returns None, which I think is a bug.
+ Trying to come up with a way to work around that. It might not even be
+ very helpful, but at least detects the issue and tries to work around it.
+ """
+
+ def __init__(self):
+ self.cookie_count = 0
+ self.magic_cookie = None
+
+ def _GetCookie(self):
+ magic_cookie = magic.open(self.cookie_count)
+ self.cookie_count += 1
+ magic_cookie.load()
+ magic_cookie.setflags(magic.MAGIC_MIME)
+ return magic_cookie
+
+ def _LazyInit(self):
+ if not self.magic_cookie:
+ self.magic_cookie = self._GetCookie()
+
+ def GetFileMimeType(self, full_path):
+ """Trying to run magic.file() a few times, not accepting None."""
+ self._LazyInit()
+ mime = None
+ for i in xrange(10):
+ mime = self.magic_cookie.file(full_path)
+ if mime:
+ break;
+ else:
+ # Returned mime is null. Re-initializing the cookie and trying again.
+ logging.error("magic_cookie.file(%s) returned None. Retrying.",
+ full_path)
+ self.magic_cookie = self._GetCookie()
+ return mime
+
+
+class InspectiveCswSrv4File(package.CswSrv4File):
+ """Allows to get the inspective version of the dir format pkg."""
+
+ # The presence of this method makes it explicit that we want an inspective
+ # version of the directory format package.
+ def GetInspectivePkg(self):
+ return self.GetDirFormatPackage()
+
+ def GetDirFormatClass(self):
+ return InspectivePackage
Modified: csw/mgar/gar/v2/lib/python/opencsw.py
===================================================================
--- csw/mgar/gar/v2/lib/python/opencsw.py 2010-11-17 10:52:17 UTC (rev 11642)
+++ csw/mgar/gar/v2/lib/python/opencsw.py 2010-11-17 10:52:51 UTC (rev 11643)
@@ -38,10 +38,6 @@
REVISION_ADDED = "revision number added"
PKG_URL_TMPL = "http://www.opencsw.org/packages/%s"
CATALOG_URL = "http://mirror.opencsw.org/opencsw/current/i386/5.10/catalog"
-BIN_MIMETYPES = (
- 'application/x-executable',
- 'application/x-sharedlib',
-)
SUBMITPKG_TMPL = """From: $from
To: $to
#if $cc
@@ -589,20 +585,3 @@
if entry["class"]: # might be None
self.classes.add(entry["class"])
return self.classes
-
-
-def IsBinary(file_info):
- """Returns True or False depending on file metadata."""
- is_a_binary = False
- if "mime_type" not in file_info:
- # This would be a problem in the data.
- return False
- if not file_info["mime_type"]:
- # This should never happen, but it seems to have happened at least once.
- # TODO: Find the affected data and figure out why.
- raise PackageError("file_info is missing mime_type:" % file_info)
- for mimetype in BIN_MIMETYPES:
- if mimetype in file_info["mime_type"]:
- is_a_binary = True
- break
- return is_a_binary
Modified: csw/mgar/gar/v2/lib/python/package.py
===================================================================
--- csw/mgar/gar/v2/lib/python/package.py 2010-11-17 10:52:17 UTC (rev 11642)
+++ csw/mgar/gar/v2/lib/python/package.py 2010-11-17 10:52:51 UTC (rev 11643)
@@ -2,10 +2,8 @@
import datetime
import difflib
-import hachoir_parser as hp
import hashlib
import logging
-import magic
import os
import re
import shutil
@@ -17,12 +15,6 @@
import opencsw
import overrides
-# Suppress unhelpful warnings
-# http://bitbucket.org/haypo/hachoir/issue/23
-import hachoir_core.config
-hachoir_core.config.quiet = True
-
-
ADMIN_FILE_CONTENT = """
basedir=default
runlevel=nocheck
@@ -178,7 +170,7 @@
if len(dirs) != 1:
raise Error("Need exactly one package in the package stream: "
"%s." % (dirs))
- self.dir_format_pkg = DirectoryFormatPackage(dirs[0])
+ self.dir_format_pkg = self.DIR_FORMAT_CLS(dirs[0])
self.transformed = True
def GetDirFormatPkg(self):
@@ -211,7 +203,8 @@
def GetPkgchkOutput(self):
"""Returns: (exit code, stdout, stderr)."""
args = ["pkgchk", "-d", self.GetGunzippedPath(), "all"]
- pkgchk_proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ pkgchk_proc = subprocess.Popen(
+ args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = pkgchk_proc.communicate()
ret = pkgchk_proc.wait()
return ret, stdout, stderr
@@ -226,7 +219,12 @@
logging.debug("Removing %s", repr(self.workdir))
shutil.rmtree(self.workdir)
+ def GetDirFormatClass(self):
+ # Derived classes can override this class member and use other classes for
+ # the directory format package.
+ return DirectoryFormatPackage
+
class DirectoryFormatPackage(ShellMixin, object):
"""Represents a package in the directory format.
@@ -373,83 +371,6 @@
raise PackageError("%s does not exist or is not a directory"
% self.directory)
- def GetFilesMetadata(self):
- """Returns a data structure with all the files plus their metadata.
-
- [
- {
- "path": ...,
- "mime_type": ...,
- },
- ]
- """
- if not self.files_metadata:
- self.CheckPkgpathExists()
- self.files_metadata = []
- files_root = os.path.join(self.directory, "root")
- if not os.path.exists(files_root):
- return self.files_metadata
- all_files = self.GetAllFilePaths()
- def StripRe(x, strip_re):
- return re.sub(strip_re, "", x)
- root_re = re.compile(r"^root/")
- file_magic = FileMagic()
- for file_path in all_files:
- full_path = unicode(self.MakeAbsolutePath(file_path))
- file_info = {
- "path": StripRe(file_path, root_re),
- "mime_type": file_magic.GetFileMimeType(full_path)
- }
- if not file_info["mime_type"]:
- logging.error("Could not establish the mime type of %s",
- full_path)
- # We really don't want that, as it misses binaries.
- raise PackageError("Could not establish the mime type of %s"
- % full_path)
- if opencsw.IsBinary(file_info):
- parser = hp.createParser(full_path)
- if not parser:
- logging.warning("Can't parse file %s", file_path)
- else:
- file_info["mime_type_by_hachoir"] = parser.mime_type
- machine_id = parser["/header/machine"].value
- file_info["machine_id"] = machine_id
- file_info["endian"] = parser["/header/endian"].display
- self.files_metadata.append(file_info)
- return self.files_metadata
-
- def ListBinaries(self):
- """Lists all the binaries from a given package.
-
- Original checkpkg code:
-
- #########################################
- # find all executables and dynamic libs,and list their filenames.
- listbinaries() {
- if [ ! -d $1 ] ; then
- print errmsg $1 not a directory
- rm -rf $EXTRACTDIR
- exit 1
- fi
- find $1 -print | xargs file |grep ELF |nawk -F: '{print $1}'
- }
-
- Returns a list of absolute paths.
-
- Now that there are files_metadata, this function can safely go away, once
- all its callers are modified to use files_metadata instead.
- """
- if self.binaries is None:
- self.CheckPkgpathExists()
- files_metadata = self.GetFilesMetadata()
- self.binaries = []
- # The nested for-loop looks inefficient.
- for file_info in files_metadata:
- if opencsw.IsBinary(file_info):
- self.binaries.append(file_info["path"])
- self.binaries.sort()
- return self.binaries
-
def GetAllFilePaths(self):
"""Returns a list of all paths from the package."""
if not self.file_paths:
@@ -529,44 +450,6 @@
def MakeAbsolutePath(self, p):
return os.path.join(self.pkgpath, p)
-
-
-class FileMagic(object):
- """Libmagic sometimes returns None, which I think is a bug.
- Trying to come up with a way to work around that.
- """
-
- def __init__(self):
- self.cookie_count = 0
- self.magic_cookie = None
-
- def _GetCookie(self):
- magic_cookie = magic.open(self.cookie_count)
- self.cookie_count += 1
- magic_cookie.load()
- magic_cookie.setflags(magic.MAGIC_MIME)
- return magic_cookie
-
- def _LazyInit(self):
- if not self.magic_cookie:
- self.magic_cookie = self._GetCookie()
-
- def GetFileMimeType(self, full_path):
- """Trying to run magic.file() a few times, not accepting None."""
- self._LazyInit()
- mime = None
- for i in xrange(10):
- mime = self.magic_cookie.file(full_path)
- if mime:
- break;
- else:
- # Returned mime is null. Re-initializing the cookie and trying again.
- logging.error("magic_cookie.file(%s) returned None. Retrying.",
- full_path)
- self.magic_cookie = self._GetCookie()
- return mime
-
-
class PackageComparator(object):
def __init__(self, file_name_a, file_name_b,
Modified: csw/mgar/gar/v2/lib/python/package_checks.py
===================================================================
--- csw/mgar/gar/v2/lib/python/package_checks.py 2010-11-17 10:52:17 UTC (rev 11642)
+++ csw/mgar/gar/v2/lib/python/package_checks.py 2010-11-17 10:52:51 UTC (rev 11643)
@@ -986,7 +986,7 @@
pkginfo_arch = pkg_data["pkginfo"]["ARCH"]
files_metadata = pkg_data["files_metadata"]
for file_metadata in files_metadata:
- if opencsw.IsBinary(file_metadata):
+ if su.IsBinary(file_metadata):
machine = HACHOIR_MACHINES[file_metadata["machine_id"]]
if machine["type"] != pkginfo_arch:
error_mgr.ReportError(
Modified: csw/mgar/gar/v2/lib/python/package_stats.py
===================================================================
--- csw/mgar/gar/v2/lib/python/package_stats.py 2010-11-17 10:52:17 UTC (rev 11642)
+++ csw/mgar/gar/v2/lib/python/package_stats.py 2010-11-17 10:52:51 UTC (rev 11643)
@@ -103,9 +103,9 @@
return True
return False
- def GetDirFormatPkg(self):
+ def GetInspectivePkg(self):
if not self.dir_format_pkg:
- self.dir_format_pkg = self.srv4_pkg.GetDirFormatPkg()
+ self.dir_format_pkg = self.srv4_pkg.GetInspectivePkg()
return self.dir_format_pkg
def GetMtime(self):
@@ -125,7 +125,7 @@
raise
def GetBinaryDumpInfo(self):
- dir_pkg = self.GetDirFormatPkg()
+ dir_pkg = self.GetInspectivePkg()
# Binaries. This could be split off to a separate function.
# man ld.so.1 for more info on this hack
env = copy.copy(os.environ)
@@ -145,7 +145,7 @@
return binaries_dump_info
def GetBasicStats(self):
- dir_pkg = self.GetDirFormatPkg()
+ dir_pkg = self.GetInspectivePkg()
basic_stats = {}
basic_stats["stats_version"] = PACKAGE_STATS_VERSION
basic_stats["pkg_path"] = self.srv4_pkg.pkg_path
@@ -158,7 +158,7 @@
return basic_stats
def GetOverrides(self):
- dir_pkg = self.GetDirFormatPkg()
+ dir_pkg = self.GetInspectivePkg()
override_list = dir_pkg.GetOverrides()
def OverrideToDict(override):
return {
@@ -171,7 +171,7 @@
def GetLddMinusRlines(self):
"""Returns ldd -r output."""
- dir_pkg = self.GetDirFormatPkg()
+ dir_pkg = self.GetInspectivePkg()
binaries = dir_pkg.ListBinaries()
ldd_output = {}
for binary in binaries:
@@ -205,7 +205,7 @@
0000000000 U abort
0000097616 T aliases_lookup
"""
- dir_pkg = self.GetDirFormatPkg()
+ dir_pkg = self.GetInspectivePkg()
binaries = dir_pkg.ListBinaries()
defined_symbols = {}
@@ -254,7 +254,7 @@
"""The list of variables needs to be synchronized with the one
at the top of this class.
"""
- dir_pkg = self.GetDirFormatPkg()
+ dir_pkg = self.GetInspectivePkg()
logging.debug("Collecting %s package statistics.", repr(dir_pkg.pkgname))
override_dicts = self.GetOverrides()
pkg_stats = {
Modified: csw/mgar/gar/v2/lib/python/sharedlib_utils.py
===================================================================
--- csw/mgar/gar/v2/lib/python/sharedlib_utils.py 2010-11-17 10:52:17 UTC (rev 11642)
+++ csw/mgar/gar/v2/lib/python/sharedlib_utils.py 2010-11-17 10:52:51 UTC (rev 11643)
@@ -15,6 +15,10 @@
AMD64_PATHS = ('amd64',)
LEGIT_CHAR_RE = re.compile(r"[a-zA-Z0-9\+]+")
SONAME_VERSION_RE = re.compile("^(?P<name>.*)\.so\.(?P<version>[\d\.]+)$")
+BIN_MIMETYPES = (
+ 'application/x-executable',
+ 'application/x-sharedlib',
+)
class SonameParsingException(Exception):
@@ -228,3 +232,20 @@
if substring_set:
current_substring = list(substring_set)[0]
return current_substring
+
+
+def IsBinary(file_info):
+ """Returns True or False depending on file metadata."""
+ is_a_binary = False
+ if "mime_type" not in file_info:
+ # This would be a problem in the data.
+ return False
+ if not file_info["mime_type"]:
+ # This should never happen, but it seems to have happened at least once.
+ # TODO: Find the affected data and figure out why.
+ raise PackageError("file_info is missing mime_type:" % file_info)
+ for mimetype in BIN_MIMETYPES:
+ if mimetype in file_info["mime_type"]:
+ is_a_binary = True
+ break
+ return is_a_binary
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
More information about the devel
mailing list