[csw-devel] SF.net SVN: gar:[10414] csw/mgar/gar/v2/lib/python
wahwah at users.sourceforge.net
wahwah at users.sourceforge.net
Sun Jul 4 09:33:16 CEST 2010
Revision: 10414
http://gar.svn.sourceforge.net/gar/?rev=10414&view=rev
Author: wahwah
Date: 2010-07-04 07:33:16 +0000 (Sun, 04 Jul 2010)
Log Message:
-----------
mGAR v2: checkpkg, performance optimizations, making the whole catalog run feasible
Modified Paths:
--------------
csw/mgar/gar/v2/lib/python/checkpkg.py
csw/mgar/gar/v2/lib/python/opencsw.py
csw/mgar/gar/v2/lib/python/package_checks.py
Modified: csw/mgar/gar/v2/lib/python/checkpkg.py
===================================================================
--- csw/mgar/gar/v2/lib/python/checkpkg.py 2010-07-04 00:12:13 UTC (rev 10413)
+++ csw/mgar/gar/v2/lib/python/checkpkg.py 2010-07-04 07:33:16 UTC (rev 10414)
@@ -1122,9 +1122,6 @@
binary_data["soname_guessed"] = False
binary_data["base_name"] = binary_base_name
if SONAME not in binary_data:
- logging.debug("The %s binary doesn't provide a SONAME. "
- "(It might be an executable)",
- binary_base_name)
# The binary doesn't tell its SONAME. We're guessing it's the
# same as the base file name.
binary_data[SONAME] = binary_base_name
Modified: csw/mgar/gar/v2/lib/python/opencsw.py
===================================================================
--- csw/mgar/gar/v2/lib/python/opencsw.py 2010-07-04 00:12:13 UTC (rev 10413)
+++ csw/mgar/gar/v2/lib/python/opencsw.py 2010-07-04 07:33:16 UTC (rev 10414)
@@ -460,11 +460,17 @@
if not os.path.isdir(destdir):
raise PackageError("%s doesn't exist or is not a directory" % destdir)
args = [os.path.join(os.path.dirname(__file__), "custom-pkgtrans"),
- src_file, destdir, pkgname ]
- pkgtrans_proc = subprocess.Popen(args)
- pkgtrans_proc.communicate()
+ src_file,
+ destdir,
+ pkgname ]
+ pkgtrans_proc = subprocess.Popen(args,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE)
+ stdout, stderr = pkgtrans_proc.communicate()
ret = pkgtrans_proc.wait()
if ret:
+ logging.error(stdout)
+ logging.error(stderr)
logging.error("% has failed" % args)
def GetPkgname(self):
@@ -827,9 +833,6 @@
machine_id = parser["/header/machine"].value
file_info["machine_id"] = machine_id
file_info["endian"] = parser["/header/endian"].display
- else:
- logging.debug("%s is not a binary, or hachoir is disabled.",
- full_path)
self.files_metadata.append(file_info)
return self.files_metadata
@@ -970,7 +973,7 @@
if strip:
strip_re = re.compile(r"^%s" % strip)
fields = [re.sub(strip_re, "", x) for x in fields]
- logging.debug(fields)
+ # logging.debug(fields)
line_to_add = None
installed_path = None
prototype_class = None
@@ -1185,7 +1188,6 @@
def IsBinary(file_info):
"""Returns True or False depending on file metadata."""
is_a_binary = False
- logging.debug("IsBinary(%s)", repr(file_info))
for mimetype in BIN_MIMETYPES:
if mimetype in file_info["mime_type"]:
is_a_binary = True
Modified: csw/mgar/gar/v2/lib/python/package_checks.py
===================================================================
--- csw/mgar/gar/v2/lib/python/package_checks.py 2010-07-04 00:12:13 UTC (rev 10413)
+++ csw/mgar/gar/v2/lib/python/package_checks.py 2010-07-04 07:33:16 UTC (rev 10414)
@@ -23,13 +23,17 @@
PATHS_ALLOWED_ONLY_IN = {
# Leading slash must be removed.
- "CSWcommon": [r"opt",
- r"opt/csw/man.*",
- r"opt/csw/doc",
- r"opt/csw/info",
- r"opt/csw/share/locale/locale.alias"],
- "CSWiconv": [r"opt/csw/lib/charset.alias"],
- "CSWtexinfo": [r"opt/csw/share/info/dir"],
+ # Using strings where possible for performance.
+ "CSWcommon": {"string": [
+ r"opt",
+ r"opt/csw/man",
+ r"opt/csw/doc",
+ r"opt/csw/info",
+ r"opt/csw/share/locale/locale.alias",
+ ],
+ "regex": [r"opt/csw/man/.*"]},
+ "CSWiconv": {"string": [r"opt/csw/lib/charset.alias"]},
+ "CSWtexinfo": {"string": [r"opt/csw/share/info/dir"]},
}
MAX_DESCRIPTION_LENGTH = 100
LICENSE_TMPL = "/opt/csw/share/doc/%s/license"
@@ -69,14 +73,14 @@
'subdir2': r"(?P<subdir2>/[\w\-\.]+)",
}
RPATH_WHITELIST = [
- ("^"
- "%(prefix)s"
- "%(prefix_extra)s"
- "/(lib|libexec)"
- "%(subdirs)s"
- "%(isalist)s?"
- "%(subdir2)s?"
- "$") % RPATH_PARTS,
+ (r"^"
+ r"%(prefix)s"
+ r"%(prefix_extra)s"
+ r"/(lib|libexec)"
+ r"%(subdirs)s"
+ r"%(isalist)s?"
+ r"%(subdir2)s?"
+ r"$") % RPATH_PARTS,
r"^\$ORIGIN$",
r"^/usr(/(ccs|dt|openwin))?/lib(/sparcv9)?$",
]
@@ -106,7 +110,28 @@
62: ("amd64", AMD64_PATHS),
}
+def RemovePackagesUnderInstallation(paths_and_pkgs_by_soname,
+ pkgs_to_be_installed):
+ """Emulates uninstallation of packages prior to installation
+ of the new ones.
+ {'libfoo.so.1': {u'/opt/csw/lib': [u'CSWlibfoo']}}
+ """
+ # for brevity
+ ptbi = set(pkgs_to_be_installed)
+ ppbs = paths_and_pkgs_by_soname
+ new_ppbs = {}
+ for soname in ppbs:
+ if soname not in new_ppbs:
+ new_ppbs[soname] = {}
+ for binary_path in ppbs[soname]:
+ for pkgname in ppbs[soname][binary_path]:
+ if pkgname not in ptbi:
+ if binary_path not in new_ppbs[soname]:
+ new_ppbs[soname][binary_path] = []
+ new_ppbs[soname][binary_path].append(pkgname)
+ return new_ppbs
+
def CatalognameLowercase(pkg_data, error_mgr, logger, messenger):
catalogname = pkg_data["basic_stats"]["catalogname"]
if catalogname != catalogname.lower():
@@ -201,28 +226,6 @@
entry["path"])
-def RemovePackagesUnderInstallation(paths_and_pkgs_by_soname,
- pkgs_to_be_installed):
- """Emulates uninstallation of packages prior to installation
- of the new ones.
- {'libfoo.so.1': {u'/opt/csw/lib': [u'CSWlibfoo']}}
- """
- # for brevity
- ptbi = set(pkgs_to_be_installed)
- ppbs = paths_and_pkgs_by_soname
- new_ppbs = {}
- for soname in ppbs:
- if soname not in new_ppbs:
- new_ppbs[soname] = {}
- for binary_path in ppbs[soname]:
- for pkgname in ppbs[soname][binary_path]:
- if pkgname not in ptbi:
- if binary_path not in new_ppbs[soname]:
- new_ppbs[soname][binary_path] = []
- new_ppbs[soname][binary_path].append(pkgname)
- return new_ppbs
-
-
def SetCheckLibraries(pkgs_data, error_mgr, logger, messenger):
"""Second version of the library checking code.
@@ -588,7 +591,16 @@
common_path = common_path[1:]
common_paths.append(common_path)
paths_only_allowed_in = copy.copy(PATHS_ALLOWED_ONLY_IN)
- paths_only_allowed_in["CSWcommon"] += common_paths
+ ss = paths_only_allowed_in["CSWcommon"]["string"]
+ paths_only_allowed_in["CSWcommon"]["string"] = set(ss).union(common_paths)
+ # Compile all the regex expressions ahead of time.
+ for pkgname in paths_only_allowed_in:
+ if "regex" in paths_only_allowed_in[pkgname]:
+ regexes = paths_only_allowed_in[pkgname]["regex"]
+ paths_only_allowed_in[pkgname]["regex"] = map(re.compile, regexes)
+ if "string" in paths_only_allowed_in[pkgname]:
+ paths_only_allowed_in[pkgname]["string"] = set(
+ paths_only_allowed_in[pkgname]["string"])
paths_in_pkg = set()
for entry in pkg_data["pkgmap"]:
entry_path = entry["path"]
@@ -599,14 +611,23 @@
paths_in_pkg.add(entry_path)
for pkgname in paths_only_allowed_in:
if pkgname != pkg_data["basic_stats"]["pkgname"]:
- for disallowed_path in paths_only_allowed_in[pkgname]:
- disallowed_re = re.compile(r"^%s$" % disallowed_path)
- for path_in_pkg in paths_in_pkg:
- if disallowed_re.match(path_in_pkg):
- error_mgr.ReportError(
- "disallowed-path", path_in_pkg,
- "This path is already provided by %s "
- "or is not allowed for other reasons." % pkgname)
+ if "string" in paths_only_allowed_in[pkgname]:
+ ss = paths_only_allowed_in[pkgname]["string"]
+ intersection = ss.intersection(paths_in_pkg)
+ for path_in_pkg in intersection:
+ error_mgr.ReportError(
+ "disallowed-path", path_in_pkg,
+ "This path is already provided by %s "
+ "or is not allowed for other reasons." % pkgname)
+ if "regex" in paths_only_allowed_in[pkgname]:
+ rr = paths_only_allowed_in[pkgname]["regex"]
+ for disallowed_re in rr:
+ badpaths = filter(disallowed_re.match, paths_in_pkg)
+ for path_in_pkg in badpaths:
+ error_mgr.ReportError(
+ "disallowed-path", path_in_pkg,
+ "This path is already provided by %s "
+ "or is not allowed for other reasons." % pkgname)
def CheckLinkingAgainstSunX11(pkg_data, error_mgr, logger, messenger):
@@ -629,7 +650,7 @@
entry["path"])
-def CheckBadPaths(pkg_data, error_mgr, logger, messenger):
+def CheckBadContent(pkg_data, error_mgr, logger, messenger):
for regex in pkg_data["bad_paths"]:
for file_name in pkg_data["bad_paths"][regex]:
messenger.Message("File %s contains bad content: %s. "
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
More information about the devel
mailing list