[csw-devel] SF.net SVN: gar:[19982] csw/mgar/gar/v2-yann
chninkel at users.sourceforge.net
chninkel at users.sourceforge.net
Mon Dec 31 02:04:31 CET 2012
Revision: 19982
http://gar.svn.sourceforge.net/gar/?rev=19982&view=rev
Author: chninkel
Date: 2012-12-31 01:04:31 +0000 (Mon, 31 Dec 2012)
Log Message:
-----------
gar/v2-yann: updated from HEAD
Modified Paths:
--------------
csw/mgar/gar/v2-yann/bin/cswproto
csw/mgar/gar/v2-yann/bin/pathfilter
csw/mgar/gar/v2-yann/gar.pkg.mk
csw/mgar/gar/v2-yann/lib/python/catalog_gc.py
csw/mgar/gar/v2-yann/lib/python/common_constants.py
csw/mgar/gar/v2-yann/lib/python/csw_upload_pkg.py
csw/mgar/gar/v2-yann/lib/python/csw_upload_pkg_test.py
csw/mgar/gar/v2-yann/lib/python/database.py
csw/mgar/gar/v2-yann/lib/python/inspective_package.py
csw/mgar/gar/v2-yann/lib/python/inspective_package_test.py
csw/mgar/gar/v2-yann/lib/python/models.py
csw/mgar/gar/v2-yann/lib/python/package.py
csw/mgar/gar/v2-yann/lib/python/package_checks.py
csw/mgar/gar/v2-yann/lib/python/package_stats.py
csw/mgar/gar/v2-yann/lib/python/rest.py
csw/mgar/gar/v2-yann/lib/python/sharedlib_utils.py
csw/mgar/gar/v2-yann/lib/python/system_pkgmap_test.py
csw/mgar/gar/v2-yann/lib/web/pkgdb_web.py
csw/mgar/gar/v2-yann/lib/web/releases_web.py
csw/mgar/gar/v2-yann/lib/web/templates/CatalogDetail.html
Added Paths:
-----------
csw/mgar/gar/v2-yann/lib/python/garbage_collection.py
Property Changed:
----------------
csw/mgar/gar/v2-yann/
Property changes on: csw/mgar/gar/v2-yann
___________________________________________________________________
Modified: svn:mergeinfo
- /csw/mgar/gar/v2:4936-6678,19287-19753
/csw/mgar/gar/v2-bwalton:9784-10011
/csw/mgar/gar/v2-checkpkg:7722-7855
/csw/mgar/gar/v2-checkpkg-override-relocation:10585-10737
/csw/mgar/gar/v2-checkpkg-stats:8454-8649
/csw/mgar/gar/v2-collapsed-modulations:6895
/csw/mgar/gar/v2-defaultchange:13903-14022
/csw/mgar/gar/v2-dirpackage:8125-8180
/csw/mgar/gar/v2-fortran:10883-12516
/csw/mgar/gar/v2-git/v2-relocate:7617
/csw/mgar/gar/v2-migrateconf:7082-7211
/csw/mgar/gar/v2-noexternals:11592-11745
/csw/mgar/gar/v2-raised-buildlevel:15906-15949
/csw/mgar/gar/v2-relocate:5028-11738
/csw/mgar/gar/v2-skayser:6087-6132
/csw/mgar/gar/v2-solaris11:18134-18236
/csw/mgar/gar/v2-sqlite:10434-10449
/csw/mgar/gar/v2-uwatch2:12141-13270
+ /csw/mgar/gar/v2:4936-6678,19234-19981
/csw/mgar/gar/v2-bwalton:9784-10011
/csw/mgar/gar/v2-checkpkg:7722-7855
/csw/mgar/gar/v2-checkpkg-override-relocation:10585-10737
/csw/mgar/gar/v2-checkpkg-stats:8454-8649
/csw/mgar/gar/v2-collapsed-modulations:6895
/csw/mgar/gar/v2-defaultchange:13903-14022
/csw/mgar/gar/v2-dirpackage:8125-8180
/csw/mgar/gar/v2-fortran:10883-12516
/csw/mgar/gar/v2-git/v2-relocate:7617
/csw/mgar/gar/v2-migrateconf:7082-7211
/csw/mgar/gar/v2-noexternals:11592-11745
/csw/mgar/gar/v2-raised-buildlevel:15906-15949
/csw/mgar/gar/v2-relocate:5028-11738
/csw/mgar/gar/v2-skayser:6087-6132
/csw/mgar/gar/v2-solaris11:18134-18236
/csw/mgar/gar/v2-sqlite:10434-10449
/csw/mgar/gar/v2-uwatch2:12141-13270
Modified: csw/mgar/gar/v2-yann/bin/cswproto
===================================================================
--- csw/mgar/gar/v2-yann/bin/cswproto 2012-12-31 00:54:47 UTC (rev 19981)
+++ csw/mgar/gar/v2-yann/bin/cswproto 2012-12-31 01:04:31 UTC (rev 19982)
@@ -20,7 +20,7 @@
use POSIX;
use vars qw/
- @XFORMS @Common $StdOwn $StdGrp $StdDirPerm $common $stamp $root
+ @XFORMS $Common $StdOwn $StdGrp $StdDirPerm $common $stamp $root
/;
# atime=8,mtime=9,ctime=10
@@ -66,15 +66,6 @@
exit 1;
}
-# Check whether a path is a 'common' file, defined in __DATA__
-sub is_common {
- my $path = shift;
- foreach my $pat (@Common) {
- return 1 if $path =~ m!^$pat$!;
- }
- return 0;
-}
-
# Returns true if the file should be excluded, false otherwise.
sub exclude {
my $path = shift;
@@ -108,7 +99,7 @@
if( $common ) {
# Load common path contents
- my %alldirs;
+ my %alldirs = ('/' => 1);
open F, $common || die "Couldn't open $common";
while (<F>) {
chomp; next if /^\s*$/ or /^#/;
@@ -119,7 +110,8 @@
$alldirs{$_} = 1 foreach (@pc);
}
close F;
- @Common = map { qr#$_$# } ('/', keys %alldirs);
+ my $re = '^(' . join( '|', keys %alldirs ) . ')$';
+ $Common = qr /$re/;
}
my @prototype;
@@ -160,7 +152,7 @@
next unless $F[2];
# Then process any excludes
- next SPECLINE if( is_common( $F[2] ) );
+ next SPECLINE if( $Common && $F[2] =~ /$Common/ );
next if exclude($realpath);
# Fix up dir permissions/file ownership.
Modified: csw/mgar/gar/v2-yann/bin/pathfilter
===================================================================
--- csw/mgar/gar/v2-yann/bin/pathfilter 2012-12-31 00:54:47 UTC (rev 19981)
+++ csw/mgar/gar/v2-yann/bin/pathfilter 2012-12-31 01:04:31 UTC (rev 19982)
@@ -14,6 +14,7 @@
# excluded also.
use strict;
+use Carp;
use Getopt::Long qw(:config no_ignore_case);
use Pod::Usage;
@@ -53,8 +54,11 @@
push @seltemp, $re;
}
-my $mre = '^(' . join( '|', @seltemp ) . ')$';
-push @selection, [ $mode, qr/$mre/ ];
+# If @selection_args is empty the @selection must also be empty
+if( defined $mode ) {
+ my $mre = '^(' . join( '|', @seltemp ) . ')$';
+ push @selection, [ $mode, qr/$mre/ ];
+}
pod2usage(-verbose => 2) if $help;
Modified: csw/mgar/gar/v2-yann/gar.pkg.mk
===================================================================
--- csw/mgar/gar/v2-yann/gar.pkg.mk 2012-12-31 00:54:47 UTC (rev 19981)
+++ csw/mgar/gar/v2-yann/gar.pkg.mk 2012-12-31 01:04:31 UTC (rev 19982)
@@ -31,7 +31,7 @@
# SRCPACKAGE is the name of the package containing the sources
ifeq ($(origin PACKAGES), undefined)
-PACKAGES = $(if $(filter %.gspec,$(DISTFILES)),,CSW$(NAME))
+PACKAGES = $(if $(filter %.gspec,$(DISTFILES)),,CSW$(subst _,-,$(NAME)))
CATALOGNAME ?= $(if $(filter %.gspec,$(DISTFILES)),,$(subst -,_,$(NAME)))
SRCPACKAGE_BASE = $(firstword $(basename $(filter %.gspec,$(DISTFILES))) $(PACKAGES))
SRCPACKAGE ?= $(SRCPACKAGE_BASE)-src
@@ -208,7 +208,14 @@
SPKG_SPOOLROOT ?= $(DESTROOT)
SPKG_SPOOLDIR ?= $(SPKG_SPOOLROOT)/spool.$(GAROSREL)-$(GARCH)
-SPKG_EXPORT ?= $(HOME)/staging/build-$(shell date '+%d.%b.%Y')
+ifdef SPKG_EXPORT
+# The definition may include variable parts like a call to "date". This would lead to different directory names
+# for multiple invocation in longs builds and a failing checkpkg due to lookup in wrong directories, so fixate
+# once what we have.
+SPKG_EXPORT := $(SPKG_EXPORT)
+else
+SPKG_EXPORT := $(HOME)/staging/build-$(shell date '+%d.%b.%Y')
+endif
SPKG_PKGROOT ?= $(PKGROOT)
SPKG_PKGBASE ?= $(PKGROOT)
SPKG_WORKDIR ?= $(CURDIR)/$(WORKDIR)
@@ -216,8 +223,20 @@
SPKG_DEPEND_DB = $(GARDIR)/csw/depend.db
-SPKG_PKGFILE ?= %{bitname}-%{SPKG_VERSION},%{SPKG_REVSTAMP}-%{SPKG_OSNAME}-%{arch}-$(or $(filter $(call _REVISION),UNCOMMITTED NOTVERSIONED NOSVN),CSW).pkg
+# These variables could change value transiently and need to be passed to subinvocations of GAR
+_PASS_GAR_SUBINVOCATION_EXPORTS += SPKG_EXPORT
+_PASS_GAR_ENV = $(foreach V,$(_PASS_GAR_SUBINVOCATION_EXPORTS),$V=$($V))
+# This is the old specification being evaluated during mkpackage. The expansion of the SPKG_REVSTAMP leads to
+# problems later on when need the filename for checkpkg again and too much time has passed. In the new approach
+# the packagename is directly put in the gspec.
+# SPKG_PKGFILE ?= %{bitname}-%{SPKG_VERSION},%{SPKG_REVSTAMP}-%{SPKG_OSNAME}-%{arch}-$(or $(filter $(call _REVISION),UNCOMMITTED NOTVERSIONED NOSVN),CSW).pkg
+
+# The filename for a package
+define _pkgfile
+$(call catalogname,$(1))-$(call pkgvar,SPKG_VERSION,$(1)),$(call pkgvar,SPKG_REVSTAMP,$(1))-$(call pkgvar,SPKG_OSNAME,$(1))-$(if $(or $(ARCHALL),$(ARCHALL_$(1))),all,$(GARCH))-$(or $(filter $(call _REVISION),UNCOMMITTED NOTVERSIONED NOSVN),CSW).pkg
+endef
+
MIGRATECONF ?= $(strip $(foreach S,$(filter-out $(OBSOLETED_PKGS),$(SPKG_SPECS)),$(if $(or $(MIGRATE_FILES_$S),$(MIGRATE_FILES)),/etc/opt/csw/pkg/$S/cswmigrateconf)))
# It is NOT sufficient to change the pathes here, they must be adjusted in merge-* also
@@ -314,6 +333,7 @@
# Where we find our mkpackage global templates
PKGLIB = $(GARDIR)/pkglib
+# These variables are for mkpackage and the gspec expansion
PKG_EXPORTS = NAME VERSION DESCRIPTION CATEGORIES GARCH GARDIR GARBIN
PKG_EXPORTS += CURDIR WORKDIR WORKDIR_FIRSTMOD WORKSRC WORKSRC_FIRSTMOD PKGROOT
PKG_EXPORTS += SPKG_REVSTAMP SPKG_PKGNAME SPKG_DESC SPKG_VERSION SPKG_CATEGORY
@@ -596,6 +616,7 @@
$(_DBG)$(if $(filter $*.gspec,$(DISTFILES)),,\
(echo "%var bitname $(call catalogname,$*)"; \
echo "%var pkgname $*"; \
+ echo "%var pkgfile $(call _pkgfile,$*)"; \
$(if $(or $(ARCHALL),$(ARCHALL_$*)),echo "%var arch all";) \
$(if $(_CATEGORY_GSPEC_INCLUDE),echo "%include url file://%{PKGLIB}/$(_CATEGORY_GSPEC_INCLUDE)")) >$@\
)
@@ -949,7 +970,7 @@
@echo
@echo "The following packages have been built:"
@echo
- @$(MAKE) -s GAR_PLATFORM=$(GAR_PLATFORM) _pkgshow
+ @$(MAKE) -s $(_PASS_GAR_ENV) GAR_PLATFORM=$(GAR_PLATFORM) _pkgshow
@echo
@$(DONADA)
@@ -958,7 +979,7 @@
dirpackage: _package
@echo "The following packages have been built:"
@echo
- @$(MAKE) -s GAR_PLATFORM=$(GAR_PLATFORM) _dirpkgshow
+ @$(MAKE) -s $(_PASS_GAR_ENV) GAR_PLATFORM=$(GAR_PLATFORM) _dirpkgshow
@echo
@$(DONADA)
@@ -1031,14 +1052,15 @@
_PROPAGATE_ENV += PARALLELMFLAGS
_PROPAGATE_ENV += PARALLELMODULATIONS
_PROPAGATE_ENV += PATH
+_PROPAGATE_ENV += SKIPTEST
platforms: _PACKAGING_PLATFORMS=$(if $(ARCHALL),$(firstword $(PACKAGING_PLATFORMS)),$(PACKAGING_PLATFORMS))
platforms:
$(foreach P,$(_PACKAGING_PLATFORMS),\
$(if $(PACKAGING_HOST_$P),\
$(if $(filter $(THISHOST),$(PACKAGING_HOST_$P)),\
- $(MAKE) GAR_PLATFORM=$P _package && ,\
- $(SSH) -t $(PACKAGING_HOST_$P) "$(foreach V,$(_PROPAGATE_ENV),$(if $($V),$V=$($V))) $(MAKE) -I $(GARDIR) -C $(CURDIR) GAR_PLATFORM=$P _package" && \
+ $(MAKE) $(_PASS_GAR_ENV) GAR_PLATFORM=$P _package && ,\
+ $(SSH) -t $(PACKAGING_HOST_$P) "$(foreach V,$(_PROPAGATE_ENV),$(if $($V),$V=$($V))) $(MAKE) -I $(GARDIR) -C $(CURDIR) $(_PASS_GAR_ENV) GAR_PLATFORM=$P _package" && \
),\
$(error *** No host has been defined for platform $P)\
)\
@@ -1051,9 +1073,9 @@
$(if $(ARCHALL),echo " (suitable for all architectures)\c";) \
$(if $(filter $(THISHOST),$(PACKAGING_HOST_$P)),\
echo " (built on this host)";\
- $(MAKE) -s GAR_PLATFORM=$P _pkgshow;echo;,\
+ $(MAKE) -s $(_PASS_GAR_ENV) GAR_PLATFORM=$P _pkgshow;echo;,\
echo " (built on host '$(PACKAGING_HOST_$P)')";\
- $(SSH) $(PACKAGING_HOST_$P) "PATH=$$PATH:/opt/csw/bin $(MAKE) -I $(GARDIR) -C $(CURDIR) -s GAR_PLATFORM=$P _pkgshow";echo;\
+ $(SSH) $(PACKAGING_HOST_$P) "PATH=$$PATH:/opt/csw/bin $(MAKE) -I $(GARDIR) -C $(CURDIR) -s $(_PASS_GAR_ENV) GAR_PLATFORM=$P _pkgshow";echo;\
)\
)
@$(MAKECOOKIE)
@@ -1063,8 +1085,8 @@
$(foreach P,$(_PACKAGING_PLATFORMS),\
$(if $(PACKAGING_HOST_$P),\
$(if $(filter $(THISHOST),$(PACKAGING_HOST_$P)),\
- $(MAKE) -s GAR_PLATFORM=$P $* && ,\
- $(SSH) -t $(PACKAGING_HOST_$P) "PATH=$$PATH:/opt/csw/bin $(MAKE) -I $(GARDIR) -C $(CURDIR) GAR_PLATFORM=$P $*" && \
+ $(MAKE) -s $(_PASS_GAR_ENV) GAR_PLATFORM=$P $* && ,\
+ $(SSH) -t $(PACKAGING_HOST_$P) "PATH=$$PATH:/opt/csw/bin $(MAKE) -I $(GARDIR) -C $(CURDIR) $(_PASS_GAR_ENV) GAR_PLATFORM=$P $*" && \
),\
$(error *** No host has been defined for platform $P)\
)\
Modified: csw/mgar/gar/v2-yann/lib/python/catalog_gc.py
===================================================================
--- csw/mgar/gar/v2-yann/lib/python/catalog_gc.py 2012-12-31 00:54:47 UTC (rev 19981)
+++ csw/mgar/gar/v2-yann/lib/python/catalog_gc.py 2012-12-31 01:04:31 UTC (rev 19982)
@@ -1,16 +1,29 @@
#!/opt/csw/bin/python2.6
-"""Garbage-collecting for a catalog.
+"""Garbage-collecting for the catalog tree.
The allpkgs directory may contain unused files. They should be deleted.
"""
+import logging
import optparse
-import logging
-import os.path
+import os
+import pipes
import re
import common_constants
+import rest
+USAGE = """%prog --catalog-tree /home/mirror/opencsw-official --dest_dir /home/mirror/gc > gc_01.sh
+less gc_01.sh
+
+# Looks good?
+
+bash gc_01.sh
+
+If everything is fine (catalog still generates, no files are missing that are
+necessary), you can remove files from /home/mirror/gc.
+"""
+
class Error(Exception):
"""Base error."""
@@ -20,11 +33,12 @@
class CatalogGarbageCollector(object):
- ADDITIONAL_CATALOGS = ("current", "stable")
+ ADDITIONAL_CATALOGS = ("legacy",)
- def __init__(self, d):
+ def __init__(self, d, dest_dir):
logging.debug("CatalogGarbageCollector(%s)", repr(d))
self.catalog_dir = d
+ self.dest_dir = dest_dir
def GarbageCollect(self):
allpkgs_path = os.path.join(self.catalog_dir, "allpkgs")
@@ -33,9 +47,15 @@
catalogs_by_files = {}
for p in os.listdir(allpkgs_path):
allpkgs.add(p)
- catalogs_to_check = (
- tuple(common_constants.DEFAULT_CATALOG_RELEASES)
- + self.ADDITIONAL_CATALOGS)
+ catalogs_to_check = tuple(common_constants.DEFAULT_CATALOG_RELEASES)
+ catalogs_to_check += self.ADDITIONAL_CATALOGS
+ rest_client = rest.RestClient()
+ catalog_triplet_list = rest_client.GetCatalogList()
+ catalogs_to_check += tuple(set([x[2] for x in catalog_triplet_list]))
+ catalogs_to_check = tuple(set(catalogs_to_check))
+ logging.info("Collecting packages from catalogs: %s",
+ catalogs_to_check)
+ file_sizes = {}
for catrel in catalogs_to_check:
for arch in common_constants.PHYSICAL_ARCHITECTURES:
for osrel_long in common_constants.OS_RELS:
@@ -49,25 +69,42 @@
for p in os.listdir(catalog_path):
if pkg_re.search(p):
# It's a package
+ full_path = os.path.join(catalog_path, p)
files_in_catalogs.add(p)
l = catalogs_by_files.setdefault(p, [])
l.append((catrel, arch, osrel_short))
- for p in allpkgs.difference(files_in_catalogs):
- logging.debug("File %s is not used by any catalogs.", p)
- print "rm %s/%s" % (allpkgs_path, p)
+ if full_path not in file_sizes:
+ s = os.stat(full_path)
+ file_sizes[full_path] = s.st_size
+ logging.info(
+ "Collected from %r, found references to %d files (out of %d in allpkgs)",
+ catrel, len(files_in_catalogs), len(allpkgs))
+ to_remove = allpkgs.difference(files_in_catalogs)
+ logging.debug("Collecting file sizes.")
+ total_size = sum(os.stat(os.path.join(allpkgs_path, x)).st_size
+ for x in to_remove)
+ logging.info("Found %d packages to remove, total size: %.1fMB.",
+ len(to_remove), float(total_size) / 1024 ** 2)
+ for p in to_remove:
+ full_path = os.path.join(allpkgs_path, p)
+ print "mv", pipes.quote(full_path), pipes.quote(self.dest_dir)
def main():
parser = optparse.OptionParser()
- parser.add_option("-c", "--catalog",
- dest="catalog",
- help="Catalog path")
+ parser.add_option("--catalog-tree",
+ dest="catalog_tree",
+ help=("Path to the catalog tree, that is the directory "
+ "containing subdirectories unstable, testing, etc."))
+ parser.add_option("--dest-dir",
+ dest="dest_dir",
+ help=("Move files out to this catalog."))
options, args = parser.parse_args()
logging.basicConfig(level=logging.DEBUG)
- if not options.catalog:
+ if not options.catalog_tree or not options.dest_dir:
parser.print_usage()
- raise UsageError("Missing catalog option, see --help.")
- gcg = CatalogGarbageCollector(options.catalog)
+ raise UsageError("Missing the catalog tree option, see --help.")
+ gcg = CatalogGarbageCollector(options.catalog_tree, options.dest_dir)
gcg.GarbageCollect()
Modified: csw/mgar/gar/v2-yann/lib/python/common_constants.py
===================================================================
--- csw/mgar/gar/v2-yann/lib/python/common_constants.py 2012-12-31 00:54:47 UTC (rev 19981)
+++ csw/mgar/gar/v2-yann/lib/python/common_constants.py 2012-12-31 01:04:31 UTC (rev 19982)
@@ -74,6 +74,7 @@
'dublin',
'unstable',
'legacy',
+ 'kiel',
])
# At some point, it was used to prevent people from linking against
Modified: csw/mgar/gar/v2-yann/lib/python/csw_upload_pkg.py
===================================================================
--- csw/mgar/gar/v2-yann/lib/python/csw_upload_pkg.py 2012-12-31 00:54:47 UTC (rev 19981)
+++ csw/mgar/gar/v2-yann/lib/python/csw_upload_pkg.py 2012-12-31 01:04:31 UTC (rev 19982)
@@ -32,28 +32,29 @@
Uploads a set of packages to the unstable catalog in opencsw-future.
- When an architecture-independent package is uploaded, it gets added to both
- sparc and i386 catalogs
+ sparc and i386 catalogs
- When a SunOS5.x package is sent, it's added to catalogs SunOS5.x,
SunOS5.(x+1), up to SunOS5.11, but only if there are no packages specific to
5.10 (and/or 5.11).
- If a package update is sent, the tool uses both the catalogname and the
- pkgname to identify the package it's updating. For example, you might upload
- foo_stub/CSWfoo and mean to replace foo/CSWfoo with it.
+ pkgname to identify the package it's updating. For example, you might upload
+ foo_stub/CSWfoo and mean to replace foo/CSWfoo with it.
The --os-release flag makes %prog only insert the package to catalog with the
given OS release.
+The --catalog-release flag allows to insert a package into a specific catalog,
+instead of the default 'unstable'.
+
= General considerations =
This tool operates on a database of packages and a package file store. It
-modifies a number of package catalogs, a cartesian product of:
+modifies a number of package catalogs, e.g.:
- {legacy,dublin,unstable}x{sparc,i386}x{5.8,5.9.5.10,5.11}
+ {{dublin,unstable,kiel,bratislava}}x{{sparc,i386}}x{{5.8,5.9.5.10,5.11}}
-This amounts to 3x2x4 = 24 package catalogs total.
-
= Removing packages from the catalog =
The --remove option works the same way as the regular use, except that it
@@ -524,7 +525,8 @@
help="Remove packages from catalogs instead of adding them")
parser.add_option("--os-release",
dest="os_release",
- help="If specified, only uploads to the specified OS release.")
+ help="If specified, only uploads to the specified OS release. "
+ "Valid values: {0}".format(" ".join(common_constants.OS_RELS)))
parser.add_option("--rest-url",
dest="rest_url",
default=BASE_URL,
@@ -568,6 +570,11 @@
else:
print "Continuing anyway."
+ if os_release and os_release not in common_constants.OS_RELS:
+ raise DataError(
+ "OS release %r is not valid. Valid values: %r"
+ % (os_release, common_constants.OS_RELS))
+
username, password = rest.GetUsernameAndPassword()
uploader = Srv4Uploader(args,
options.rest_url,
Modified: csw/mgar/gar/v2-yann/lib/python/csw_upload_pkg_test.py
===================================================================
--- csw/mgar/gar/v2-yann/lib/python/csw_upload_pkg_test.py 2012-12-31 00:54:47 UTC (rev 19981)
+++ csw/mgar/gar/v2-yann/lib/python/csw_upload_pkg_test.py 2012-12-31 01:04:31 UTC (rev 19982)
@@ -98,7 +98,7 @@
def test_MatchSrv4ToCatalogsSame(self):
rest_client_mock = self.mox.CreateMock(rest.RestClient)
self.mox.StubOutWithMock(rest, "RestClient")
- rest.RestClient(None).AndReturn(rest_client_mock)
+ rest.RestClient(None, username=None, password=None).AndReturn(rest_client_mock)
rest_client_mock.Srv4ByCatalogAndCatalogname(
'unstable', 'sparc', u'SunOS5.9', 'gdb').AndReturn(GDB_STRUCT_9)
rest_client_mock.Srv4ByCatalogAndCatalogname(
@@ -121,7 +121,7 @@
def test_MatchSrv4ToCatalogsDifferent(self):
rest_client_mock = self.mox.CreateMock(rest.RestClient)
self.mox.StubOutWithMock(rest, "RestClient")
- rest.RestClient(None).AndReturn(rest_client_mock)
+ rest.RestClient(None, username=None, password=None).AndReturn(rest_client_mock)
rest_client_mock.Srv4ByCatalogAndCatalogname(
'unstable', 'sparc', u'SunOS5.9', 'gdb').AndReturn(GDB_STRUCT_9)
rest_client_mock.Srv4ByCatalogAndCatalogname(
@@ -144,7 +144,7 @@
# uploading a 5.10 package.
rest_client_mock = self.mox.CreateMock(rest.RestClient)
self.mox.StubOutWithMock(rest, "RestClient")
- rest.RestClient(None).AndReturn(rest_client_mock)
+ rest.RestClient(None, username=None, password=None).AndReturn(rest_client_mock)
rest_client_mock.Srv4ByCatalogAndCatalogname(
'unstable', 'sparc', u'SunOS5.10', 'gdb').AndReturn(GDB_STRUCT_9)
rest_client_mock.Srv4ByCatalogAndCatalogname(
@@ -164,7 +164,7 @@
def test_MatchSrv4ToCatalogsSameSpecificOsrel(self):
rest_client_mock = self.mox.CreateMock(rest.RestClient)
self.mox.StubOutWithMock(rest, "RestClient")
- rest.RestClient(None).AndReturn(rest_client_mock)
+ rest.RestClient(None, username=None, password=None).AndReturn(rest_client_mock)
rest_client_mock.Srv4ByCatalogAndCatalogname(
'unstable', 'sparc', u'SunOS5.9', 'gdb').AndReturn(GDB_STRUCT_9)
rest_client_mock.Srv4ByCatalogAndCatalogname(
@@ -185,7 +185,7 @@
def test_MatchSrv4ToCatalogsAbsentFromAll(self):
rest_client_mock = self.mox.CreateMock(rest.RestClient)
self.mox.StubOutWithMock(rest, "RestClient")
- rest.RestClient(None).AndReturn(rest_client_mock)
+ rest.RestClient(None, username=None, password=None).AndReturn(rest_client_mock)
rest_client_mock.Srv4ByCatalogAndCatalogname(
'unstable', 'sparc', u'SunOS5.9', 'gdb').AndReturn(None)
rest_client_mock.Srv4ByCatalogAndCatalogname(
@@ -208,7 +208,7 @@
def test_MatchSrv4ToCatalogsSameSpecificOsrelAlreadyPresent(self):
rest_client_mock = self.mox.CreateMock(rest.RestClient)
self.mox.StubOutWithMock(rest, "RestClient")
- rest.RestClient(None).AndReturn(rest_client_mock)
+ rest.RestClient(None, username=None, password=None).AndReturn(rest_client_mock)
rest_client_mock.Srv4ByCatalogAndCatalogname(
'unstable', 'sparc', u'SunOS5.9', 'gdb').AndReturn(GDB_STRUCT_9)
rest_client_mock.Srv4ByCatalogAndCatalogname(
@@ -229,7 +229,7 @@
def test_MatchSrv4ToCatalogsNotPresent(self):
rest_client_mock = self.mox.CreateMock(rest.RestClient)
self.mox.StubOutWithMock(rest, "RestClient")
- rest.RestClient(None).AndReturn(rest_client_mock)
+ rest.RestClient(None, username=None, password=None).AndReturn(rest_client_mock)
rest_client_mock.Srv4ByCatalogAndCatalogname(
'unstable', 'sparc', u'SunOS5.9', 'gdb').AndReturn(GDB_STRUCT_9)
rest_client_mock.Srv4ByCatalogAndCatalogname(
@@ -250,7 +250,7 @@
def test_MatchSrv4ToCatalogsFirstNotPresent(self):
rest_client_mock = self.mox.CreateMock(rest.RestClient)
self.mox.StubOutWithMock(rest, "RestClient")
- rest.RestClient(None).AndReturn(rest_client_mock)
+ rest.RestClient(None, username=None, password=None).AndReturn(rest_client_mock)
rest_client_mock.Srv4ByCatalogAndCatalogname(
'unstable', 'sparc', u'SunOS5.9', 'gdb').AndReturn(None)
rest_client_mock.Srv4ByCatalogAndCatalogname(
@@ -271,7 +271,7 @@
def test_MatchSrv4ToCatalogsSolaris8(self):
rest_client_mock = self.mox.CreateMock(rest.RestClient)
self.mox.StubOutWithMock(rest, "RestClient")
- rest.RestClient(None).AndReturn(rest_client_mock)
+ rest.RestClient(None, username=None, password=None).AndReturn(rest_client_mock)
rest_client_mock.Srv4ByCatalogAndCatalogname(
'unstable', 'sparc', u'SunOS5.9', 'gdb').AndReturn(GDB_STRUCT_8)
rest_client_mock.Srv4ByCatalogAndCatalogname(
@@ -340,7 +340,7 @@
}
rest_client_mock = self.mox.CreateMock(rest.RestClient)
self.mox.StubOutWithMock(rest, "RestClient")
- rest.RestClient(None).AndReturn(rest_client_mock)
+ rest.RestClient(None, username=None, password=None).AndReturn(rest_client_mock)
for i, os_n in enumerate(in_catalog, 3 - len(in_catalog)):
pkg_struct = pkg_struct_map[os_n]
rest_client_mock.Srv4ByCatalogAndCatalogname(
@@ -416,7 +416,6 @@
import_metadata_mock = self.mox.StubOutWithMock(su, '_GetFileMd5sum')
import_metadata_mock = self.mox.StubOutWithMock(su, '_ImportMetadata')
import_metadata_mock = self.mox.StubOutWithMock(su, '_InsertIntoCatalog')
- import_metadata_mock = self.mox.StubOutWithMock(su, '_RemoveFromCatalog')
import_metadata_mock = self.mox.StubOutWithMock(su, '_PostFile')
import_metadata_mock = self.mox.StubOutWithMock(su, '_GetSrv4FileMetadata')
import_metadata_mock = self.mox.StubOutWithMock(su, '_MatchSrv4ToCatalogs')
Modified: csw/mgar/gar/v2-yann/lib/python/database.py
===================================================================
--- csw/mgar/gar/v2-yann/lib/python/database.py 2012-12-31 00:54:47 UTC (rev 19981)
+++ csw/mgar/gar/v2-yann/lib/python/database.py 2012-12-31 01:04:31 UTC (rev 19982)
@@ -25,8 +25,8 @@
m.Pkginst,
m.Srv4DependsOn,
m.Srv4FileInCatalog,
- m.Srv4FileStats,
- m.Srv4FileStatsBlob)
+ m.Srv4FileStatsBlob,
+ m.Srv4FileStats)
# Shouldn't this be in common_constants?
SYSTEM_PKGMAP = "/var/sadm/install/contents"
CONFIG_MTIME = "mtime"
@@ -153,7 +153,13 @@
def CreateTables(self):
for table in TABLES:
- table.createTable(ifNotExists=True)
+ try:
+ logging.debug("Creating table %r", table)
+ table.createTable(ifNotExists=True)
+ except sqlobject.dberrors.OperationalError, e:
+ logging.error("Could not create table %r: %s", table, e)
+ raise
+
def InitialDataImport(self):
"""Imports initial data into the db.
@@ -183,10 +189,6 @@
pass
self.SetDatabaseSchemaVersion()
- def CreateTables(self):
- for table in TABLES:
- table.createTable(ifNotExists=True)
-
def ClearTablesForUpdates(self):
for table in TABLES_THAT_NEED_UPDATES:
table.clearTable()
Copied: csw/mgar/gar/v2-yann/lib/python/garbage_collection.py (from rev 19981, csw/mgar/gar/v2/lib/python/garbage_collection.py)
===================================================================
--- csw/mgar/gar/v2-yann/lib/python/garbage_collection.py (rev 0)
+++ csw/mgar/gar/v2-yann/lib/python/garbage_collection.py 2012-12-31 01:04:31 UTC (rev 19982)
@@ -0,0 +1,43 @@
+#!/opt/csw/bin/python2.6
+# coding=utf-8
+#
+# $Id$
+#
+# The idea is to remove the package stats entries (and their blobs, and files)
+# for packages that aren't part of any catalogs.
+#
+# The main query can take a couple minutes. Please be careful with editing
+# this script, because if you screw up the main query, it can obliterate the
+# whole database. Make backups!
+
+import logging
+import sys
+
+import configuration
+import models as m
+from sqlobject import sqlbuilder
+
+def main():
+ configuration.SetUpSqlobjectConnection()
+ total_pkgs = m.Srv4FileStats.select().count()
+ logging.info("There are {0} packages to inspect.".format(total_pkgs))
+ res = m.Srv4FileStats.select(
+ sqlbuilder.NOTEXISTS(
+ sqlbuilder.Select(m.Srv4FileInCatalog.q.id,
+ where=(
+ sqlbuilder.Outer(m.Srv4FileStats).q.id == m.Srv4FileInCatalog.q.srv4file))
+ )
+ ).orderBy('id')
+ deleted_pkgs = 0
+ for stats in res:
+ # logging.info("Package {0} ({1}) is not in any catalogs. Removing.".format(stats.basename, stats.md5_sum))
+ stats.DeleteAllDependentObjects()
+ stats.destroySelf()
+ deleted_pkgs += 1
+ sys.stdout.write(".")
+ sys.stdout.flush()
+ logging.info("Deleted {0} unused packages.".format(deleted_pkgs))
+
+if __name__ == '__main__':
+ logging.basicConfig(level=logging.INFO)
+ main()
Modified: csw/mgar/gar/v2-yann/lib/python/inspective_package.py
===================================================================
--- csw/mgar/gar/v2-yann/lib/python/inspective_package.py 2012-12-31 00:54:47 UTC (rev 19981)
+++ csw/mgar/gar/v2-yann/lib/python/inspective_package.py 2012-12-31 01:04:31 UTC (rev 19982)
@@ -54,14 +54,21 @@
"will probably finish successfully when do you that."
% full_path)
raise package.PackageError(msg)
- if sharedlib_utils.IsBinary(file_info):
+ if sharedlib_utils.IsBinary(file_info, check_consistency=False):
parser = hachoir_parser.createParser(full_path)
if not parser:
logging.warning("Can't parse file %s", file_path)
else:
try:
+ machine_id = parser["/header/machine"].value
+ except hachoir_core.field.field.MissingField, e:
+ logging.fatal(
+ "hachoir_parser failed to retrieve machine_id for %r. "
+ "checkpkg cannot continue.",
+ file_info)
+ raise
+ try:
file_info["mime_type_by_hachoir"] = parser.mime_type
- machine_id = parser["/header/machine"].value
file_info["machine_id"] = machine_id
file_info["endian"] = parser["/header/endian"].display
except hachoir_core.field.field.MissingField, e:
Modified: csw/mgar/gar/v2-yann/lib/python/inspective_package_test.py
===================================================================
--- csw/mgar/gar/v2-yann/lib/python/inspective_package_test.py 2012-12-31 00:54:47 UTC (rev 19981)
+++ csw/mgar/gar/v2-yann/lib/python/inspective_package_test.py 2012-12-31 01:04:31 UTC (rev 19982)
@@ -1,6 +1,6 @@
#!/usr/bin/env python2.6
-import unittest
+import unittest2 as unittest
import inspective_package
import mox
import hachoir_parser
Modified: csw/mgar/gar/v2-yann/lib/python/models.py
===================================================================
--- csw/mgar/gar/v2-yann/lib/python/models.py 2012-12-31 00:54:47 UTC (rev 19981)
+++ csw/mgar/gar/v2-yann/lib/python/models.py 2012-12-31 01:04:31 UTC (rev 19982)
@@ -52,11 +52,11 @@
if self.email:
email = self.email.split("@")
else:
- email = ["unknown"]
+ email = ["unknown"]
if len(email) == 2:
username, domain = email
else:
- username, domain = email[0], "no domain"
+ username, domain = email[0], "no domain"
username = username[:-3] + "..."
return "@".join((username, domain))
Modified: csw/mgar/gar/v2-yann/lib/python/package.py
===================================================================
--- csw/mgar/gar/v2-yann/lib/python/package.py 2012-12-31 00:54:47 UTC (rev 19981)
+++ csw/mgar/gar/v2-yann/lib/python/package.py 2012-12-31 01:04:31 UTC (rev 19982)
@@ -192,10 +192,9 @@
def GetMd5sum(self):
if not self.md5sum:
logging.debug("GetMd5sum() reading file %s", repr(self.pkg_path))
- fp = open(self.pkg_path)
hash = hashlib.md5()
- hash.update(fp.read())
- fp.close()
+ with open(self.pkg_path) as fp:
+ hash.update(fp.read())
self.md5sum = hash.hexdigest()
return self.md5sum
Modified: csw/mgar/gar/v2-yann/lib/python/package_checks.py
===================================================================
--- csw/mgar/gar/v2-yann/lib/python/package_checks.py 2012-12-31 00:54:47 UTC (rev 19981)
+++ csw/mgar/gar/v2-yann/lib/python/package_checks.py 2012-12-31 01:04:31 UTC (rev 19982)
@@ -503,7 +503,10 @@
pkgmap_paths = [x["path"] for x in pkgmap]
for pkgmap_path in pkgmap_paths:
try:
- path_str = str(pkgmap_path)
+ if type(pkgmap_path) is unicode:
+ path_str = pkgmap_path.encode("utf-8")
+ else:
+ path_str = str(pkgmap_path)
if re.search(ARCH_RE, path_str):
reasons_to_be_arch_specific.append((
"archall-with-arch-paths",
Modified: csw/mgar/gar/v2-yann/lib/python/package_stats.py
===================================================================
--- csw/mgar/gar/v2-yann/lib/python/package_stats.py 2012-12-31 00:54:47 UTC (rev 19981)
+++ csw/mgar/gar/v2-yann/lib/python/package_stats.py 2012-12-31 01:04:31 UTC (rev 19982)
@@ -365,7 +365,7 @@
if stats.registered and not replace:
logging.debug(
"@classmethod ImportPkg(): "
- "Package %s is already registered. Exiting.", stats)
+ "Package %r is already registered. Exiting.", stats)
return stats
stats.RemoveAllCswFiles()
for pkgmap_entry in pkg_stats["pkgmap"]:
Modified: csw/mgar/gar/v2-yann/lib/python/rest.py
===================================================================
--- csw/mgar/gar/v2-yann/lib/python/rest.py 2012-12-31 00:54:47 UTC (rev 19981)
+++ csw/mgar/gar/v2-yann/lib/python/rest.py 2012-12-31 01:04:31 UTC (rev 19982)
@@ -74,10 +74,18 @@
"maintainer_email": pkg["maintainer_email"],
}
+ def GetCatalogList(self):
+ url = self.rest_url + self.PKGDB_APP + "/catalogs/"
+ data = urllib2.urlopen(url).read()
+ return cjson.decode(data)
+
def GetCatalog(self, catrel, arch, osrel):
if not catrel:
raise ArgumentError("Missing catalog release.")
- url = self.rest_url + self.PKGDB_APP + "/catalogs/%s/%s/%s/?quick=true" % (catrel, arch, osrel)
+ url = (
+ self.rest_url
+ + self.PKGDB_APP
+ + "/catalogs/%s/%s/%s/?quick=true" % (catrel, arch, osrel))
logging.debug("GetCatalog(): GET %s", url)
try:
data = urllib2.urlopen(url).read()
@@ -184,10 +192,6 @@
http_code,
c.getinfo(pycurl.EFFECTIVE_URL))
c.close()
- # if self.debug:
- # logging.debug("*** Headers")
- # logging.debug(h.getvalue())
- # logging.debug("*** Data")
if http_code >= 400 and http_code <= 599:
if not self.debug:
# In debug mode, all headers are printed to screen, and we aren't
Modified: csw/mgar/gar/v2-yann/lib/python/sharedlib_utils.py
===================================================================
--- csw/mgar/gar/v2-yann/lib/python/sharedlib_utils.py 2012-12-31 00:54:47 UTC (rev 19981)
+++ csw/mgar/gar/v2-yann/lib/python/sharedlib_utils.py 2012-12-31 01:04:31 UTC (rev 19982)
@@ -47,6 +47,10 @@
pass
+class DataInconsistencyError(Error):
+ """Inconsistency in the data."""
+
+
def ParseLibPath(directory):
arch_subdirs = (SPARCV8_PATHS + SPARCV8PLUS_PATHS + SPARCV9_PATHS
+ INTEL_386_PATHS + AMD64_PATHS)
@@ -213,8 +217,16 @@
return LCS
-def IsBinary(file_info):
- """Returns True or False depending on file metadata."""
+def IsBinary(file_info, check_consistency=True):
+ """Returns True or False depending on file metadata.
+
+ Args:
+ file_info: A dictionary containing file information
+ check_consistency: Whether to check for consistency between the mimetype
+ information and other information, such as machine_id. During the
+ initial data gathering, the check must be disabled, because IsBinary()
+ is used to determine whether we should collect machine_id or not.
+ """
is_a_binary = False
if "mime_type" not in file_info:
# This would be a problem in the data.
@@ -227,6 +239,13 @@
if mimetype in file_info["mime_type"]:
is_a_binary = True
break
+ if check_consistency and is_a_binary and not "machine_id" in file_info:
+ raise DataInconsistencyError(
+ "'machine_id' not found in file_info: %r. checkpkg can't continue, "
+ "but it's not a problem with checkpkg; it's a problem with the underlying "
+ "libraries. In this case it's the hachoir library, which failed to "
+ "detect the processor type for this binary."
+ % file_info)
return is_a_binary
Modified: csw/mgar/gar/v2-yann/lib/python/system_pkgmap_test.py
===================================================================
--- csw/mgar/gar/v2-yann/lib/python/system_pkgmap_test.py 2012-12-31 00:54:47 UTC (rev 19981)
+++ csw/mgar/gar/v2-yann/lib/python/system_pkgmap_test.py 2012-12-31 01:04:31 UTC (rev 19982)
@@ -1,6 +1,6 @@
#!/usr/bin/env python2.6
-import unittest
+import unittest2 as unittest
import system_pkgmap
import test_base
import models
@@ -45,6 +45,10 @@
class IndexerUnitTest(unittest.TestCase):
+ def setUp(self):
+ super(IndexerUnitTest, self).setUp()
+ self.maxDiff = None
+
def test_ParseSrv4PkginfoLine(self):
spi = system_pkgmap.Indexer()
expected = ('SUNWwpau', 'Wireless WPA Supplicant, (Usr)')
@@ -274,8 +278,8 @@
'path': '/bin', 'pkgnames': ['SUNWsystem-core-os'], 'target': './usr/bin', 'type': 's'},
{'group': 'sys', 'line': 'dev\tdir\tsystem/core-os\t\t0755\troot\tsys', 'mode': '0755', 'owner': 'root',
'path': '/dev', 'pkgnames': ['SUNWsystem-core-os'], 'target': None, 'type': 'd'}],
- 'pkginfo': {'SUNWbin': u'link system/core-os ./usr/bin',
- 'SUNWdev': u'dir system/core-os 0755 root sys',
+ 'pkginfo': {'SUNWdeveloper-solarisstudio-122-c': u'C++ Compilers',
+ 'SUNWdeveloper-versioning-sccs': u'Source Code Control System',
'SUNWpcan': u'Cisco-Aironet 802.11b driver',
'SUNWwpau': u'Wireless WPA Supplicant, (Usr)'}
}
Modified: csw/mgar/gar/v2-yann/lib/web/pkgdb_web.py
===================================================================
--- csw/mgar/gar/v2-yann/lib/web/pkgdb_web.py 2012-12-31 00:54:47 UTC (rev 19981)
+++ csw/mgar/gar/v2-yann/lib/web/pkgdb_web.py 2012-12-31 01:04:31 UTC (rev 19982)
@@ -378,8 +378,11 @@
def GET(self, catrel_name, arch_name, osrel_name, catalogname):
"""Get a srv4 reference by catalog ane catalogname."""
configuration.SetUpSqlobjectConnection()
- sqo_osrel, sqo_arch, sqo_catrel = pkgdb.GetSqoTriad(
- osrel_name, arch_name, catrel_name)
+ try:
+ sqo_osrel, sqo_arch, sqo_catrel = pkgdb.GetSqoTriad(
+ osrel_name, arch_name, catrel_name)
+ except sqlobject.main.SQLObjectNotFound:
+ raise web.notfound()
join = [
sqlbuilder.INNERJOINOn(None,
models.Srv4FileInCatalog,
@@ -401,7 +404,7 @@
web.header('Access-Control-Allow-Origin', '*')
return cjson.encode(data)
except sqlobject.main.SQLObjectNotFound:
- return cjson.encode(None)
+ raise web.notfound()
except sqlobject.dberrors.OperationalError, e:
raise web.internalerror(e)
@@ -434,6 +437,7 @@
srv4 = res.getOne()
mimetype, data = srv4.GetRestRepr()
web.header('Content-type', mimetype)
+ web.header('Access-Control-Allow-Origin', '*')
return cjson.encode(data)
except sqlobject.main.SQLObjectNotFound:
return cjson.encode(None)
Modified: csw/mgar/gar/v2-yann/lib/web/releases_web.py
===================================================================
--- csw/mgar/gar/v2-yann/lib/web/releases_web.py 2012-12-31 00:54:47 UTC (rev 19981)
+++ csw/mgar/gar/v2-yann/lib/web/releases_web.py 2012-12-31 01:04:31 UTC (rev 19982)
@@ -218,7 +218,7 @@
self.ReturnError(
"%s is not one of %s (OS releases)"
% (osrel_name, common_constants.OS_RELS))
- if osrel_name in common_constants.OBSOLETE_OS_RELS:
+ if osrel_name in common_constants.OBSOLETE_OS_RELS and catrel_name == 'unstable':
self.ReturnError(
"package deletions from an obsolete OS release such as %s "
"are not allowed" % osrel_name)
Modified: csw/mgar/gar/v2-yann/lib/web/templates/CatalogDetail.html
===================================================================
--- csw/mgar/gar/v2-yann/lib/web/templates/CatalogDetail.html 2012-12-31 00:54:47 UTC (rev 19981)
+++ csw/mgar/gar/v2-yann/lib/web/templates/CatalogDetail.html 2012-12-31 01:04:31 UTC (rev 19982)
@@ -9,13 +9,18 @@
<body>
<h3>$cat_name</h3>
<p>$pkgs.count() packages</p>
- <ul>
+<table>
+ <tr>
+ <th>filename</th>
+ <th>catalogname</th>
+ <th>version</th>
+ </tr>
$for pkg in pkgs:
- <li>
- <a href="../../srv4/$pkg.md5_sum/">
- $pkg
- </a>
- </li>
-</ul>
+ <tr>
+ <td><a href="../../srv4/$pkg.md5_sum/">$pkg.basename</a>
+ <td><a href="../../catalognames/$pkg.catalogname/" title="All packages named '$pkg.catalogname'.">$pkg.catalogname</a></td>
+ <td>$pkg.version_string</td>
+ </tr>
+</table>
</body>
</html>
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
More information about the devel
mailing list