[csw-devel] SF.net SVN: gar:[20796] csw/mgar/gar/v2/lib
wahwah at users.sourceforge.net
wahwah at users.sourceforge.net
Thu Apr 18 17:17:17 CEST 2013
Revision: 20796
http://gar.svn.sourceforge.net/gar/?rev=20796&view=rev
Author: wahwah
Date: 2013-04-18 15:17:17 +0000 (Thu, 18 Apr 2013)
Log Message:
-----------
pkgdb: Various cleanup
Collected numerous changes that have no meritoric significance.
pkgdb: Sort imports in the rest.py file
Modified Paths:
--------------
csw/mgar/gar/v2/lib/python/checkpkg_lib.py
csw/mgar/gar/v2/lib/python/inspective_package.py
csw/mgar/gar/v2/lib/python/package.py
csw/mgar/gar/v2/lib/python/package_stats.py
csw/mgar/gar/v2/lib/python/pkgdb.py
csw/mgar/gar/v2/lib/python/rest.py
csw/mgar/gar/v2/lib/python/system_pkgmap.py
csw/mgar/gar/v2/lib/web/releases_web.py
Modified: csw/mgar/gar/v2/lib/python/checkpkg_lib.py
===================================================================
--- csw/mgar/gar/v2/lib/python/checkpkg_lib.py 2013-04-18 14:18:02 UTC (rev 20795)
+++ csw/mgar/gar/v2/lib/python/checkpkg_lib.py 2013-04-18 15:17:17 UTC (rev 20796)
@@ -1092,7 +1092,7 @@
logging.debug("AddSrv4ToCatalog(%s, %s, %s, %s, %s)",
sqo_srv4, osrel, arch, catrel, who)
if not who:
- who = 'unknown'
+ who = 'unknown'
# There are only i386 and sparc catalogs.
if arch != 'i386' and arch != 'sparc':
raise CatalogDatabaseError("Wrong architecture: %s" % arch)
Modified: csw/mgar/gar/v2/lib/python/inspective_package.py
===================================================================
--- csw/mgar/gar/v2/lib/python/inspective_package.py 2013-04-18 14:18:02 UTC (rev 20795)
+++ csw/mgar/gar/v2/lib/python/inspective_package.py 2013-04-18 15:17:17 UTC (rev 20796)
@@ -56,8 +56,8 @@
"will probably finish successfully when do you that."
% full_path)
if "/opt/csw/share" in full_path:
- file_info["mime_type"] = "application/octet-stream; fallback"
- logging.error(msg)
+ file_info["mime_type"] = "application/octet-stream; fallback"
+ logging.error(msg)
else:
raise package.PackageError(msg)
if sharedlib_utils.IsBinary(file_info, check_consistency=False):
@@ -214,7 +214,7 @@
retcode, stdout, stderr = shell.ShellCommand(args)
if retcode:
logging.error("%s returned an error: %s", args, stderr)
- # Should it just skip over an error?
+ # Should it just skip over an error?
continue
nm_out = stdout.splitlines()
@@ -300,7 +300,6 @@
cur_section = None
for line in elfdump_out:
-
try:
elf_info, cur_section = self._ParseElfdumpLine(line, cur_section)
except package.StdoutSyntaxError as e:
@@ -510,7 +509,7 @@
elfdump_data = m.groupdict()
if not m:
- raise package.StdoutSyntaxError("Could not parse %s" % (repr(line)))
+ raise package.StdoutSyntaxError("Could not parse %r" % line)
return elfdump_data, section
Modified: csw/mgar/gar/v2/lib/python/package.py
===================================================================
--- csw/mgar/gar/v2/lib/python/package.py 2013-04-18 14:18:02 UTC (rev 20795)
+++ csw/mgar/gar/v2/lib/python/package.py 2013-04-18 15:17:17 UTC (rev 20796)
@@ -35,19 +35,23 @@
proxy=
"""
+
class Error(Exception):
- pass
+ """A generic error."""
class SystemUtilityError(Error):
"""A problem occurred while running system utility, e.g. ldd."""
+
class PackageError(Error):
- pass
+ """A problem with the package itself that causes problems with examining it."""
+
class StdoutSyntaxError(Error):
- pass
+ """Problem with data returned by a process."""
+
class CswSrv4File(shell.ShellMixin, object):
"""Represents a package in the srv4 format (pkg)."""
Modified: csw/mgar/gar/v2/lib/python/package_stats.py
===================================================================
--- csw/mgar/gar/v2/lib/python/package_stats.py 2013-04-18 14:18:02 UTC (rev 20795)
+++ csw/mgar/gar/v2/lib/python/package_stats.py 2013-04-18 15:17:17 UTC (rev 20796)
@@ -55,6 +55,8 @@
class PackageStatsMixin(object):
"""Collects stats about a package and saves them.
+
+ Takes care of processing data from a SVR4 package and returing them as data structures.
"""
def __init__(self, srv4_pkg, stats_basedir=None, md5sum=None, debug=False):
@@ -223,7 +225,7 @@
"binaries_elf_info": dir_pkg.GetBinaryElfInfo(),
}
self.SaveStats(pkg_stats)
- logging.debug("Statistics of %s have been collected and saved in the db.",
+ logging.debug("_CollectStats(): Stats of %s have been collected and saved in the db.",
repr(dir_pkg.pkgname))
return pkg_stats
Modified: csw/mgar/gar/v2/lib/python/pkgdb.py
===================================================================
--- csw/mgar/gar/v2/lib/python/pkgdb.py 2013-04-18 14:18:02 UTC (rev 20795)
+++ csw/mgar/gar/v2/lib/python/pkgdb.py 2013-04-18 15:17:17 UTC (rev 20796)
@@ -202,6 +202,7 @@
if catrel not in CATALOGS_ALLOWED_TO_BE_IMPORTED:
raise UsageError("Catalogs that can be imported: %s"
% CATALOGS_ALLOWED_TO_BE_IMPORTED)
+
catalog_dir = os.path.dirname(catalog_file)
# The plan:
# - read in the catalog file, and build a md5-filename correspondence
@@ -214,16 +215,20 @@
for catalog_entry in catalog_data:
cat_entry_by_md5[catalog_entry["md5sum"]] = catalog_entry
cat_entry_by_basename[catalog_entry["file_basename"]] = catalog_entry
+
# - import all srv4 files that were not in the database so far
entries_to_import = []
+
logging.debug("Checking which srv4 files are already in the db.")
for md5 in cat_entry_by_md5:
try:
sqo_list = m.Srv4FileStats.selectBy(md5_sum=md5).getOne()
except sqlobject.main.SQLObjectNotFound, e:
entries_to_import.append(cat_entry_by_md5[md5])
+
basenames = [x["file_basename"] for x in entries_to_import]
file_list = []
+
if entries_to_import:
logging.info("Srv4 files to import:")
for basename in sorted(basenames):
@@ -242,6 +247,7 @@
for statdict in new_statdicts:
new_statdicts_by_md5[statdict["basic_stats"]["md5_sum"]] = statdict
package_stats.PackageStats.ImportPkg(statdict)
+
# - sync the specific catalog
# - find the md5 sum list of the current catalog
logging.debug("Retrieving current catalog assigments from the db.")
@@ -264,6 +270,7 @@
# Since the srv4_in_cat object has lost its reference, there's no use
# keeping it around.
srv4_in_cat.destroySelf()
+
disk_md5s = set(cat_entry_by_md5)
db_md5s = set(db_srv4s_in_cat_by_md5)
# - match the md5 sum lists between db and disk
@@ -278,12 +285,14 @@
logging.info(
" - %s",
db_srv4s_in_cat_by_md5[md5].srv4file.basename)
+
if md5_sums_to_add:
logging.info("To add to from %s %s %s:", osrel, arch, catrel)
for md5 in md5_sums_to_add:
logging.info(
" + %s",
cat_entry_by_md5[md5]["file_basename"])
+
user = getpass.getuser()
# Remove
# We could use checkpkg_lib.Catalog.RemoveSrv4(), but it would redo
@@ -292,6 +301,7 @@
logging.info("Removing assignments from the catalog.")
for md5 in md5_sums_to_remove:
db_srv4s_in_cat_by_md5[md5].destroySelf()
+
# Add
if md5_sums_to_add:
logging.info("Adding srv4 files to the %s %s %s catalog.",
@@ -497,14 +507,16 @@
c = checkpkg_lib.Catalog()
md5_sums = args[3:]
for md5_sum in md5_sums:
- logging.debug("Adding %s to the catalog", md5_sum)
try:
sqo_srv4 = m.Srv4FileStats.select(
m.Srv4FileStats.q.md5_sum==md5_sum).getOne()
+ logging.debug('Adding %s to the catalog', md5_sum)
c.AddSrv4ToCatalog(sqo_srv4, osrel, arch, catrel, who=user)
- except sqlobject.main.SQLObjectNotFound, e:
- logging.warning("Srv4 file %s was not found in the database.",
- md5_sum)
+ except sqlobject.main.SQLObjectNotFound as e:
+ logging.warning('Srv4 file %r, osrel %r, arch %r or catrel %r '
+ 'was not found in the database, '
+ 'and the package was not added: %s',
+ md5_sum, osrel, arch, catrel, e)
elif command == 'del-from-cat':
if len(args) < 4:
raise UsageError("Not enough arguments, see usage.")
Modified: csw/mgar/gar/v2/lib/python/rest.py
===================================================================
--- csw/mgar/gar/v2/lib/python/rest.py 2013-04-18 14:18:02 UTC (rev 20795)
+++ csw/mgar/gar/v2/lib/python/rest.py 2013-04-18 15:17:17 UTC (rev 20796)
@@ -1,12 +1,12 @@
#!/usr/bin/env python2.6
-import os
from StringIO import StringIO
+import anydbm
import cjson
-import anydbm
import logging
+import os
+import pycurl
import urllib2
-import pycurl
DEFAULT_URL = "http://buildfarm.opencsw.org"
RELEASES_APP = "/releases"
Modified: csw/mgar/gar/v2/lib/python/system_pkgmap.py
===================================================================
--- csw/mgar/gar/v2/lib/python/system_pkgmap.py 2013-04-18 14:18:02 UTC (rev 20795)
+++ csw/mgar/gar/v2/lib/python/system_pkgmap.py 2013-04-18 15:17:17 UTC (rev 20796)
@@ -552,20 +552,13 @@
try:
sqo_srv4 = self._GetFakeSrv4(pkgname, osrel, arch)
except sqlobject.main.SQLObjectNotFound, e:
- print d
+ logging.fatal(d)
raise
if sqo_srv4 not in cleaned_pkgs:
sqo_srv4.RemoveAllCswFiles()
cleaned_pkgs.add(sqo_srv4)
sqo_pkginst = self._GetPkginst(pkgname)
f_path, f_basename = os.path.split(d["path"])
- # This is really slow (one run ~1h), but works.
- # To speed it up, raw SQL + cursor.executemany() could be used, but
- # there's a incompatibility between MySQL and sqlite drivers:
- # MySQL: INSERT ... VALUES (%s, %s, %s);
- # sqlite: INSERT ... VALUES (?, ?, ?);
- # For now, using the sqlobject ORM which is slow, but at least
- # handles compatibility issues.
csw_file = m.CswFile(pkginst=sqo_pkginst,
line=d["line"], path=f_path, basename=f_basename,
srv4_file=sqo_srv4)
Modified: csw/mgar/gar/v2/lib/web/releases_web.py
===================================================================
--- csw/mgar/gar/v2/lib/web/releases_web.py 2013-04-18 14:18:02 UTC (rev 20795)
+++ csw/mgar/gar/v2/lib/web/releases_web.py 2013-04-18 15:17:17 UTC (rev 20796)
@@ -21,6 +21,7 @@
import hashlib
import logging
+
urls = (
r'/', 'Index',
r'/srv4/', 'Srv4List',
@@ -35,7 +36,12 @@
OPENCSW_ROOT = "/home/mirror/opencsw-official"
ALLPKGS_DIR = os.path.join(OPENCSW_ROOT, "allpkgs")
-CAN_UPLOAD_TO_CATALOGS = frozenset(["unstable", "kiel", "bratislava", "beanie"])
+CAN_UPLOAD_TO_CATALOGS = frozenset([
+ "unstable",
+ "kiel",
+ "bratislava",
+ "beanie",
+])
def ConnectToDatabase():
configuration.SetUpSqlobjectConnection()
@@ -50,14 +56,10 @@
messages = []
configuration.SetUpSqlobjectConnection()
x = web.input(srv4_file={})
- # x['srv4_file'].filename
- # x['srv4_file'].value
- # x['srv4_file'].file.read()
web.header(
'Content-type',
'application/x-vnd.opencsw.pkg;type=upload-results')
hash = hashlib.md5()
- # hash.update(x['srv4_file'].file.read())
hash.update(x['srv4_file'].value)
data_md5_sum = hash.hexdigest()
declared_md5_sum = x['md5_sum']
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
More information about the devel
mailing list