[csw-devel] SF.net SVN: gar:[13065] csw/mgar/gar/v2/lib
wahwah at users.sourceforge.net
wahwah at users.sourceforge.net
Sun Jan 23 09:03:51 CET 2011
Revision: 13065
http://gar.svn.sourceforge.net/gar/?rev=13065&view=rev
Author: wahwah
Date: 2011-01-23 08:03:51 +0000 (Sun, 23 Jan 2011)
Log Message:
-----------
csw_upload_pkg: Publishing packages to unstable
Consists of two parts: The client-side bit uses pycurl and talks to a web
server via a REST interface. The server side process takes care of catalog
manipulation.
This is a prototype. It lacks many abstractions that a production application
needs to have. Nevertheless, proves the concept.
Added Paths:
-----------
csw/mgar/gar/v2/lib/python/csw_upload_pkg.py
csw/mgar/gar/v2/lib/web/releases_web.py
Added: csw/mgar/gar/v2/lib/python/csw_upload_pkg.py
===================================================================
--- csw/mgar/gar/v2/lib/python/csw_upload_pkg.py (rev 0)
+++ csw/mgar/gar/v2/lib/python/csw_upload_pkg.py 2011-01-23 08:03:51 UTC (rev 13065)
@@ -0,0 +1,175 @@
+#!/usr/bin/env python2.6
+
+"""csw_upload_pkg.py - uploads packages to the database.
+
+POST using pycurl code example taken from:
+http://pycurl.cvs.sourceforge.net/pycurl/pycurl/tests/test_post2.py?view=markup
+"""
+
+from StringIO import StringIO
+import pycurl
+import logging
+import optparse
+import hashlib
+import os.path
+import opencsw
+import json
+
+
+BASE_URL = "http://buildfarm.opencsw.org/releases/"
+
+
+class Error(Exception):
+ pass
+
+
+class RestCommunicationError(Error):
+ pass
+
+
+class Srv4Uploader(object):
+
+ def __init__(self, filenames, debug=False):
+ self.filenames = filenames
+ self.md5_by_filename = {}
+ self.debug = debug
+
+ def Upload(self):
+ for filename in self.filenames:
+ self._UploadFile(filename)
+
+ def _GetFileMd5sum(self, filename):
+ if filename not in self.md5_by_filename:
+ logging.debug("_GetFileMd5sum(%s): Reading the file", filename)
+ with open(filename, "rb") as fd:
+ hash = hashlib.md5()
+ hash.update(fd.read())
+ md5_sum = hash.hexdigest()
+ self.md5_by_filename[filename] = md5_sum
+ return self.md5_by_filename[filename]
+
+ def _UploadFile(self, filename):
+ md5_sum = self._GetFileMd5sum(filename)
+ file_in_allpkgs, file_metadata = self._GetSrv4FileMetadata(md5_sum)
+ if file_in_allpkgs:
+ logging.debug("File %s already uploaded.", filename)
+ else:
+ logging.debug("Uploading %s.", filename)
+ self._PostFile(filename)
+ file_in_allpkgs, file_metadata = self._GetSrv4FileMetadata(md5_sum)
+ logging.debug("file_metadata %s", repr(file_metadata))
+ self._InsertIntoCatalog(filename, file_metadata)
+
+ def _InsertIntoCatalog(self, filename, file_metadata):
+ logging.info("_InsertIntoCatalog(%s)", repr(filename))
+ md5_sum = self._GetFileMd5sum(filename)
+ basename = os.path.basename(filename)
+ parsed_basename = opencsw.ParsePackageFileName(basename)
+ logging.debug("parsed_basename: %s", parsed_basename)
+
+ arch = file_metadata['arch']
+ osrel = file_metadata['osrel']
+ url = (
+ "%scatalogs/unstable/%s/%s/%s/"
+ % (BASE_URL, arch, osrel, md5_sum))
+ logging.debug("URL: %s %s", type(url), url)
+ c = pycurl.Curl()
+ d = StringIO()
+ h = StringIO()
+ c.setopt(pycurl.URL, str(url))
+ c.setopt(pycurl.PUT, 1)
+ c.setopt(pycurl.WRITEFUNCTION, d.write)
+ c.setopt(pycurl.HEADERFUNCTION, h.write)
+ if self.debug:
+ c.setopt(c.VERBOSE, 1)
+ c.perform()
+ http_code = c.getinfo(pycurl.HTTP_CODE)
+ logging.debug(
+ "curl getinfo: %s %s %s",
+ type(http_code),
+ http_code,
+ c.getinfo(pycurl.EFFECTIVE_URL))
+ c.close()
+ if self.debug:
+ logging.debug("*** Headers")
+ logging.debug(h.getvalue())
+ logging.debug("*** Data")
+ logging.debug(d.getvalue())
+ logging.info("Response: %s", d.getvalue())
+ return http_code
+
+ def _GetSrv4FileMetadata(self, md5_sum):
+ logging.debug("_GetSrv4FileMetadata(%s)", repr(md5_sum))
+ url = BASE_URL + "srv4/" + md5_sum + "/"
+ c = pycurl.Curl()
+ d = StringIO()
+ h = StringIO()
+ c.setopt(pycurl.URL, url)
+ c.setopt(pycurl.WRITEFUNCTION, d.write)
+ c.setopt(pycurl.HEADERFUNCTION, h.write)
+ if self.debug:
+ c.setopt(c.VERBOSE, 1)
+ c.perform()
+ http_code = c.getinfo(pycurl.HTTP_CODE)
+ logging.debug(
+ "curl getinfo: %s %s %s",
+ type(http_code),
+ http_code,
+ c.getinfo(pycurl.EFFECTIVE_URL))
+ c.close()
+ if self.debug:
+ logging.debug("*** Headers")
+ logging.debug(h.getvalue())
+ logging.debug("*** Data")
+ logging.debug(d.getvalue())
+ successful = http_code >= 200 and http_code <= 299
+ metadata = None
+ if successful:
+ metadata = json.loads(d.getvalue())
+ return successful, metadata
+
+ def _PostFile(self, filename):
+ logging.info("_PostFile(%s)", repr(filename))
+ md5_sum = self._GetFileMd5sum(filename)
+ c = pycurl.Curl()
+ d = StringIO()
+ h = StringIO()
+ c.setopt(pycurl.URL, BASE_URL + "srv4/")
+ c.setopt(pycurl.POST, 1)
+ post_data = [
+ ('srv4_file', (pycurl.FORM_FILE, filename)),
+ ('submit', 'Upload'),
+ ('md5_sum', md5_sum),
+ ('basename', os.path.basename(filename)),
+ ]
+ c.setopt(pycurl.HTTPPOST, post_data)
+ c.setopt(pycurl.WRITEFUNCTION, d.write)
+ c.setopt(pycurl.HEADERFUNCTION, h.write)
+ if self.debug:
+ c.setopt(c.VERBOSE, 1)
+ c.perform()
+ http_code = c.getinfo(pycurl.HTTP_CODE)
+ c.close()
+ if self.debug:
+ logging.debug("*** Headers")
+ logging.debug(h.getvalue())
+ logging.debug("*** Data")
+ logging.debug(d.getvalue())
+ logging.debug("File POST http code: %s", http_code)
+ if http_code >= 400 and http_code <= 499:
+ raise RestCommunicationError("HTTP code: %s" % http_code)
+
+
+if __name__ == '__main__':
+ parser = optparse.OptionParser()
+ parser.add_option("-d", "--debug",
+ dest="debug",
+ default=False, action="store_true")
+ options, args = parser.parse_args()
+ print "args:", args
+ if options.debug:
+ logging.basicConfig(level=logging.DEBUG)
+ else:
+ logging.basicConfig(level=logging.INFO)
+ uploader = Srv4Uploader(args, debug=options.debug)
+ uploader.Upload()
Property changes on: csw/mgar/gar/v2/lib/python/csw_upload_pkg.py
___________________________________________________________________
Added: svn:executable
+ *
Added: csw/mgar/gar/v2/lib/web/releases_web.py
===================================================================
--- csw/mgar/gar/v2/lib/web/releases_web.py (rev 0)
+++ csw/mgar/gar/v2/lib/web/releases_web.py 2011-01-23 08:03:51 UTC (rev 13065)
@@ -0,0 +1,208 @@
+#!/opt/csw/bin/python2.6
+
+# A webpy application to allow HTTP access to the checkpkg database.
+
+import web
+import sqlobject
+import json
+from lib.python import models
+from lib.python import configuration
+from lib.python import pkgdb
+from lib.python import checkpkg_lib
+from lib.python import package_stats
+import datetime
+import os
+import os.path
+import hashlib
+
+urls = (
+ r'/', 'Index',
+ r'/srv4/', 'Srv4List',
+ r'/srv4/([0-9a-f]{32})/', 'Srv4Detail',
+ # We only accept submissions into unstable.
+ # /catalogs/unstable/sparc/SunOS5.9/<md5-sum>/
+ r'/catalogs/([^/]+)/([^/]+)/([^/]+)/([0-9a-f]{32})/', 'Srv4CatalogAssignment',
+)
+
+# render = web.template.render('templates/')
+render = web.template.render('/home/maciej/src/pkgdb_web/templates/')
+
+OPENCSW_ROOT = "/home/mirror/opencsw-future"
+ALLPKGS_DIR = os.path.join(OPENCSW_ROOT, "allpkgs")
+
+def ConnectToDatabase():
+ configuration.SetUpSqlobjectConnection()
+
+
+class Index(object):
+ def GET(self):
+ return "It works!\n"
+
+class Srv4List(object):
+ def POST(self):
+ configuration.SetUpSqlobjectConnection()
+ x = web.input(srv4_file={})
+ # x['srv4_file'].filename
+ # x['srv4_file'].value
+ # x['srv4_file'].file.read()
+ web.header(
+ 'Content-type',
+ 'application/x-vnd.opencsw.pkg;type=upload-results')
+ hash = hashlib.md5()
+ # hash.update(x['srv4_file'].file.read())
+ hash.update(x['srv4_file'].value)
+ data_md5_sum = hash.hexdigest()
+ declared_md5_sum = x['md5_sum']
+ basename = x['basename']
+ save_attempt = False
+ if declared_md5_sum == data_md5_sum:
+ srv4 = models.Srv4FileStats.selectBy(md5_sum=data_md5_sum).getOne()
+ if srv4.use_to_generate_catalogs:
+ SaveToAllpkgs(basename, x['srv4_file'].value)
+ save_attempt = True
+ else:
+ save_attempt = False
+ response_data = {
+ "received_md5": data_md5_sum,
+ "declared_md5": declared_md5_sum,
+ "save_attempt": save_attempt,
+ }
+ return json.dumps(response_data)
+
+
+class Srv4Detail(object):
+ def GET(self, md5_sum):
+ """Allows to verify whether a given srv4 file exists."""
+ configuration.SetUpSqlobjectConnection()
+ srv4 = None
+ try:
+ srv4 = models.Srv4FileStats.selectBy(md5_sum=md5_sum).getOne()
+ except sqlobject.main.SQLObjectNotFound, e:
+ raise web.notfound()
+ # Verifying whether the package exists in allpkgs.
+ basename_in_allpkgs = os.path.join(ALLPKGS_DIR, srv4.basename)
+ if not os.path.exists(basename_in_allpkgs):
+ raise web.notfound()
+ web.header(
+ 'Content-type',
+ 'application/x-vnd.opencsw.pkg;type=srv4-details')
+ send_filename = "srv4-exists-%s.txt" % md5_sum
+ web.header('Content-Disposition',
+ 'attachment; filename=%s' % send_filename)
+ response_data = {
+ "md5_sum": srv4.md5_sum,
+ "catalogname": srv4.catalogname,
+ "basename": srv4.basename,
+ "pkgname": srv4.pkginst.pkgname,
+ "maintainer": unicode(srv4.maintainer),
+ "arch": srv4.arch.name,
+ "osrel": srv4.os_rel.short_name,
+ }
+ return json.dumps(response_data)
+
+
+class Srv4CatalogAssignment(object):
+ def GET(self, catrel_name, arch_name, osrel_name):
+ """See if that package is in that catalog."""
+ configuration.SetUpSqlobjectConnection()
+ sqo_osrel, sqo_arch, sqo_catrel = pkgdb.GetSqoTriad(
+ osrel_name, arch_name, catrel_name)
+ srv4 = models.Srv4FileStats.selectBy(md5_sum=md5_sum).getOne()
+ srv4_in_c = models.Srv4FileInCatalog.selectBy(
+ osrel=sqo_osrel,
+ arch=sqo_arch,
+ catrel=sqo_catrel,
+ srv4file=srv4)
+ web.header(
+ 'Content-type',
+ 'application/x-vnd.opencsw.pkg;type=srv4-catalog-assignment')
+ response_data = {
+ 'srv': unicode(srv4),
+ }
+ return json.dumps(response_data)
+
+ def PUT(self, catrel_name, arch_name, osrel_name, md5_sum):
+ """Adds package to a catalog.
+
+ When pycurl calls this function, it often hangs, waiting. A fix for that
+ is to add the 'Content-Length' header. However, it sometimes still gets
+ stuck and I don't know why.
+ """
+ code_version = "0x02"
+ configuration.SetUpSqlobjectConnection()
+ if catrel_name != 'unstable':
+ # Updates via web are allowed only for the unstable catalog.
+ # We should return an error message instead.
+ raise web.notfound()
+ try:
+ srv4 = models.Srv4FileStats.selectBy(md5_sum=md5_sum).getOne()
+ if not srv4.registered:
+ # Package needs to be registered for releases
+ stats = srv4.GetStatsStruct()
+ # This can throw CatalogDatabaseError if the db user doesn't have
+ # enough permissions.
+ package_stats.PackageStats.ImportPkg(stats, True)
+ srv4 = models.Srv4FileStats.selectBy(md5_sum=md5_sum).getOne()
+ c = checkpkg_lib.Catalog()
+ if arch_name == 'all':
+ # The arch='all' packages need special handling; they need to be added
+ # to both sparc and i386 catalogs.
+ archs = ('sparc', 'i386')
+ else:
+ archs = (arch_name,)
+ for arch_name in archs:
+ # See if there already is a package with that catalogname.
+ sqo_osrel, sqo_arch, sqo_catrel = pkgdb.GetSqoTriad(
+ osrel_name, arch_name, catrel_name)
+ res = c.GetConflictingSrv4ByCatalognameResult(
+ srv4, srv4.catalogname,
+ sqo_osrel, sqo_arch, sqo_catrel)
+ if res.count() == 1:
+ # Removing old version of the package from the catalog
+ for pkg_in_catalog in res:
+ srv4_to_remove = pkg_in_catalog.srv4file
+ c.RemoveSrv4(srv4_to_remove, osrel_name, arch_name, catrel_name)
+ c.AddSrv4ToCatalog(srv4, osrel_name, arch_name, catrel_name)
+ web.header(
+ 'Content-type',
+ 'application/x-vnd.opencsw.pkg;type=catalog-update')
+ response = json.dumps({
+ "message": "Added to catalog %s %s %s\n%s"
+ % (catrel_name, arch_name, osrel_name, srv4.basename),
+ "code_version": code_version,
+ })
+ web.header('Content-Length', len(response))
+ return response
+ except (
+ checkpkg_lib.CatalogDatabaseError,
+ sqlobject.dberrors.OperationalError), e:
+ web.header(
+ 'Content-type',
+ 'application/x-vnd.opencsw.pkg;type=error-message')
+ response = json.dumps({
+ "error_message": unicode(e),
+ "code_version": code_version,
+ })
+ web.header('Content-Length', len(response))
+ return response
+
+
+def SaveToAllpkgs(basename, data):
+ """Saves a file to allpkgs."""
+ target_path = os.path.join(ALLPKGS_DIR, basename)
+ fd = None
+ try:
+ fd = os.open(target_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0644)
+ os.write(fd, data)
+ except IOError, e:
+ if fd:
+ os.close(fd)
+
+
+web.webapi.internalerror = web.debugerror
+
+app = web.application(urls, globals(), autoreload=False)
+main = app.wsgifunc()
+
+if __name__ == "__main__":
+ app.run()
Property changes on: csw/mgar/gar/v2/lib/web/releases_web.py
___________________________________________________________________
Added: svn:executable
+ *
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
More information about the devel
mailing list