[csw-devel] SF.net SVN: gar:[13065] csw/mgar/gar/v2/lib

wahwah at users.sourceforge.net wahwah at users.sourceforge.net
Sun Jan 23 09:03:51 CET 2011


Revision: 13065
          http://gar.svn.sourceforge.net/gar/?rev=13065&view=rev
Author:   wahwah
Date:     2011-01-23 08:03:51 +0000 (Sun, 23 Jan 2011)

Log Message:
-----------
csw_upload_pkg: Publishing packages to unstable

Consists of two parts: The client-side bit uses pycurl and talks to a web
server via a REST interface.  The server side process takes care of catalog
manipulation.

This is a prototype.  It lacks many abstractions that a production application
needs to have.  Nevertheless, proves the concept.

Added Paths:
-----------
    csw/mgar/gar/v2/lib/python/csw_upload_pkg.py
    csw/mgar/gar/v2/lib/web/releases_web.py

Added: csw/mgar/gar/v2/lib/python/csw_upload_pkg.py
===================================================================
--- csw/mgar/gar/v2/lib/python/csw_upload_pkg.py	                        (rev 0)
+++ csw/mgar/gar/v2/lib/python/csw_upload_pkg.py	2011-01-23 08:03:51 UTC (rev 13065)
@@ -0,0 +1,175 @@
+#!/usr/bin/env python2.6
+
+"""csw_upload_pkg.py - uploads packages to the database.
+
+POST using pycurl code example taken from:
+http://pycurl.cvs.sourceforge.net/pycurl/pycurl/tests/test_post2.py?view=markup
+"""
+
+from StringIO import StringIO
+import pycurl
+import logging
+import optparse
+import hashlib
+import os.path
+import opencsw
+import json
+
+
+BASE_URL = "http://buildfarm.opencsw.org/releases/"
+
+
+class Error(Exception):
+  pass
+
+
+class RestCommunicationError(Error):
+  pass
+
+
+class Srv4Uploader(object):
+
+  def __init__(self, filenames, debug=False):
+    self.filenames = filenames
+    self.md5_by_filename = {}
+    self.debug = debug
+
+  def Upload(self):
+    for filename in self.filenames:
+      self._UploadFile(filename)
+
+  def _GetFileMd5sum(self, filename):
+    if filename not in self.md5_by_filename:
+      logging.debug("_GetFileMd5sum(%s): Reading the file", filename)
+      with open(filename, "rb") as fd:
+        hash = hashlib.md5()
+        hash.update(fd.read())
+        md5_sum = hash.hexdigest()
+        self.md5_by_filename[filename] = md5_sum
+    return self.md5_by_filename[filename]
+
+  def _UploadFile(self, filename):
+    md5_sum = self._GetFileMd5sum(filename)
+    file_in_allpkgs, file_metadata = self._GetSrv4FileMetadata(md5_sum)
+    if file_in_allpkgs:
+      logging.debug("File %s already uploaded.", filename)
+    else:
+      logging.debug("Uploading %s.", filename)
+      self._PostFile(filename)
+    file_in_allpkgs, file_metadata = self._GetSrv4FileMetadata(md5_sum)
+    logging.debug("file_metadata %s", repr(file_metadata))
+    self._InsertIntoCatalog(filename, file_metadata)
+
+  def _InsertIntoCatalog(self, filename, file_metadata):
+    logging.info("_InsertIntoCatalog(%s)", repr(filename))
+    md5_sum = self._GetFileMd5sum(filename)
+    basename = os.path.basename(filename)
+    parsed_basename = opencsw.ParsePackageFileName(basename)
+    logging.debug("parsed_basename: %s", parsed_basename)
+
+    arch = file_metadata['arch']
+    osrel = file_metadata['osrel']
+    url = (
+        "%scatalogs/unstable/%s/%s/%s/"
+        % (BASE_URL, arch, osrel, md5_sum))
+    logging.debug("URL: %s %s", type(url), url)
+    c = pycurl.Curl()
+    d = StringIO()
+    h = StringIO()
+    c.setopt(pycurl.URL, str(url))
+    c.setopt(pycurl.PUT, 1)
+    c.setopt(pycurl.WRITEFUNCTION, d.write)
+    c.setopt(pycurl.HEADERFUNCTION, h.write)
+    if self.debug:
+      c.setopt(c.VERBOSE, 1)
+    c.perform()
+    http_code = c.getinfo(pycurl.HTTP_CODE)
+    logging.debug(
+        "curl getinfo: %s %s %s",
+        type(http_code),
+        http_code,
+        c.getinfo(pycurl.EFFECTIVE_URL))
+    c.close()
+    if self.debug:
+      logging.debug("*** Headers")
+      logging.debug(h.getvalue())
+      logging.debug("*** Data")
+      logging.debug(d.getvalue())
+    logging.info("Response: %s", d.getvalue())
+    return http_code
+
+  def _GetSrv4FileMetadata(self, md5_sum):
+    logging.debug("_GetSrv4FileMetadata(%s)", repr(md5_sum))
+    url = BASE_URL + "srv4/" + md5_sum + "/"
+    c = pycurl.Curl()
+    d = StringIO()
+    h = StringIO()
+    c.setopt(pycurl.URL, url)
+    c.setopt(pycurl.WRITEFUNCTION, d.write)
+    c.setopt(pycurl.HEADERFUNCTION, h.write)
+    if self.debug:
+      c.setopt(c.VERBOSE, 1)
+    c.perform()
+    http_code = c.getinfo(pycurl.HTTP_CODE)
+    logging.debug(
+        "curl getinfo: %s %s %s",
+        type(http_code),
+        http_code,
+        c.getinfo(pycurl.EFFECTIVE_URL))
+    c.close()
+    if self.debug:
+      logging.debug("*** Headers")
+      logging.debug(h.getvalue())
+      logging.debug("*** Data")
+      logging.debug(d.getvalue())
+    successful = http_code >= 200 and http_code <= 299
+    metadata = None
+    if successful:
+      metadata = json.loads(d.getvalue())
+    return successful, metadata
+
+  def _PostFile(self, filename):
+    logging.info("_PostFile(%s)", repr(filename))
+    md5_sum = self._GetFileMd5sum(filename)
+    c = pycurl.Curl()
+    d = StringIO()
+    h = StringIO()
+    c.setopt(pycurl.URL, BASE_URL + "srv4/")
+    c.setopt(pycurl.POST, 1)
+    post_data = [
+        ('srv4_file', (pycurl.FORM_FILE, filename)),
+        ('submit', 'Upload'),
+        ('md5_sum', md5_sum),
+        ('basename', os.path.basename(filename)),
+    ]
+    c.setopt(pycurl.HTTPPOST, post_data)
+    c.setopt(pycurl.WRITEFUNCTION, d.write)
+    c.setopt(pycurl.HEADERFUNCTION, h.write)
+    if self.debug:
+      c.setopt(c.VERBOSE, 1)
+    c.perform()
+    http_code = c.getinfo(pycurl.HTTP_CODE)
+    c.close()
+    if self.debug:
+      logging.debug("*** Headers")
+      logging.debug(h.getvalue())
+      logging.debug("*** Data")
+      logging.debug(d.getvalue())
+    logging.debug("File POST http code: %s", http_code)
+    if http_code >= 400 and http_code <= 499:
+      raise RestCommunicationError("HTTP code: %s" % http_code)
+
+
+if __name__ == '__main__':
+  parser = optparse.OptionParser()
+  parser.add_option("-d", "--debug",
+      dest="debug",
+      default=False, action="store_true")
+  options, args = parser.parse_args()
+  print "args:", args
+  if options.debug:
+    logging.basicConfig(level=logging.DEBUG)
+  else:
+    logging.basicConfig(level=logging.INFO)
+  uploader = Srv4Uploader(args, debug=options.debug)
+  uploader.Upload()


Property changes on: csw/mgar/gar/v2/lib/python/csw_upload_pkg.py
___________________________________________________________________
Added: svn:executable
   + *

Added: csw/mgar/gar/v2/lib/web/releases_web.py
===================================================================
--- csw/mgar/gar/v2/lib/web/releases_web.py	                        (rev 0)
+++ csw/mgar/gar/v2/lib/web/releases_web.py	2011-01-23 08:03:51 UTC (rev 13065)
@@ -0,0 +1,208 @@
+#!/opt/csw/bin/python2.6
+
+# A webpy application to allow HTTP access to the checkpkg database.
+
+import web
+import sqlobject
+import json
+from lib.python import models
+from lib.python import configuration
+from lib.python import pkgdb
+from lib.python import checkpkg_lib
+from lib.python import package_stats
+import datetime
+import os
+import os.path
+import hashlib
+
+urls = (
+  r'/', 'Index',
+  r'/srv4/', 'Srv4List',
+  r'/srv4/([0-9a-f]{32})/', 'Srv4Detail',
+  # We only accept submissions into unstable.
+  # /catalogs/unstable/sparc/SunOS5.9/<md5-sum>/
+  r'/catalogs/([^/]+)/([^/]+)/([^/]+)/([0-9a-f]{32})/', 'Srv4CatalogAssignment',
+)
+
+# render = web.template.render('templates/')
+render = web.template.render('/home/maciej/src/pkgdb_web/templates/')
+
+OPENCSW_ROOT = "/home/mirror/opencsw-future"
+ALLPKGS_DIR = os.path.join(OPENCSW_ROOT, "allpkgs")
+
+def ConnectToDatabase():
+  configuration.SetUpSqlobjectConnection()
+
+
+class Index(object):
+  def GET(self):
+    return "It works!\n"
+
+class Srv4List(object):
+  def POST(self):
+    configuration.SetUpSqlobjectConnection()
+    x = web.input(srv4_file={})
+    # x['srv4_file'].filename
+    # x['srv4_file'].value
+    # x['srv4_file'].file.read()
+    web.header(
+        'Content-type',
+        'application/x-vnd.opencsw.pkg;type=upload-results')
+    hash = hashlib.md5()
+    # hash.update(x['srv4_file'].file.read())
+    hash.update(x['srv4_file'].value)
+    data_md5_sum = hash.hexdigest()
+    declared_md5_sum = x['md5_sum']
+    basename = x['basename']
+    save_attempt = False
+    if declared_md5_sum == data_md5_sum:
+      srv4 = models.Srv4FileStats.selectBy(md5_sum=data_md5_sum).getOne()
+      if srv4.use_to_generate_catalogs:
+        SaveToAllpkgs(basename, x['srv4_file'].value)
+        save_attempt = True
+    else:
+      save_attempt = False
+    response_data = {
+        "received_md5": data_md5_sum,
+        "declared_md5": declared_md5_sum,
+        "save_attempt": save_attempt,
+    }
+    return json.dumps(response_data)
+
+
+class Srv4Detail(object):
+  def GET(self, md5_sum):
+    """Allows to verify whether a given srv4 file exists."""
+    configuration.SetUpSqlobjectConnection()
+    srv4 = None
+    try:
+      srv4 = models.Srv4FileStats.selectBy(md5_sum=md5_sum).getOne()
+    except sqlobject.main.SQLObjectNotFound, e:
+      raise web.notfound()
+    # Verifying whether the package exists in allpkgs.
+    basename_in_allpkgs = os.path.join(ALLPKGS_DIR, srv4.basename)
+    if not os.path.exists(basename_in_allpkgs):
+      raise web.notfound()
+    web.header(
+        'Content-type',
+        'application/x-vnd.opencsw.pkg;type=srv4-details')
+    send_filename = "srv4-exists-%s.txt" % md5_sum
+    web.header('Content-Disposition',
+               'attachment; filename=%s' % send_filename)
+    response_data = {
+        "md5_sum": srv4.md5_sum,
+        "catalogname": srv4.catalogname,
+        "basename": srv4.basename,
+        "pkgname": srv4.pkginst.pkgname,
+        "maintainer": unicode(srv4.maintainer),
+        "arch": srv4.arch.name,
+        "osrel": srv4.os_rel.short_name,
+    }
+    return json.dumps(response_data)
+
+
+class Srv4CatalogAssignment(object):
+  def GET(self, catrel_name, arch_name, osrel_name):
+    """See if that package is in that catalog."""
+    configuration.SetUpSqlobjectConnection()
+    sqo_osrel, sqo_arch, sqo_catrel = pkgdb.GetSqoTriad(
+        osrel_name, arch_name, catrel_name)
+    srv4 = models.Srv4FileStats.selectBy(md5_sum=md5_sum).getOne()
+    srv4_in_c = models.Srv4FileInCatalog.selectBy(
+        osrel=sqo_osrel,
+        arch=sqo_arch,
+        catrel=sqo_catrel,
+        srv4file=srv4)
+    web.header(
+        'Content-type',
+        'application/x-vnd.opencsw.pkg;type=srv4-catalog-assignment')
+    response_data = {
+        'srv': unicode(srv4),
+    }
+    return json.dumps(response_data)
+
+  def PUT(self, catrel_name, arch_name, osrel_name, md5_sum):
+    """Adds package to a catalog.
+
+    When pycurl calls this function, it often hangs, waiting.  A fix for that
+    is to add the 'Content-Length' header.  However, it sometimes still gets
+    stuck and I don't know why.
+    """
+    code_version = "0x02"
+    configuration.SetUpSqlobjectConnection()
+    if catrel_name != 'unstable':
+      # Updates via web are allowed only for the unstable catalog.
+      # We should return an error message instead.
+      raise web.notfound()
+    try:
+      srv4 = models.Srv4FileStats.selectBy(md5_sum=md5_sum).getOne()
+      if not srv4.registered:
+        # Package needs to be registered for releases
+        stats = srv4.GetStatsStruct()
+        # This can throw CatalogDatabaseError if the db user doesn't have
+        # enough permissions.
+        package_stats.PackageStats.ImportPkg(stats, True)
+        srv4 = models.Srv4FileStats.selectBy(md5_sum=md5_sum).getOne()
+      c = checkpkg_lib.Catalog()
+      if arch_name == 'all':
+        # The arch='all' packages need special handling; they need to be added
+        # to both sparc and i386 catalogs.
+        archs = ('sparc', 'i386')
+      else:
+        archs = (arch_name,)
+      for arch_name in archs:
+        # See if there already is a package with that catalogname.
+        sqo_osrel, sqo_arch, sqo_catrel = pkgdb.GetSqoTriad(
+            osrel_name, arch_name, catrel_name)
+        res = c.GetConflictingSrv4ByCatalognameResult(
+            srv4, srv4.catalogname,
+            sqo_osrel, sqo_arch, sqo_catrel)
+        if res.count() == 1:
+          # Removing old version of the package from the catalog
+          for pkg_in_catalog in res:
+            srv4_to_remove = pkg_in_catalog.srv4file
+            c.RemoveSrv4(srv4_to_remove, osrel_name, arch_name, catrel_name)
+        c.AddSrv4ToCatalog(srv4, osrel_name, arch_name, catrel_name)
+        web.header(
+            'Content-type',
+            'application/x-vnd.opencsw.pkg;type=catalog-update')
+        response = json.dumps({
+          "message": "Added to catalog %s %s %s\n%s"
+                     % (catrel_name, arch_name, osrel_name, srv4.basename),
+          "code_version": code_version,
+        })
+        web.header('Content-Length', len(response))
+        return response
+    except (
+        checkpkg_lib.CatalogDatabaseError,
+        sqlobject.dberrors.OperationalError), e:
+      web.header(
+          'Content-type',
+          'application/x-vnd.opencsw.pkg;type=error-message')
+      response = json.dumps({
+        "error_message": unicode(e),
+        "code_version": code_version,
+      })
+      web.header('Content-Length', len(response))
+      return response
+
+
+def SaveToAllpkgs(basename, data):
+  """Saves a file to allpkgs."""
+  target_path = os.path.join(ALLPKGS_DIR, basename)
+  fd = None
+  try:
+    fd = os.open(target_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0644)
+    os.write(fd, data)
+  except IOError, e:
+    if fd:
+      os.close(fd)
+
+
+web.webapi.internalerror = web.debugerror
+
+app = web.application(urls, globals(), autoreload=False)
+main = app.wsgifunc()
+
+if __name__ == "__main__":
+  app.run()


Property changes on: csw/mgar/gar/v2/lib/web/releases_web.py
___________________________________________________________________
Added: svn:executable
   + *


This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.


More information about the devel mailing list