SF.net SVN: gar:[23664] csw/mgar/gar/v2/lib
wahwah at users.sourceforge.net
wahwah at users.sourceforge.net
Tue May 20 00:21:53 CEST 2014
Revision: 23664
http://sourceforge.net/p/gar/code/23664
Author: wahwah
Date: 2014-05-19 22:21:52 +0000 (Mon, 19 May 2014)
Log Message:
-----------
pkgdb: Use transactions
It's safer to bind removal+insertion operations in a single transaction - less
risk of getting the database in an inconsistent state.
Modified Paths:
--------------
csw/mgar/gar/v2/lib/python/checkpkg_lib.py
csw/mgar/gar/v2/lib/web/releases_web.py
Modified: csw/mgar/gar/v2/lib/python/checkpkg_lib.py
===================================================================
--- csw/mgar/gar/v2/lib/python/checkpkg_lib.py 2014-05-19 22:21:43 UTC (rev 23663)
+++ csw/mgar/gar/v2/lib/python/checkpkg_lib.py 2014-05-19 22:21:52 UTC (rev 23664)
@@ -1126,37 +1126,37 @@
def GetConflictingSrv4ByCatalognameResult(self,
sqo_srv4, catalogname,
- sqo_osrel, sqo_arch, sqo_catrel):
+ sqo_osrel, sqo_arch, sqo_catrel, trans):
res = m.Srv4FileStats.select(
- m.Srv4FileStats.q.catalogname==catalogname
- ).throughTo.in_catalogs.filter(
- sqlobject.AND(
- m.Srv4FileInCatalog.q.osrel==sqo_osrel,
- m.Srv4FileInCatalog.q.arch==sqo_arch,
- m.Srv4FileInCatalog.q.catrel==sqo_catrel,
- m.Srv4FileInCatalog.q.srv4file!=sqo_srv4))
+ m.Srv4FileStats.q.catalogname==catalogname,
+ forUpdate=True,
+ connection=trans
+ ).throughTo.in_catalogs.filter(
+ sqlobject.AND(
+ m.Srv4FileInCatalog.q.osrel==sqo_osrel,
+ m.Srv4FileInCatalog.q.arch==sqo_arch,
+ m.Srv4FileInCatalog.q.catrel==sqo_catrel,
+ m.Srv4FileInCatalog.q.srv4file!=sqo_srv4))
return res
def GetConflictingSrv4ByPkgnameResult(self,
sqo_srv4, pkgname,
- sqo_osrel, sqo_arch, sqo_catrel):
- join = [
- sqlbuilder.INNERJOINOn(None,
- m.Pkginst,
- m.Srv4FileStats.q.pkginst==m.Pkginst.q.id),
- ]
+ sqo_osrel, sqo_arch, sqo_catrel, trans):
+ pkginst = sqo_srv4.pkginst
res = m.Srv4FileStats.select(
- m.Pkginst.q.pkgname==pkgname,
- join=join
- ).throughTo.in_catalogs.filter(
- sqlobject.AND(
- m.Srv4FileInCatalog.q.osrel==sqo_osrel,
- m.Srv4FileInCatalog.q.arch==sqo_arch,
- m.Srv4FileInCatalog.q.catrel==sqo_catrel,
- m.Srv4FileInCatalog.q.srv4file!=sqo_srv4))
+ m.Srv4FileStats.q.pkginst==pkginst,
+ forUpdate=True,
+ connection=trans
+ ).throughTo.in_catalogs.filter(
+ sqlobject.AND(
+ m.Srv4FileInCatalog.q.osrel==sqo_osrel,
+ m.Srv4FileInCatalog.q.arch==sqo_arch,
+ m.Srv4FileInCatalog.q.catrel==sqo_catrel,
+ m.Srv4FileInCatalog.q.srv4file!=sqo_srv4))
return res
- def AddSrv4ToCatalog(self, sqo_srv4, osrel, arch, catrel, who=None):
+ def AddSrv4ToCatalog(self, sqo_srv4, osrel, arch, catrel,
+ who, trans):
"""Registers a srv4 file in a catalog."""
self.logger.debug(
"AddSrv4ToCatalog(%s, %r, %r, %r, %r)",
@@ -1177,34 +1177,28 @@
raise CatalogDatabaseError(
"Package %s (%s) is not registered for releases."
% (sqo_srv4.basename, sqo_srv4.md5_sum))
- # TODO(maciej): Make sure the package's files are present in the database.
# Checking for presence of a different srv4 with the same pkginst in the
# same catalog
- pkginst = sqo_srv4.pkginst
- res = m.Srv4FileStats.select(
- m.Srv4FileStats.q.pkginst==pkginst).throughTo.in_catalogs.filter(
- sqlobject.AND(
- m.Srv4FileInCatalog.q.osrel==sqo_osrel,
- m.Srv4FileInCatalog.q.arch==sqo_arch,
- m.Srv4FileInCatalog.q.catrel==sqo_catrel,
- m.Srv4FileInCatalog.q.srv4file!=sqo_srv4))
+ res = self.GetConflictingSrv4ByPkgnameResult(
+ sqo_srv4, sqo_srv4.pkginst.pkgname, sqo_osrel, sqo_arch, sqo_catrel, trans)
if res.count():
raise CatalogDatabaseError(
"There already is a package with that pkgname: %s" % pkginst.pkgname)
res = self.GetConflictingSrv4ByCatalognameResult(
sqo_srv4, sqo_srv4.catalogname,
- sqo_osrel, sqo_arch, sqo_catrel)
+ sqo_osrel, sqo_arch, sqo_catrel, trans)
if res.count():
raise CatalogDatabaseError(
"There already is a package with that catalogname: %s"
- % sqo_srv4.catalogname)
+ % sqo_srv4)
# Checking for presence of the same srv4 already in the catalog.
res = m.Srv4FileInCatalog.select(
sqlobject.AND(
m.Srv4FileInCatalog.q.osrel==sqo_osrel,
m.Srv4FileInCatalog.q.arch==sqo_arch,
m.Srv4FileInCatalog.q.catrel==sqo_catrel,
- m.Srv4FileInCatalog.q.srv4file==sqo_srv4))
+ m.Srv4FileInCatalog.q.srv4file==sqo_srv4),
+ connection=trans)
if res.count():
self.logger.debug(
"%s is already part of %s %s %s, count=%d",
@@ -1212,34 +1206,34 @@
# Our srv4 is already part of that catalog.
return
# SQL INSERT happens here.
- m.Srv4FileInCatalog(
+ pkg_in_cat = m.Srv4FileInCatalog(
arch=sqo_arch,
osrel=sqo_osrel,
catrel=sqo_catrel,
srv4file=sqo_srv4,
- created_by=who)
- # The package is now in the catalog.
+ created_by=who,
+ connection=trans)
self.logger.debug('The package %s was inserted into catalog %s %s %s: %s',
sqo_srv4.basename, osrel, arch, catrel, pkg_in_cat)
- def RemoveSrv4(self, sqo_srv4, osrel, arch, catrel):
+ def RemoveSrv4(self, sqo_srv4, osrel, arch, catrel, trans):
catspec = CatalogSpec(catrel, arch, osrel)
self.logger.debug('RemoveSrv4(), %s %s' % (sqo_srv4, catspec))
sqo_osrel, sqo_arch, sqo_catrel = self.GetSqlobjectTriad(
osrel, arch, catrel)
try:
- # There's a race condition in here. Maybe SQLObject allows to delete
- # atomically?
sqo_srv4_in_cat = m.Srv4FileInCatalog.select(
sqlobject.AND(
m.Srv4FileInCatalog.q.arch==sqo_arch,
m.Srv4FileInCatalog.q.osrel==sqo_osrel,
m.Srv4FileInCatalog.q.catrel==sqo_catrel,
- m.Srv4FileInCatalog.q.srv4file==sqo_srv4)).getOne()
+ m.Srv4FileInCatalog.q.srv4file==sqo_srv4),
+ forUpdate=True,
+ connection=trans).getOne()
+ sqo_srv4_in_cat.destroySelf()
self.logger.debug('Package %s should be now gone from %s.' % (sqo_srv4, catspec))
# Files belonging to this package should not be removed from the catalog
# as the package might be still present in another catalog.
- sqo_srv4_in_cat.destroySelf()
except sqlobject.main.SQLObjectNotFound as e:
self.logger.debug('The object went away when we were trying to delete it.')
self.logger.warning(e)
Modified: csw/mgar/gar/v2/lib/web/releases_web.py
===================================================================
--- csw/mgar/gar/v2/lib/web/releases_web.py 2014-05-19 22:21:43 UTC (rev 23663)
+++ csw/mgar/gar/v2/lib/web/releases_web.py 2014-05-19 22:21:52 UTC (rev 23664)
@@ -11,6 +11,8 @@
import sys
import tempfile
+from contextlib import contextmanager
+
import cjson
import lockfile
import sqlobject
@@ -73,6 +75,20 @@
applogger.addHandler(log_handler)
+ at contextmanager
+def Transaction(cls):
+ """Provide a transactional scope around a series of operations."""
+ transaction = cls._connection.transaction()
+ try:
+ yield transaction
+ transaction.commit()
+ except:
+ transaction.rollback()
+ raise
+ finally:
+ transaction.commit(close=True)
+
+
class Index(object):
def GET(self):
@@ -291,46 +307,53 @@
c = checkpkg_lib.Catalog()
sqo_osrel, sqo_arch, sqo_catrel = models.GetSqoTriad(
osrel_name, arch_name, catrel_name)
- applogger.info('See if there already is a package with that '
- 'catalogname (%s)', srv4.basename)
- res = c.GetConflictingSrv4ByCatalognameResult(
- srv4, srv4.catalogname,
- sqo_osrel, sqo_arch, sqo_catrel)
- if res.count() >= 1:
- for pkg_in_catalog in res:
- srv4_to_remove = pkg_in_catalog.srv4file
- applogger.info('Removing %s from the %s catalog',
- srv4_to_remove.catalogname, catspec)
- c.RemoveSrv4(srv4_to_remove, osrel_name, arch_name, catrel_name)
- else:
- applogger.info('Package with the same catalogname (%s) not found in %s (good)',
- srv4.catalogname, catspec)
- # See if there already is a package with that pkgname.
- res = c.GetConflictingSrv4ByPkgnameResult(
- srv4, srv4.pkginst.pkgname,
- sqo_osrel, sqo_arch, sqo_catrel)
- if res.count() >= 1:
- # Removing old version of the package from the catalog
- for pkg_in_catalog in res:
- srv4_to_remove = pkg_in_catalog.srv4file
- applogger.info('Removing %s from %s', srv4_to_remove.basename, catspec)
- c.RemoveSrv4(srv4_to_remove, osrel_name, arch_name, catrel_name)
+ with Transaction(models.Srv4FileStats) as trans:
+ applogger.info('Looking if catalog %s already contains a package '
+ 'with catalogname %s', catspec, srv4.catalogname)
+ res = c.GetConflictingSrv4ByCatalognameResult(
+ srv4, srv4.catalogname,
+ sqo_osrel, sqo_arch, sqo_catrel,
+ trans)
+ if res.count() >= 1:
+ for pkg_in_catalog in res:
+ srv4_to_remove = pkg_in_catalog.srv4file
+ applogger.info('Removing previous %s catalogname from the %s catalog, '
+ 'matched by catalogname',
+ srv4_to_remove.catalogname, catspec)
+ c.RemoveSrv4(srv4_to_remove, osrel_name, arch_name, catrel_name, trans)
+ else:
+ applogger.info('Package with the same catalogname (%s) not found in %s',
+ srv4.catalogname, catspec)
+ # See if there already is a package with that pkgname.
+ res = c.GetConflictingSrv4ByPkgnameResult(
+ srv4, srv4.pkginst.pkgname,
+ sqo_osrel, sqo_arch, sqo_catrel,
+ trans)
+ if res.count() >= 1:
+ # Removing old version of the package from the catalog
+ for pkg_in_catalog in res:
+ srv4_to_remove = pkg_in_catalog.srv4file
+ applogger.info('Removing %s from %s', srv4_to_remove.basename, catspec)
+ c.RemoveSrv4(srv4_to_remove, osrel_name, arch_name, catrel_name, trans)
+ else:
+ applogger.info('Package with the same pkgname (%s) not found in %s',
+ srv4.pkginst.pkgname, catspec)
- # This is set by basic HTTP auth.
- username = web.ctx.env.get('REMOTE_USER')
+ # This is set by basic HTTP auth.
+ username = web.ctx.env.get('REMOTE_USER')
- applogger.info('Adding %s to the %s catalog', srv4.basename, catspec)
- c.AddSrv4ToCatalog(srv4, osrel_name, arch_name, catrel_name, who=username)
- web.header(
- 'Content-type',
- 'application/x-vnd.opencsw.pkg;type=catalog-update')
- response = cjson.encode([
- u"Added to catalog %s %s %s" % (catrel_name, arch_name, osrel_name),
- u"%s" % srv4.basename,
- u"%s" % srv4.md5_sum,
- ])
- web.header('Content-Length', len(response))
- return response
+ applogger.info('Adding %s to the %s catalog', srv4.basename, catspec)
+ c.AddSrv4ToCatalog(srv4, osrel_name, arch_name, catrel_name, username, trans)
+ web.header(
+ 'Content-type',
+ 'application/x-vnd.opencsw.pkg;type=catalog-update')
+ response = cjson.encode([
+ u"Added to catalog %s %s %s" % (catrel_name, arch_name, osrel_name),
+ u"%s" % srv4.basename,
+ u"%s" % srv4.md5_sum,
+ ])
+ web.header('Content-Length', len(response))
+ return response
except (
checkpkg_lib.CatalogDatabaseError,
sqlobject.dberrors.OperationalError) as exc:
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
More information about the devel
mailing list