[csw-devel] SF.net SVN: gar:[20532] csw/mgar/gar/v2/lib/python
wahwah at users.sourceforge.net
wahwah at users.sourceforge.net
Fri Mar 29 17:11:24 CET 2013
Revision: 20532
http://gar.svn.sourceforge.net/gar/?rev=20532&view=rev
Author: wahwah
Date: 2013-03-29 16:11:23 +0000 (Fri, 29 Mar 2013)
Log Message:
-----------
pkgdb: Use JSON for internal storage
Also: Increase the database schema version
JSON data in tests
Modified Paths:
--------------
csw/mgar/gar/v2/lib/python/database.py
csw/mgar/gar/v2/lib/python/models.py
csw/mgar/gar/v2/lib/python/package_stats.py
csw/mgar/gar/v2/lib/python/package_stats_test.py
csw/mgar/gar/v2/lib/python/testdata/apr_util_stats.py
csw/mgar/gar/v2/lib/python/testdata/bdb48_stats.py
csw/mgar/gar/v2/lib/python/testdata/cadaver_stats.py
csw/mgar/gar/v2/lib/python/testdata/javasvn_stats.py
csw/mgar/gar/v2/lib/python/testdata/libnet_stats.py
csw/mgar/gar/v2/lib/python/testdata/mercurial_stats.py
csw/mgar/gar/v2/lib/python/testdata/neon_stats.py
csw/mgar/gar/v2/lib/python/testdata/sudo_stats.py
csw/mgar/gar/v2/lib/python/testdata/tree_stats.py
Modified: csw/mgar/gar/v2/lib/python/database.py
===================================================================
--- csw/mgar/gar/v2/lib/python/database.py 2013-03-29 16:10:56 UTC (rev 20531)
+++ csw/mgar/gar/v2/lib/python/database.py 2013-03-29 16:11:23 UTC (rev 20532)
@@ -10,7 +10,7 @@
import system_pkgmap
CONFIG_DB_SCHEMA = "db_schema_version"
-DB_SCHEMA_VERSION = 9L
+DB_SCHEMA_VERSION = 10L
TABLES_THAT_NEED_UPDATES = (m.CswFile,)
# This list of tables is sensitive to the order in which tables are created.
Modified: csw/mgar/gar/v2/lib/python/models.py
===================================================================
--- csw/mgar/gar/v2/lib/python/models.py 2013-03-29 16:10:56 UTC (rev 20531)
+++ csw/mgar/gar/v2/lib/python/models.py 2013-03-29 16:11:23 UTC (rev 20532)
@@ -7,8 +7,17 @@
import sqlobject
import os.path
from sqlobject import sqlbuilder
+import cjson
import cPickle
+class Error(Exception):
+ """Generic error."""
+
+
+class DataError(Error):
+ """A problem with data in the database."""
+
+
class CatalogReleaseType(sqlobject.SQLObject):
"Unstable, testing, stable."
name = sqlobject.UnicodeCol(length=255, unique=True, notNone=True)
@@ -142,6 +151,7 @@
# The data structure can be missing - necessary for fake SUNW
# packages.
data_obj = sqlobject.ForeignKey('Srv4FileStatsBlob', notNone=False)
+ data_obj_mimetype = sqlobject.UnicodeCol(notNone=True, length=250)
filename_arch = sqlobject.ForeignKey('Architecture', notNone=True)
latest = sqlobject.BoolCol(notNone=True)
maintainer = sqlobject.ForeignKey('Maintainer', notNone=False)
@@ -246,7 +256,12 @@
return s
def GetStatsStruct(self):
- pkgstats = cPickle.loads(str(self.data_obj.pickle))
+ if self.data_obj_mimetype == 'application/json':
+ pkgstats = cjson.decode(str(self.data_obj.pickle))
+ elif self.data_obj_mimetype == 'application/python-pickle':
+ pkgstats = cPickle.loads(str(self.data_obj.pickle))
+ else:
+ raise DataError("Unrecognized mime type: %s" % self.data_obj_mimetype)
# There was a problem with bad utf-8 in the VENDOR field.
# This is a workaround.
if "VENDOR" in pkgstats["pkginfo"]:
@@ -308,7 +323,7 @@
'file_basename': self.basename,
'catalogname': self.catalogname,
'md5_sum': self.md5_sum,
- 'mtime': unicode(self.mtime),
+ 'mtime': self.mtime,
'rev': self.rev,
'size': self.size,
'version_string': self.version_string,
Modified: csw/mgar/gar/v2/lib/python/package_stats.py
===================================================================
--- csw/mgar/gar/v2/lib/python/package_stats.py 2013-03-29 16:10:56 UTC (rev 20531)
+++ csw/mgar/gar/v2/lib/python/package_stats.py 2013-03-29 16:11:23 UTC (rev 20532)
@@ -1,13 +1,15 @@
#!/usr/bin/env python2.6
-import cPickle
+import cjson
import copy
+import dateutil.parser
import itertools
import logging
import os
import progressbar
import mute_progressbar
import re
+import pprint
import sqlobject
import catalog
@@ -204,7 +206,8 @@
"depends": depends,
"i_depends": i_depends,
"obsoleteness_info": dir_pkg.GetObsoletedBy(),
- "isalist": sharedlib_utils.GetIsalist(arch),
+ # GetIsaList returns a frozenset, but we need a list.
+ "isalist": list(sharedlib_utils.GetIsalist(arch)),
"overrides": override_dicts,
"pkgchk": self.GetPkgchkData(),
"pkginfo": dir_pkg.GetParsedPkginfo(),
@@ -212,7 +215,10 @@
"bad_paths": dir_pkg.GetFilesContaining(BAD_CONTENT_REGEXES),
"basic_stats": basic_stats,
"files_metadata": dir_pkg.GetFilesMetadata(),
- "mtime": self.GetMtime(),
+ # Data in json must be stored using simple structures such as numbers
+ # or strings. We cannot store a datetime.datetime object, we must
+ # convert it into a string.
+ "mtime": self.GetMtime().isoformat(),
"ldd_info": dir_pkg.GetLddMinusRlines(),
"binaries_elf_info": dir_pkg.GetBinaryElfInfo(),
}
@@ -298,8 +304,12 @@
basename)
pass
# Creating the object in the database.
- data_obj = m.Srv4FileStatsBlob(
- pickle=cPickle.dumps(pkg_stats))
+ try:
+ data_obj = m.Srv4FileStatsBlob(pickle=cjson.encode(pkg_stats))
+ data_obj_mimetype = 'application/json'
+ except cjson.EncodeError as e:
+ logging.fatal(pprint.pformat(pkg_stats))
+ raise
if db_pkg_stats:
# If the database row exists already, update it.
#
@@ -310,13 +320,14 @@
db_pkg_stats.basename = pkg_stats["basic_stats"]["pkg_basename"]
db_pkg_stats.catalogname = pkg_stats["basic_stats"]["catalogname"]
db_pkg_stats.data_obj = data_obj
+ db_pkg_stats.data_obj_mimetype = data_obj_mimetype
db_pkg_stats.use_to_generate_catalogs = True
db_pkg_stats.filename_arch = filename_arch
db_pkg_stats.latest = True
db_pkg_stats.maintainer = maintainer
db_pkg_stats.md5_sum = pkg_stats["basic_stats"]["md5_sum"]
db_pkg_stats.size = pkg_stats["basic_stats"]["size"]
- db_pkg_stats.mtime = pkg_stats["mtime"]
+ db_pkg_stats.mtime = dateutil.parser.parser(pkg_stats["mtime"])
db_pkg_stats.os_rel = os_rel
db_pkg_stats.pkginst = pkginst
db_pkg_stats.registered = register
@@ -329,13 +340,14 @@
basename=pkg_stats["basic_stats"]["pkg_basename"],
catalogname=pkg_stats["basic_stats"]["catalogname"],
data_obj=data_obj,
+ data_obj_mimetype=data_obj_mimetype,
use_to_generate_catalogs=True,
filename_arch=filename_arch,
latest=True,
maintainer=maintainer,
md5_sum=pkg_stats["basic_stats"]["md5_sum"],
size=pkg_stats["basic_stats"]["size"],
- mtime=pkg_stats["mtime"],
+ mtime=dateutil.parser.parse(pkg_stats["mtime"]),
os_rel=os_rel,
pkginst=pkginst,
registered=register,
Modified: csw/mgar/gar/v2/lib/python/package_stats_test.py
===================================================================
--- csw/mgar/gar/v2/lib/python/package_stats_test.py 2013-03-29 16:10:56 UTC (rev 20531)
+++ csw/mgar/gar/v2/lib/python/package_stats_test.py 2013-03-29 16:11:23 UTC (rev 20532)
@@ -69,7 +69,7 @@
self.assertEqual(
"1234.12.11",
data_structure["basic_stats"]["parsed_basename"]["revision_info"]["REV"])
- self.assertEqual(datetime.datetime(2010, 12, 8, 7, 52, 54),
+ self.assertEqual('2010-12-08T07:52:54',
data_structure["mtime"])
Modified: csw/mgar/gar/v2/lib/python/testdata/apr_util_stats.py
===================================================================
--- csw/mgar/gar/v2/lib/python/testdata/apr_util_stats.py 2013-03-29 16:10:56 UTC (rev 20531)
+++ csw/mgar/gar/v2/lib/python/testdata/apr_util_stats.py 2013-03-29 16:11:23 UTC (rev 20532)
@@ -1,4 +1,3 @@
-import datetime
pkgstats = [{'bad_paths': {},
'basic_stats': {'catalogname': 'apr_util',
'md5_sum': 'e19f2b377884db51cc20a88ff06c8eb7',
@@ -241,7 +240,7 @@
'path': 'opt/csw/lib/apr-util-1/apr_dbd_odbc-1.so',
'runpath': ('/opt/csw/bdb47/lib', '/opt/csw/lib'),
'soname': 'apr_dbd_odbc-1.so'},
- 'mtime': datetime.datetime(2010, 8, 27, 11, 0, 10),
+ 'mtime': '2010-08-27T11:00:10',
'overrides': [],
'pkgchk': {'return_code': 0,
'stderr_lines': ['rm: Cannot remove any directory in the path of the current working directory',
Modified: csw/mgar/gar/v2/lib/python/testdata/bdb48_stats.py
===================================================================
--- csw/mgar/gar/v2/lib/python/testdata/bdb48_stats.py 2013-03-29 16:10:56 UTC (rev 20531)
+++ csw/mgar/gar/v2/lib/python/testdata/bdb48_stats.py 2013-03-29 16:11:23 UTC (rev 20532)
@@ -1,4 +1,3 @@
-import datetime
pkgstat_objs = [{'bad_paths': {},
'basic_stats': {'catalogname': 'berkeleydb48',
'md5_sum': '884e86c865b6cf30d21da6436318c289',
@@ -807,7 +806,7 @@
'opt/csw/bdb48/lib/sparcv9/libdb-4.8.so': {},
'opt/csw/bdb48/lib/sparcv9/libdb_cxx-4.8.so': {},
'opt/csw/bdb48/lib/sparcv9/libdb_java-4.8.so': {}},
- 'mtime': datetime.datetime(2010, 3, 2, 18, 9, 30),
+ 'mtime': '2010-03-02T18:09:30',
'overrides': [],
'pkgchk': {'return_code': 0,
'stderr_lines': ['rm: Cannot remove any directory in the path of the current working directory',
Modified: csw/mgar/gar/v2/lib/python/testdata/cadaver_stats.py
===================================================================
--- csw/mgar/gar/v2/lib/python/testdata/cadaver_stats.py 2013-03-29 16:10:56 UTC (rev 20531)
+++ csw/mgar/gar/v2/lib/python/testdata/cadaver_stats.py 2013-03-29 16:11:23 UTC (rev 20532)
@@ -1,4 +1,3 @@
-import datetime
pkgstats = [{'bad_paths': {},
'basic_stats': {'catalogname': 'cadaver',
'md5_sum': 'd74a2f65ef0caff0bdde7310007764a8',
@@ -146,7 +145,7 @@
'libexpat1 - XML parser toolkit, libexpat.so.1'),
('CSWlibz1',
'libz1 - Zlib data compression library, libz.so.1')],
- 'isalist': frozenset(['amd64',
+ 'isalist': (['amd64',
'i386',
'i486',
'i86',
@@ -186,7 +185,7 @@
'soname': 'libexpat.so.1',
'state': 'soname-unused',
'symbol': None}]},
- 'mtime': datetime.datetime(2012, 6, 6, 20, 21, 14),
+ 'mtime': '2012-06-06T20:21:14',
'overrides': [],
'pkgmap': [{'class': None,
'group': None,
Modified: csw/mgar/gar/v2/lib/python/testdata/javasvn_stats.py
===================================================================
--- csw/mgar/gar/v2/lib/python/testdata/javasvn_stats.py 2013-03-29 16:10:56 UTC (rev 20531)
+++ csw/mgar/gar/v2/lib/python/testdata/javasvn_stats.py 2013-03-29 16:11:23 UTC (rev 20532)
@@ -1,4 +1,3 @@
-import datetime
pkgstats = [{'bad_paths': {},
'basic_stats': {'catalogname': 'javasvn',
'md5_sum': 'd759d2536a8ecb46530a9ca218efe210',
@@ -121,7 +120,7 @@
]
}
},
- 'mtime': datetime.datetime(2010, 7, 12, 19, 6, 15),
+ 'mtime': '2010-07-12T19:06:15',
'overrides': [],
'pkgchk': {'return_code': 0,
'stderr_lines': ['rm: Cannot remove any directory in the path of the current working directory',
Modified: csw/mgar/gar/v2/lib/python/testdata/libnet_stats.py
===================================================================
--- csw/mgar/gar/v2/lib/python/testdata/libnet_stats.py 2013-03-29 16:10:56 UTC (rev 20531)
+++ csw/mgar/gar/v2/lib/python/testdata/libnet_stats.py 2013-03-29 16:11:23 UTC (rev 20532)
@@ -1,4 +1,3 @@
-import datetime
pkgstat_objs = [{'bad_paths': {},
'basic_stats': {'catalogname': 'libnet',
'md5_sum': '903c6ab7e055fd7a732f69544f84e05a',
@@ -33,7 +32,7 @@
'sparc'),
'ldd_info': {},
'binaries_elf_info': {},
- 'mtime': datetime.datetime(2008, 8, 20, 10, 26, 15),
+ 'mtime': '2008-08-20T10:26:15',
'overrides': [],
'pkgchk': {'return_code': 0,
'stderr_lines': ['rm: Cannot remove any directory in the path of the current working directory',
Modified: csw/mgar/gar/v2/lib/python/testdata/mercurial_stats.py
===================================================================
--- csw/mgar/gar/v2/lib/python/testdata/mercurial_stats.py 2013-03-29 16:10:56 UTC (rev 20531)
+++ csw/mgar/gar/v2/lib/python/testdata/mercurial_stats.py 2013-03-29 16:11:23 UTC (rev 20532)
@@ -1,4 +1,3 @@
-import datetime
pkgstat_objs = [{'bad_paths': {},
'basic_stats': {'catalogname': 'mercurial',
'md5_sum': '0783020e5628f432b92d1f97903fd15e',
@@ -693,7 +692,7 @@
'path': 'opt/csw/bin/hg'},
{'mime_type': 'text/plain; charset=us-ascii',
'path': 'opt/csw/share/doc/mercurial/license'}],
- 'isalist': frozenset(['sparc',
+ 'isalist': (['sparc',
'sparcv7',
'sparcv8',
'sparcv8-fsmuld',
@@ -738,7 +737,7 @@
'symbol table': []
},
},
- 'mtime': datetime.datetime(2011, 2, 15, 7, 46, 49),
+ 'mtime': '2011-02-15T07:46:49',
'overrides': [{'pkgname': 'CSWmercurial',
'tag_info': None,
'tag_name': 'pkgname-does-not-start-with-CSWpy-'},
Modified: csw/mgar/gar/v2/lib/python/testdata/neon_stats.py
===================================================================
--- csw/mgar/gar/v2/lib/python/testdata/neon_stats.py 2013-03-29 16:10:56 UTC (rev 20531)
+++ csw/mgar/gar/v2/lib/python/testdata/neon_stats.py 2013-03-29 16:11:23 UTC (rev 20532)
@@ -1,4 +1,3 @@
-import datetime
pkgstats = [{'bad_paths': {},
'basic_stats': {'catalogname': 'neon',
'md5_sum': 'd74a2f65ef0caff0bdde7310007764a8',
@@ -155,7 +154,7 @@
'mime_type': 'application/x-sharedlib; charset=binary',
'mime_type_by_hachoir': u'application/x-executable',
'path': 'opt/csw/lib/sparcv9/libneon.so.26.0.4'}],
- 'isalist': frozenset(['amd64',
+ 'isalist': (['amd64',
'i386',
'i486',
'i86',
@@ -191,7 +190,7 @@
'symbol table': [],
},
},
- 'mtime': datetime.datetime(2009, 9, 23, 20, 21, 14),
+ 'mtime': '2009-09-23T20:21:14',
'overrides': [],
'pkgchk': {'return_code': 0,
'stderr_lines': [],
Modified: csw/mgar/gar/v2/lib/python/testdata/sudo_stats.py
===================================================================
--- csw/mgar/gar/v2/lib/python/testdata/sudo_stats.py 2013-03-29 16:10:56 UTC (rev 20531)
+++ csw/mgar/gar/v2/lib/python/testdata/sudo_stats.py 2013-03-29 16:11:23 UTC (rev 20532)
@@ -1,4 +1,3 @@
-import datetime
pkgstats = [{'bad_paths': {},
'basic_stats': {'catalogname': 'sudo_common',
'md5_sum': '2b3bb1d2a9190b5d1813562fb2d1472a',
@@ -92,7 +91,7 @@
],
}
},
- 'mtime': datetime.datetime(2010, 3, 2, 22, 34, 40),
+ 'mtime': '2010-03-02T22:34:40',
'overrides': [],
'pkgchk': {'return_code': 0,
'stderr_lines': ['rm: Cannot remove any directory in the path of the current working directory',
@@ -328,7 +327,7 @@
]
}
},
- 'mtime': datetime.datetime(2010, 3, 2, 22, 34, 39),
+ 'mtime': '2010-03-02T22:34:39',
'overrides': [],
'pkgchk': {'return_code': 0,
'stderr_lines': ['rm: Cannot remove any directory in the path of the current working directory',
Modified: csw/mgar/gar/v2/lib/python/testdata/tree_stats.py
===================================================================
--- csw/mgar/gar/v2/lib/python/testdata/tree_stats.py 2013-03-29 16:10:56 UTC (rev 20531)
+++ csw/mgar/gar/v2/lib/python/testdata/tree_stats.py 2013-03-29 16:11:23 UTC (rev 20532)
@@ -1,4 +1,3 @@
-import datetime
pkgstats = [{'bad_paths': {},
'basic_stats': {'catalogname': 'tree',
'md5_sum': '1e43fa1c7e637b25d9356ad516ae0403',
@@ -37,7 +36,7 @@
'mime_type': 'application/x-executable; charset=binary',
'mime_type_by_hachoir': u'application/x-executable',
'path': 'opt/csw/bin/tree'}],
- 'isalist': frozenset(['sparc',
+ 'isalist': (['sparc',
'sparcv7',
'sparcv8',
'sparcv8-fsmuld',
@@ -58,7 +57,7 @@
} ]
}
},
- 'mtime': datetime.datetime(2010, 7, 5, 23, 48, 10),
+ 'mtime': '2010-07-05T23:48:10',
'overrides': [],
'pkgchk': {'return_code': 0,
'stderr_lines': [],
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
More information about the devel
mailing list