[csw-devel] SF.net SVN: gar:[20103] csw/mgar/gar/v2/lib/python/system_pkgmap.py

[wahwah at users.sourceforge.net]

Revision: 20103
          http://gar.svn.sourceforge.net/gar/?rev=20103&view=rev
Author:   wahwah
Date:     2013-01-13 11:21:08 +0000 (Sun, 13 Jan 2013)
Log Message:
-----------
pkgdb: Use marshal instead of pickle for system files

The pickle module performs cycle detection, which uses a lot of memory. We know
we don't have cycles in our data structure, so we'll use the less
resource-hungry marshal module, allowing us to run in a VM with 1GB or RAM.

Modified Paths:
--------------
    csw/mgar/gar/v2/lib/python/system_pkgmap.py

Modified: csw/mgar/gar/v2/lib/python/system_pkgmap.py
===================================================================
--- csw/mgar/gar/v2/lib/python/system_pkgmap.py	2013-01-13 10:45:40 UTC (rev 20102)
+++ csw/mgar/gar/v2/lib/python/system_pkgmap.py	2013-01-13 11:21:08 UTC (rev 20103)
@@ -6,7 +6,7 @@
 import subprocess
 import logging
 import common_constants
-import cPickle
+import marshal
 import itertools
 import progressbar
 import models as m
@@ -98,7 +98,7 @@
     if not self.arch:
       self.arch = self._GetArch()
     if not self.outfile:
-      self.outfile = ("install-contents-%s-%s.pickle"
+      self.outfile = ("install-contents-%s-%s.marshal"
                       % (self.osrel, self.arch))
     logging.debug("Indexer(): infile_contents=%s, outfile=%s, osrel=%s, arch=%s",
                   repr(self.infile_contents), repr(self.outfile), repr(self.osrel),
@@ -304,7 +304,7 @@
   def GetDataStructure(self, srv4_pkgcontent_stream, srv4_pkginfo_stream,
                        ips_pkgcontent_stream, ips_pkginfo_stream,
                        osrel, arch, show_progress=False):
-    """Gets the data structure to be pickled.
+    """Gets the data structure to be serialized.
 
     Does not interact with the OS.
     """
@@ -342,7 +342,7 @@
     data = self.Index()
     out_fd = open(self.outfile, "w")
     logging.debug("IndexAndSave(): pickling the data.")
-    cPickle.dump(data, out_fd, cPickle.HIGHEST_PROTOCOL)
+    marshal.dump(data, out_fd)
     logging.debug("IndexAndSave(): pickling done.")
 
   def _GetSrv4PkgcontentStream(self):
@@ -401,7 +401,7 @@
     return "SUNW" + "-".join(re.findall (ALPHANUMERIC_RE, ips_name))
 
 class InstallContentsImporter(object):
-  """Responsible for importing a pickled file into the database."""
+  """Responsible for importing a serialized file into the database."""
 
   def __init__(self):
     self.pkginst_cache = {}
@@ -442,7 +442,7 @@
 
   def ImportFromFile(self, in_fd, show_progress=False):
     logging.debug("Unpickling data")
-    data = cPickle.load(in_fd)
+    data = marshal.load(in_fd)
     self.ImportData(data, show_progress)
 
   def ImportData(self, data, show_progress=False, include_prefixes=None):

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.