[csw-devel] SF.net SVN: gar:[20979] csw/mgar/gar/v2/lib/python/package.py

[chninkel at users.sourceforge.net]

Revision: 20979
          http://gar.svn.sourceforge.net/gar/?rev=20979&view=rev
Author:   chninkel
Date:     2013-05-04 09:55:46 +0000 (Sat, 04 May 2013)
Log Message:
-----------
use mmap instead of reading the whole file during regex search to limit unecessary memory usage

Modified Paths:
--------------
    csw/mgar/gar/v2/lib/python/package.py

Modified: csw/mgar/gar/v2/lib/python/package.py
===================================================================
--- csw/mgar/gar/v2/lib/python/package.py	2013-05-04 09:55:23 UTC (rev 20978)
+++ csw/mgar/gar/v2/lib/python/package.py	2013-05-04 09:55:46 UTC (rev 20979)
@@ -10,6 +10,7 @@
 import subprocess
 import tempfile
 import time
+import mmap
 
 import configuration as c
 import opencsw
@@ -412,12 +413,16 @@
     full_paths = self.GetAllFilePaths()
     files_by_pattern = {}
     for full_path in full_paths:
-      content = open(self.MakeAbsolutePath(full_path), "rb").read()
-      for regex in regex_list:
-        if re.search(regex, content):
-          if regex not in files_by_pattern:
-            files_by_pattern[regex] = []
-          files_by_pattern[regex].append(full_path)
+      abs_path = self.MakeAbsolutePath(full_path)   
+      if not os.path.getsize(abs_path):
+        continue
+      with open(abs_path, "rb") as f:
+        content = mmap.mmap(f.fileno(), 0, prot=mmap.PROT_READ)
+        for regex in regex_list:
+          if re.search(regex, content):
+            if regex not in files_by_pattern:
+              files_by_pattern[regex] = []
+            files_by_pattern[regex].append(full_path)
     return files_by_pattern
 
   def MakeAbsolutePath(self, p):

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.