From 0338e2af6cfc24be3631b84c6ed44ca21e56bbef Mon Sep 17 00:00:00 2001
From: Richard Purdie <richard.purdie@linuxfoundation.org>
Date: Sun, 11 Mar 2012 14:18:02 +0000
Subject: [PATCH] codeparser: Compute extra cache components on the fly rather
 than later

In the initial implementation there was a relcutance on my part to
generate incremental cache components on the fly since it would lead
to some duplicate code.

We are now seeing problems where each thread reading in the saved cache
file causes significant overhead and can make the process appear to hang
on a many core build, particularly when the cache file is large.

This patch changes the code to maintain the delta in a separate dict
right from the start. The code duplication isn't too bad and could be
mitigated in other ways if it becomes an issue.

[YOCTO #2039 partial]

(Bitbake rev: cdd5d0dee6ab12326b252b6b505a316a52638cac)

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 bitbake/lib/bb/codeparser.py | 41 ++++++++++++++++++++---------------------
 1 file changed, 20 insertions(+), 21 deletions(-)

diff --git a/bitbake/lib/bb/codeparser.py b/bitbake/lib/bb/codeparser.py
index 2590e5c..04a34f9 100644
--- a/bitbake/lib/bb/codeparser.py
+++ b/bitbake/lib/bb/codeparser.py
@@ -34,6 +34,9 @@ def check_indent(codestr):
 
 pythonparsecache = {}
 shellparsecache = {}
+pythonparsecacheextras = {}
+shellparsecacheextras = {}
+
 
 def parser_cachefile(d):
     cachedir = (d.getVar("PERSISTENT_DIR", True) or
@@ -86,22 +89,8 @@ def parser_cache_save(d):
             i = i + 1
             continue
 
-        try:
-            p = pickle.Unpickler(file(cachefile, "rb"))
-            data, version = p.load()
-        except (IOError, EOFError, ValueError):
-            data, version = None, None
-
-        if version != PARSERCACHE_VERSION:
-            shellcache = shellparsecache
-            pythoncache = pythonparsecache
-        else:
-            for h in pythonparsecache:
-                if h not in data[0]:
-                    pythoncache[h] = pythonparsecache[h]
-            for h in shellparsecache:
-                if h not in data[1]:
-                    shellcache[h] = shellparsecache[h]
+        shellcache = shellparsecacheextras
+        pythoncache = pythonparsecacheextras
 
         p = pickle.Pickler(file(cachefile + "-" + str(i), "wb"), -1)
         p.dump([[pythoncache, shellcache], PARSERCACHE_VERSION])
@@ -230,6 +219,12 @@ class PythonParser():
             self.execs = pythonparsecache[h]["execs"]
             return
 
+        if h in pythonparsecacheextras:
+            self.references = pythonparsecacheextras[h]["refs"]
+            self.execs = pythonparsecacheextras[h]["execs"]
+            return
+
+
         code = compile(check_indent(str(node)), "<string>", "exec",
                        ast.PyCF_ONLY_AST)
 
@@ -240,9 +235,9 @@ class PythonParser():
         self.references.update(self.var_references)
         self.references.update(self.var_execs)
 
-        pythonparsecache[h] = {}
-        pythonparsecache[h]["refs"] = self.references
-        pythonparsecache[h]["execs"] = self.execs
+        pythonparsecacheextras[h] = {}
+        pythonparsecacheextras[h]["refs"] = self.references
+        pythonparsecacheextras[h]["execs"] = self.execs
 
 class ShellParser():
     def __init__(self, name, log):
@@ -264,6 +259,10 @@ class ShellParser():
             self.execs = shellparsecache[h]["execs"]
             return self.execs
 
+        if h in shellparsecacheextras:
+            self.execs = shellparsecacheextras[h]["execs"]
+            return self.execs
+
         try:
             tokens, _ = pyshyacc.parse(value, eof=True, debug=False)
         except pyshlex.NeedMore:
@@ -273,8 +272,8 @@ class ShellParser():
             self.process_tokens(token)
         self.execs = set(cmd for cmd in self.allexecs if cmd not in self.funcdefs)
 
-        shellparsecache[h] = {}
-        shellparsecache[h]["execs"] = self.execs
+        shellparsecacheextras[h] = {}
+        shellparsecacheextras[h]["execs"] = self.execs
 
         return self.execs
 
-- 
2.7.4