add db cache

Author: fstachura

elixir/data.py

@@ -18,6 +18,7 @@
 #  You should have received a copy of the GNU Affero General Public License
 #  along with Elixir.  If not, see <http://www.gnu.org/licenses/>.
 
+from typing import OrderedDict
 import berkeleydb
 import re
 from . import lib
@@ -208,18 +209,94 @@ def close(self):
     def __len__(self):
         return self.db.stat()["nkeys"]
 
+class CachedBsdDB:
+    def __init__(self, filename, readonly, contentType, cachesize):
+        self.filename = filename
+        self.db = berkeleydb.db.DB()
+        flags = 0
+
+        self.cachesize = cachesize
+        self.cache = OrderedDict()
+
+        if readonly:
+            flags |= berkeleydb.db.DB_RDONLY
+            self.db.open(filename, flags=flags)
+        else:
+            flags |= berkeleydb.db.DB_CREATE
+            self.db.open(filename, flags=flags, mode=0o644, dbtype=berkeleydb.db.DB_BTREE)
+        self.ctype = contentType
+
+    def exists(self, key):
+        if key in self.cache:
+            return True
+
+        key = autoBytes(key)
+        return self.db.exists(key)
+
+    def get(self, key):
+        if key in self.cache:
+            self.cache.move_to_end(key)
+            return self.cache[key]
+
+        key = autoBytes(key)
+        p = self.db.get(key)
+        if p is None:
+            return None
+        p = self.ctype(p)
+
+        self.cache[key] = p
+        self.cache.move_to_end(key)
+        if len(self.cache) > self.cachesize:
+            old_k, old_v = self.cache.popitem(last=False)
+            self.put_raw(old_k, old_v)
+
+        return p
+
+    def get_keys(self):
+        return self.db.keys()
+
+    def put(self, key, val):
+        self.cache[key] = val
+        self.cache.move_to_end(key)
+        if len(self.cache) > self.cachesize:
+            old_k, old_v = self.cache.popitem(last=False)
+            self.put_raw(old_k, old_v)
+
+    def put_raw(self, key, val, sync=False):
+        key = autoBytes(key)
+        val = autoBytes(val)
+        if type(val) is not bytes:
+            val = val.pack()
+        self.db.put(key, val)
+        if sync:
+            self.db.sync()
+
+    def sync(self):
+        for k, v in self.cache.items():
+            self.put_raw(k, v)
+
+        self.db.sync()
+    
+    def close(self):
+        self.sync()
+        self.db.close()
+
+    def __len__(self):
+        return self.db.stat()["nkeys"]
+
 class DB:
-    def __init__(self, dir, readonly=True, dtscomp=False, shared=False, update_cache=False):
+    def __init__(self, dir, readonly=True, dtscomp=False, shared=False, update_cache=None):
         if os.path.isdir(dir):
             self.dir = dir
         else:
             raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), dir)
 
         ro = readonly
-        cachesize = None
 
         if update_cache:
-            cachesize = CACHESIZE
+            db_cls = lambda dir, ro, ctype: CachedBsdDB(dir, ro, ctype, cachesize=update_cache)
+        else:
+            db_cls = lambda dir, ro, ctype: BsdDB(dir, ro, ctype, shared=shared)
 
         self.vars = BsdDB(dir + '/variables.db', ro, lambda x: int(x.decode()), shared=shared)
             # Key-value store of basic information
@@ -230,20 +307,20 @@ def __init__(self, dir, readonly=True, dtscomp=False, shared=False, update_cache
         self.file = BsdDB(dir + '/filenames.db', ro, lambda x: x.decode(), shared=shared)
             # Map serial number to filename
         self.vers = BsdDB(dir + '/versions.db', ro, PathList, shared=shared)
-        self.defs = BsdDB(dir + '/definitions.db', ro, DefList, shared=shared, cachesize=cachesize)
+        self.defs = db_cls(dir + '/definitions.db', ro, DefList)
         self.defs_cache = {}
         NOOP = lambda x: x
         self.defs_cache['C'] = BsdDB(dir + '/definitions-cache-C.db', ro, NOOP, shared=shared)
         self.defs_cache['K'] = BsdDB(dir + '/definitions-cache-K.db', ro, NOOP, shared=shared)
         self.defs_cache['D'] = BsdDB(dir + '/definitions-cache-D.db', ro, NOOP, shared=shared)
         self.defs_cache['M'] = BsdDB(dir + '/definitions-cache-M.db', ro, NOOP, shared=shared)
         assert sorted(self.defs_cache.keys()) == sorted(lib.CACHED_DEFINITIONS_FAMILIES)
-        self.refs = BsdDB(dir + '/references.db', ro, RefList, shared=shared, cachesize=cachesize)
-        self.docs = BsdDB(dir + '/doccomments.db', ro, RefList, shared=shared, cachesize=cachesize)
+        self.refs = db_cls(dir + '/references.db', ro, RefList)
+        self.docs = db_cls(dir + '/doccomments.db', ro, RefList)
         self.dtscomp = dtscomp
         if dtscomp:
-            self.comps = BsdDB(dir + '/compatibledts.db', ro, RefList, shared=shared, cachesize=cachesize)
-            self.comps_docs = BsdDB(dir + '/compatibledts_docs.db', ro, RefList, shared=shared, cachesize=cachesize)
+            self.comps = db_cls(dir + '/compatibledts.db', ro, RefList)
+            self.comps_docs = db_cls(dir + '/compatibledts_docs.db', ro, RefList)
             # Use a RefList in case there are multiple doc comments for an identifier
 
     def close(self):

elixir/update.py

@@ -1,6 +1,6 @@
 import os.path
 import logging
-from multiprocessing import cpu_count
+from multiprocessing import cpu_count, set_start_method
 from multiprocessing.pool import Pool
 from typing import Dict, Iterable, List, Optional, Tuple, Set
 
@@ -36,17 +36,13 @@
 IdxCache = Dict[int, Tuple[bytes, str, bool]]
 
 # Check if definition for ident is visible in current version
-def def_in_version(db: DB, def_cache: Set[bytes], idx_to_hash_and_filename: IdxCache, ident: bytes) -> bool:
-    if ident in def_cache:
-        return True
-
+def def_in_version(db: DB, idx_to_hash_and_filename: IdxCache, ident: bytes) -> bool:
     defs_this_ident = db.defs.get(ident)
     if not defs_this_ident:
         return False
 
     for def_idx, _, _, _ in defs_this_ident.iter():
         if def_idx in idx_to_hash_and_filename:
-            def_cache.add(ident)
             return True
 
     return False
@@ -64,12 +60,14 @@ def add_defs(db: DB, defs: DefsDict):
         db.defs.put(ident, obj)
 
 # Add references to database
-def add_refs(db: DB, def_cache: Set[bytes], idx_to_hash_and_filename: IdxCache, refs: RefsDict):
+def add_refs(db: DB, idx_to_hash_and_filename: IdxCache, refs: RefsDict):
     for ident, idx_to_lines in refs.items():
         deflist = db.defs.get(ident)
-        in_version = def_in_version(db, def_cache, idx_to_hash_and_filename, ident)
+        if deflist is None:
+            continue
 
-        if deflist is None or not in_version:
+        in_version = def_in_version(db, idx_to_hash_and_filename, ident)
+        if not in_version:
             continue
 
         def deflist_exists(idx: int, line: int):
@@ -320,10 +318,9 @@ def update_version(db: DB, tag: bytes, pool: Pool, dts_comp_support: bool):
 
         logger.info("dts comps docs done")
 
-    def_cache = set()
     for result in pool.imap_unordered(get_refs, idxes, chunksize):
         if result is not None:
-            add_refs(db, def_cache, idx_to_hash_and_filename, result)
+            add_refs(db, idx_to_hash_and_filename, result)
 
     logger.info("refs done")
 
@@ -334,10 +331,11 @@ def update_version(db: DB, tag: bytes, pool: Pool, dts_comp_support: bool):
     dts_comp_support = bool(int(script('dts-comp')))
     db = None
 
+    set_start_method('spawn')
     with Pool() as pool:
         for tag in scriptLines('list-tags'):
             if db is None:
-                db = DB(getDataDir(), readonly=False, dtscomp=dts_comp_support, shared=False, update_cache=True)
+                db = DB(getDataDir(), readonly=False, dtscomp=dts_comp_support, shared=False, update_cache=50000)
 
             if not db.vers.exists(tag):
                 logger.info("updating tag %s", tag)