data: add definitions-cache-[FAMILY].db databases

Lookup if a definition exists is taking too long to render source code.
Generate small databases that only tell us if a definition exists for a
given family. Because the database is much smaller, it is faster to
query.

Many URLs could only be queried at 12 req/s. With that patch, I can do
>80 req/s on the same URLs, with the same config.

We generate the caches from update.py. We also add an edge-case to
generate the files (if they don't exist) even if no new tag exists.

Signed-off-by: Théo Lebrun <theo.lebrun@bootlin.com>

Author: tleb

elixir/data.py

@@ -201,6 +201,12 @@ def __init__(self, dir, readonly=True, dtscomp=False, shared=False):
             # Map serial number to filename
         self.vers = BsdDB(dir + '/versions.db', ro, PathList, shared=shared)
         self.defs = BsdDB(dir + '/definitions.db', ro, DefList, shared=shared)
+        self.defs_cache = {}
+        NOOP = lambda x: x
+        self.defs_cache['C'] = BsdDB(dir + '/definitions-cache-C.db', ro, NOOP, shared=shared)
+        self.defs_cache['K'] = BsdDB(dir + '/definitions-cache-K.db', ro, NOOP, shared=shared)
+        self.defs_cache['D'] = BsdDB(dir + '/definitions-cache-D.db', ro, NOOP, shared=shared)
+        self.defs_cache['M'] = BsdDB(dir + '/definitions-cache-M.db', ro, NOOP, shared=shared)
         self.refs = BsdDB(dir + '/references.db', ro, RefList, shared=shared)
         self.docs = BsdDB(dir + '/doccomments.db', ro, RefList, shared=shared)
         self.dtscomp = dtscomp
@@ -216,6 +222,10 @@ def close(self):
         self.file.close()
         self.vers.close()
         self.defs.close()
+        self.defs_cache['C'].close()
+        self.defs_cache['K'].close()
+        self.defs_cache['D'].close()
+        self.defs_cache['M'].close()
         self.refs.close()
         self.docs.close()
         if self.dtscomp:

elixir/query.py

@@ -188,9 +188,7 @@ def query(self, cmd, *args):
                 for tok in tokens:
                     even = not even
                     tok2 = prefix + tok
-                    if (even and self.db.defs.exists(tok2) and
-                        (lib.compatibleFamily(self.db.defs.get(tok2).get_families(), family) or
-                        lib.compatibleMacro(self.db.defs.get(tok2).get_macros(), family))):
+                    if even and self.db.defs_cache[family].exists(tok2):
                         tok = b'\033[31m' + tok2 + b'\033[0m'
                     else:
                         tok = lib.unescape(tok)

update.py

@@ -189,6 +189,15 @@ def update_versions(self, tag):
         db.vers.put(tag, obj, sync=True)
 
 
+def generate_defs_caches():
+    for key in db.defs.get_keys():
+        value = db.defs.get(key)
+        for family in ['C', 'K', 'D', 'M']:
+            if (lib.compatibleFamily(value.get_families(), family) or
+                        lib.compatibleMacro(value.get_macros(), family)):
+                db.defs_cache[family].put(key, b'')
+
+
 class UpdateDefs(Thread):
     def __init__(self, start, inc):
         Thread.__init__(self, name="UpdateDefsElixir")
@@ -256,6 +265,8 @@ def update_definitions(self, idxes):
                         print(f"def {type} {ident} in #{idx} @ {line}")
                     db.defs.put(ident, obj)
 
+        generate_defs_caches()
+
 
 class UpdateRefs(Thread):
     def __init__(self, start, inc):
@@ -586,6 +597,9 @@ def progress(msg, current):
 print(project + ' - found ' + str(num_tags) + ' new tags')
 
 if not num_tags:
+    # Backward-compatibility: generate defs caches if they are empty.
+    if db.defs_cache['C'].db.stat()['nkeys'] == 0:
+        generate_defs_caches()
     exit(0)
 
 threads_list.append(UpdateIds(tag_buf))