Make bindings generation dramatically faster

src/Common.py

@@ -32,4 +32,5 @@ def getGlobalIncludes() -> Set[str]:
     "/emsdk/upstream/emscripten/system/lib/libcxx/include/__support/newlib/"
   ])) + \
   list(map(lambda x: "-I" + x, ocIncludePaths + additionalIncludePaths))
-
+
+ocIncludeStatements = os.linesep.join(map(lambda x: "#include \"" + os.path.basename(x) + "\"", list(sorted(ocIncludeFiles))))

src/TuInfo.py

@@ -0,0 +1,80 @@
+import clang.cindex
+from Common import includePathArgs, ocIncludeStatements, occtBasePath
+from filter.filterTypedefs import filterTypedef
+from filter.filterEnums import filterEnum
+from wasmGenerator.Common import ignoreDuplicateTypedef
+
+def parse(additionalCppCode = ""):
+  index = clang.cindex.Index.create()
+  translationUnit = index.parse(
+    "myMain.h", [
+      "-x",
+      "c++",
+      "-stdlib=libc++",
+      "-D__EMSCRIPTEN__"
+    ] + includePathArgs,
+    [["myMain.h", ocIncludeStatements + "\n" + additionalCppCode]]
+  )
+
+  if len(translationUnit.diagnostics) > 0:
+    print("Diagnostic Messages:")
+    for d in translationUnit.diagnostics:
+      print("  " + d.format())
+
+  return translationUnit
+
+def templateTypedefGenerator(tu):
+  return list(filter(
+    lambda x:
+      x.kind == clang.cindex.CursorKind.TYPEDEF_DECL and
+      not (x.get_definition() is None or not x == x.get_definition()) and
+      filterTypedef(x) and
+      x.type.get_num_template_arguments() != -1 and
+      not ignoreDuplicateTypedef(x),
+    tu.cursor.get_children()))
+
+def typedefGenerator(tu):
+  return list(filter(lambda x: x.kind == clang.cindex.CursorKind.TYPEDEF_DECL, tu.cursor.get_children()))
+
+def allChildrenGenerator(tu):
+  return list(tu.cursor.get_children())
+
+def enumGenerator(tu):
+  return list(filter(lambda x: x.kind == clang.cindex.CursorKind.ENUM_DECL and filterEnum(x), tu.cursor.get_children()))
+
+def classDict(tu):
+  d = dict()
+  for x in tu.cursor.get_children():
+    if (
+      x.kind == clang.cindex.CursorKind.CLASS_DECL or
+      x.kind == clang.cindex.CursorKind.STRUCT_DECL
+    ) and not (
+      x.get_definition() is None or
+      not x == x.get_definition()
+    ):
+      if x.spelling not in d:
+        # Original code didn't handle duplicate names, that seems bad?
+        d[x.spelling] = x
+  return d
+
+def underlyingDict(l, checkOcctBasePath: bool):
+  d = dict()
+  for x in l:
+    if checkOcctBasePath and not x.location.file.name.startswith(occtBasePath):
+      continue
+    if x.underlying_typedef_type.spelling not in d:
+      # Original code didn't handle duplicate names, that seems bad?
+      d[x.underlying_typedef_type.spelling] = x
+  return d
+
+
+class TuInfo:
+  def __init__(self, customCode):
+    self.tu = parse(customCode)
+    self.allChildren = allChildrenGenerator(self.tu)
+    self.typedefs = typedefGenerator(self.tu)
+    self.enums = enumGenerator(self.tu)
+    self.templateTypedefs = templateTypedefGenerator(self.tu)
+    self.classDict = classDict(self.tu)
+    self.typedefUnderlyingDict = underlyingDict(self.typedefs, True)
+    self.templateTypedefUnderlyingDict = underlyingDict(self.templateTypedefs, False)

src/bindings.py

@@ -4,7 +4,6 @@
 from wasmGenerator.Common import SkipException, isAbstractClass, getMethodOverloadPostfix
 from filter.filterClasses import filterClass
 from filter.filterMethodOrProperties import filterMethodOrProperty
-from Common import occtBasePath
 from typing import Tuple, List
 
 def merge(sep: str, *strings: List[str]):
@@ -81,20 +80,27 @@ def getClassTypeName(theClass, templateDecl = None):
   return templateDecl.spelling if templateDecl is not None else theClass.spelling
 
 class Bindings:
-  def __init__(self, typedefs, templateTypedefs, translationUnit):
-    self.templateTypedefs = templateTypedefs
-    self.translationUnit = translationUnit
-    self.typedefs = typedefs
+  def __init__(self, tuInfo):
+    self.tuInfo = tuInfo
 
   def getTypedefedTemplateTypeAsString(self, theTypeSpelling, templateDecl = None, templateArgs = None):
     if templateDecl is None:
-      typedefType = next((x for x in self.typedefs if x.location.file.name.startswith(occtBasePath) and x.underlying_typedef_type.spelling == theTypeSpelling), None)
-      typedefType = None if typedefType is None else typedefType.spelling
+      tud = self.tuInfo.typedefUnderlyingDict
+      if theTypeSpelling in tud:
+        typedefType = tud[theTypeSpelling].spelling
+      else:
+        typedefType = None
     else:
       templateType = self.replaceTemplateArgs(theTypeSpelling, templateArgs)
       rawTemplateType = templateType.replace("&", "").replace("const", "").strip()
-      rawTypedefType = next((x for x in self.templateTypedefs if (x.underlying_typedef_type.spelling == rawTemplateType or x.underlying_typedef_type.spelling == "opencascade::" + rawTemplateType)), None)
-      rawTypedefType = rawTemplateType if rawTypedefType is None else rawTypedefType.spelling
+      ttud = self.tuInfo.templateTypedefUnderlyingDict
+      oc_rawTemplateType = "opencascade::" + rawTemplateType
+      if rawTemplateType in ttud:
+        rawTypedefType = ttud[rawTemplateType].spelling
+      elif oc_rawTemplateType in ttud:
+        rawTypedefType = ttud[oc_rawTemplateType].spelling
+      else:
+        rawTypedefType = rawTemplateType
       typedefType = templateType.replace(rawTemplateType, rawTypedefType)
     return theTypeSpelling if typedefType is None else typedefType
 
@@ -109,7 +115,7 @@ def replaceTemplateArgs(self, string, templateArgs = None):
 
   def processClass(self, theClass, templateDecl = None, templateArgs = None):
     output = ""
-    isAbstract = isAbstractClass(theClass, self.translationUnit)
+    isAbstract = isAbstractClass(theClass, self.tuInfo.classDict)
     if not isAbstract:
       output += self.processSimpleConstructor(theClass)
     for method in theClass.get_children():
@@ -130,10 +136,9 @@ def processClass(self, theClass, templateDecl = None, templateArgs = None):
 class EmbindBindings(Bindings):
   def __init__(
     self,
-    typedefs, templateTypedefs,
-    translationUnit
+    tuInfo
   ):
-    super().__init__(typedefs, templateTypedefs, translationUnit)
+    super().__init__(tuInfo)
 
   def processClass(self, theClass, templateDecl = None, templateArgs = None):
     output = ""
@@ -451,10 +456,9 @@ def processEnum(self, theEnum):
 class TypescriptBindings(Bindings):
   def __init__(
     self,
-    typedefs, templateTypedefs,
-    translationUnit
+    tuInfo
   ):
-    super().__init__(typedefs, templateTypedefs, translationUnit)
+    super().__init__(tuInfo)
     self.imports = {}
 
     self.exports = []

src/generateBindings.py

@@ -5,20 +5,16 @@
 import clang.cindex
 import os
 import errno
-from filter.filterTypedefs import filterTypedef
-from filter.filterEnums import filterEnum
-from wasmGenerator.Common import ignoreDuplicateTypedef, SkipException
-from Common import ocIncludeFiles, includePathArgs
+from wasmGenerator.Common import SkipException
+from Common import ocIncludeStatements
 import json
-import multiprocessing
 import os
 from filter.filterPackages import filterPackages
-from functools import partial
+from TuInfo import TuInfo
 
 libraryBasePath = "/opencascade.js/build/bindings"
 buildDirectory = "/opencascade.js/build"
 occtBasePath = "/occt/src/"
-ocIncludeStatements = os.linesep.join(map(lambda x: "#include \"" + os.path.basename(x) + "\"", list(sorted(ocIncludeFiles))))
 
 def mkdirp(name: str) -> None:
   try:
@@ -70,23 +66,20 @@ def filterEnums(child, customBuild):
     child.kind == clang.cindex.CursorKind.ENUM_DECL
   )
 
-def processChildBatch(customCode, generator, buildType: str, extension: str, filterFunction: Callable[[any], bool], processFunction: Callable[[any, any], str], typedefGenerator: any, templateTypedefGenerator: any, preamble: str, customBuild: bool, batch):
-  tu = parse(customCode)
-  children = list(generator(tu)[batch.start:batch.stop])
-
+def processChildren(tuInfo: TuInfo, children, extension: str, filterFunction: Callable[[any], bool], processFunction: Callable[[any, any], str], preamble: str, customBuild: bool):
   for child in children:
     if not filterFunction(child, customBuild) or child.spelling == "":
       continue
 
     relOcFileName: str = child.extent.start.file.name.replace(occtBasePath, "")
-    mkdirp(buildDirectory + "/" + buildType + "/" + os.path.dirname(relOcFileName))
-    mkdirp(buildDirectory + "/" + buildType + "/" + relOcFileName)
-    filename = buildDirectory + "/" + buildType + "/" + relOcFileName + "/" + (child.spelling if not child.spelling == "" else child.type.spelling) + extension
+    mkdirp(buildDirectory + "/bindings/" + os.path.dirname(relOcFileName))
+    mkdirp(buildDirectory + "/bindings/" + relOcFileName)
+    filename = buildDirectory + "/bindings/" + relOcFileName + "/" + (child.spelling if not child.spelling == "" else child.type.spelling) + extension
 
     if not os.path.exists(filename):
       print("Processing " + child.spelling)
       try:
-        output = processFunction(tu, preamble, child, typedefGenerator(tu), templateTypedefGenerator(tu))
+        output = processFunction(tuInfo, preamble, child)
         bindingsFile = open(filename, "w")
         bindingsFile.write(output)
       except SkipException as e:
@@ -98,17 +91,6 @@ def split(a, n):
   k, m = divmod(len(a), n)
   return (a[i * k + min(i, m):(i + 1) * k + min(i + 1, m)] for i in range(n))
 
-def processChildren(generator, buildType: str, extension: str, filterFunction: Callable[[any], bool], processFunction: Callable[[any, any], str], typedefs: any, templateTypedefs: any, preamble: str, customCode, customBuild):
-  tu = parse(customCode)
-  func = partial(processChildBatch, customCode, generator, buildType, extension, filterFunction, processFunction, typedefs, templateTypedefs, preamble, customBuild)
-  if not customBuild:
-    numthreads = multiprocessing.cpu_count()
-    batches = split(range(len(generator(tu))), numthreads)
-    with multiprocessing.Pool(processes=numthreads) as p:
-      p.map(func, batches)
-  else:
-    func(range(len(generator(tu))))
-
 def processTemplate(child):
   templateRefs = list(filter(lambda x: x.kind == clang.cindex.CursorKind.TEMPLATE_REF, child.get_children()))
   if len(templateRefs) != 1:
@@ -127,51 +109,32 @@ def processTemplate(child):
 
   return [templateClass, templateArgs]
 
-def embindGenerationFuncClasses(tu, preamble, child, typedefs, templateTypedefs) -> str:
-  embindings = EmbindBindings(typedefs, templateTypedefs, tu)
+def embindGenerationFuncClasses(tuInfo: TuInfo, preamble, child) -> str:
+  embindings = EmbindBindings(tuInfo)
   output = embindings.processClass(child)
 
   return preamble + output
 
-def embindGenerationFuncTemplates(tu, preamble, child, typedefs, templateTypedefs) -> str:
+def embindGenerationFuncTemplates(tuInfo: TuInfo, preamble, child) -> str:
   [templateClass, templateArgs] = processTemplate(child)
-  embindings = EmbindBindings(typedefs, templateTypedefs, tu)
+  embindings = EmbindBindings(tuInfo)
   output = embindings.processClass(templateClass, child, templateArgs)
 
   return preamble + output
 
-def embindGenerationFuncEnums(tu, preamble, child, typedefs, templateTypedefs) -> str:
-  embindings = EmbindBindings(typedefs, templateTypedefs, tu)
+def embindGenerationFuncEnums(tuInfo: TuInfo, preamble, child) -> str:
+  embindings = EmbindBindings(tuInfo)
   output = embindings.processEnum(child)
 
   return preamble + output
 
-def templateTypedefGenerator(tu):
-  return list(filter(
-    lambda x:
-      x.kind == clang.cindex.CursorKind.TYPEDEF_DECL and
-      not (x.get_definition() is None or not x == x.get_definition()) and
-      filterTypedef(x) and
-      x.type.get_num_template_arguments() != -1 and
-      not ignoreDuplicateTypedef(x),
-    tu.cursor.get_children()))
-
-def typedefGenerator(tu):
-  return list(filter(lambda x: x.kind == clang.cindex.CursorKind.TYPEDEF_DECL, tu.cursor.get_children()))
-
-def allChildrenGenerator(tu):
-  return list(tu.cursor.get_children())
-
-def enumGenerator(tu):
-  return list(filter(lambda x: x.kind == clang.cindex.CursorKind.ENUM_DECL and filterEnum(x), tu.cursor.get_children()))
-
-def process(extension, embindGenerationFuncClasses, embindGenerationFuncTemplates, embindGenerationFuncEnums, preamble, customCode, customBuild):
-  processChildren(allChildrenGenerator, "bindings", extension, filterClasses, embindGenerationFuncClasses, typedefGenerator, templateTypedefGenerator, preamble, customCode, customBuild)
-  processChildren(templateTypedefGenerator, "bindings", extension, filterTemplates, embindGenerationFuncTemplates, typedefGenerator, templateTypedefGenerator, preamble, customCode, customBuild)
-  processChildren(enumGenerator, "bindings", extension, filterEnums, embindGenerationFuncEnums, typedefGenerator, templateTypedefGenerator, preamble, customCode, customBuild)
+def process(tuInfo: TuInfo, extension, embindGenerationFuncClasses, embindGenerationFuncTemplates, embindGenerationFuncEnums, preamble, customBuild):
+  processChildren(tuInfo, tuInfo.allChildren, extension, filterClasses, embindGenerationFuncClasses, preamble, customBuild)
+  processChildren(tuInfo, tuInfo.templateTypedefs, extension, filterTemplates, embindGenerationFuncTemplates, preamble, customBuild)
+  processChildren(tuInfo, tuInfo.enums, extension, filterEnums, embindGenerationFuncEnums, preamble, customBuild)
 
-def typescriptGenerationFuncClasses(tu, preamble, child, typedefs, templateTypedefs) -> str:
-  typescript = TypescriptBindings(typedefs, templateTypedefs, tu)
+def typescriptGenerationFuncClasses(tuInfo: TuInfo, preamble, child) -> str:
+  typescript = TypescriptBindings(tuInfo)
   output = typescript.processClass(child)
 
   return json.dumps({
@@ -180,9 +143,9 @@ def typescriptGenerationFuncClasses(tu, preamble, child, typedefs, templateTyped
     "exports": typescript.exports,
   })
 
-def typescriptGenerationFuncTemplates(tu, preamble, child, typedefs, templateTypedefs) -> str:
+def typescriptGenerationFuncTemplates(tuInfo: TuInfo, preamble, child) -> str:
   [templateClass, templateArgs] = processTemplate(child)
-  typescript = TypescriptBindings(typedefs, templateTypedefs, tu)
+  typescript = TypescriptBindings(tuInfo)
   output = typescript.processClass(templateClass, child, templateArgs)
 
   return json.dumps({
@@ -191,8 +154,8 @@ def typescriptGenerationFuncTemplates(tu, preamble, child, typedefs, templateTyp
     "exports": typescript.exports,
   })
 
-def typescriptGenerationFuncEnums(tu, preamble, child, typedefs, templateTypedefs) -> str:
-  typescript = TypescriptBindings(typedefs, templateTypedefs, tu)
+def typescriptGenerationFuncEnums(tuInfo: TuInfo, preamble, child) -> str:
+  typescript = TypescriptBindings(tuInfo)
   output = typescript.processEnum(child)
 
   return json.dumps({
@@ -201,25 +164,6 @@ def typescriptGenerationFuncEnums(tu, preamble, child, typedefs, templateTypedef
     "exports": typescript.exports,
   })
 
-def parse(additionalCppCode = ""):
-  index = clang.cindex.Index.create()
-  translationUnit = index.parse(
-    "myMain.h", [
-      "-x",
-      "c++",
-      "-stdlib=libc++",
-      "-D__EMSCRIPTEN__"
-    ] + includePathArgs,
-    [["myMain.h", ocIncludeStatements + "\n" + additionalCppCode]]
-  )
-
-  if len(translationUnit.diagnostics) > 0:
-    print("Diagnostic Messages:")
-    for d in translationUnit.diagnostics:
-      print("  " + d.format())
-
-  return translationUnit
-
 referenceTypeTemplateDefs = \
   "\n" + \
   "#include <emscripten/bind.h>\n" + \
@@ -252,16 +196,19 @@ def generateCustomCodeBindings(customCode):
 
   embindPreamble = ocIncludeStatements + "\n" + referenceTypeTemplateDefs + "\n" + customCode
 
-  process(".cpp", embindGenerationFuncClasses, embindGenerationFuncTemplates, embindGenerationFuncEnums, embindPreamble, customCode, True)
-  process(".d.ts.json", typescriptGenerationFuncClasses, typescriptGenerationFuncTemplates, typescriptGenerationFuncEnums, "", customCode, True)
+  tuInfo = TuInfo(customCode)
+  process(tuInfo, ".cpp", embindGenerationFuncClasses, embindGenerationFuncTemplates, embindGenerationFuncEnums, embindPreamble, True)
+  process(tuInfo, ".d.ts.json", typescriptGenerationFuncClasses, typescriptGenerationFuncTemplates, typescriptGenerationFuncEnums, "", True)
 
 if __name__ == "__main__":
   try:
     os.makedirs(libraryBasePath)
   except Exception:
     pass
 
+  tuInfo = TuInfo("")
+
   embindPreamble = ocIncludeStatements + "\n" + referenceTypeTemplateDefs
-  process(".cpp", embindGenerationFuncClasses, embindGenerationFuncTemplates, embindGenerationFuncEnums, embindPreamble, "", False)
+  process(tuInfo, ".cpp", embindGenerationFuncClasses, embindGenerationFuncTemplates, embindGenerationFuncEnums, embindPreamble, False)
 
-  process(".d.ts.json", typescriptGenerationFuncClasses, typescriptGenerationFuncTemplates, typescriptGenerationFuncEnums, "", "", False)
+  process(tuInfo, ".d.ts.json", typescriptGenerationFuncClasses, typescriptGenerationFuncTemplates, typescriptGenerationFuncEnums, "", False)

src/wasmGenerator/Common.py

@@ -6,19 +6,9 @@ class SkipException(Exception):
 def getPureVirtualMethods(theClass):
   return list(filter(lambda x: x.is_pure_virtual_method(), list(theClass.get_children())))
 
-def isAbstractClass(theClass, tu):
-  allClasses = list(filter(lambda x:
-    (
-      x.kind == clang.cindex.CursorKind.CLASS_DECL or
-      x.kind == clang.cindex.CursorKind.STRUCT_DECL
-    ) and
-    not (
-      x.get_definition() is None or
-      not x == x.get_definition()
-    ),
-    tu.cursor.get_children()))
+def isAbstractClass(theClass, classDict):
   baseSpec = list(filter(lambda x: x.kind == clang.cindex.CursorKind.CXX_BASE_SPECIFIER and x.access_specifier == clang.cindex.AccessSpecifier.PUBLIC, list(theClass.get_children())))
-  baseClasses = list(map(lambda y: next((x for x in allClasses if x.spelling == y.type.spelling)), baseSpec))
+  baseClasses = [classDict[y.type.spelling] for y in baseSpec if y.type.spelling in classDict]
 
   pureVirtualMethods = getPureVirtualMethods(theClass)
   if len(pureVirtualMethods) > 0: