Clean up generic Genny bindings

.github/workflows/c.yml

@@ -32,3 +32,5 @@ jobs:
       run: gcc -o tests/test_pixie_c tests/test_pixie_c.c -I ../pixie/bindings/generated -L ../pixie/bindings/generated -l:libpixie.so -lm -Wl,-rpath,'$ORIGIN/../../pixie/bindings/generated'
     - name: Run Pixie C test
       run: ./tests/test_pixie_c
+    - name: Compare Pixie render goldens
+      run: nim r --mm:arc --path:../pixie/src tests/test_no_gold_diff.nim

.github/workflows/cpp.yml

@@ -32,3 +32,5 @@ jobs:
       run: g++ -o tests/test_pixie_cpp tests/test_pixie_cpp.cpp -I ../pixie/bindings/generated -L ../pixie/bindings/generated -l:libpixie.so -Wl,-rpath,'$ORIGIN/../../pixie/bindings/generated'
     - name: Run Pixie C++ test
       run: ./tests/test_pixie_cpp
+    - name: Compare Pixie render goldens
+      run: nim r --mm:arc --path:../pixie/src tests/test_no_gold_diff.nim

.github/workflows/nim.yml

@@ -34,3 +34,5 @@ jobs:
       run: |
         nim c --mm:arc -o:tests/test_pixie_nim tests/test_pixie_nim.nim
         PIXIE_ROOT=../pixie LD_LIBRARY_PATH=$PWD/../pixie/bindings/generated ./tests/test_pixie_nim
+    - name: Compare Pixie render goldens
+      run: nim r --mm:arc --path:../pixie/src tests/test_no_gold_diff.nim

.github/workflows/node.yml

@@ -37,3 +37,5 @@ jobs:
       run: cd ../pixie && nim c --mm:arc --app:lib -d:gennyNode --path:../genny/src --path:src -o:bindings/generated/libpixie.so bindings/bindings.nim
     - name: Run Pixie Node tests
       run: NODE_PATH=$PWD/tests/node_modules PIXIE_ROOT=../pixie PIXIE_BINDINGS_DIR=../pixie/bindings/generated LD_LIBRARY_PATH=$PWD/../pixie/bindings/generated node tests/test_pixie_node.js
+    - name: Compare Pixie render goldens
+      run: nim r --mm:arc --path:../pixie/src tests/test_no_gold_diff.nim

.github/workflows/python-native.yml

@@ -39,3 +39,5 @@ jobs:
     - name: Run non-gating benchmark
       continue-on-error: true
       run: python tests/bench_python_native.py
+    - name: Compare Pixie render goldens
+      run: nim r --mm:arc --path:../pixie/src tests/test_no_gold_diff.nim

.github/workflows/python.yml

@@ -34,3 +34,5 @@ jobs:
     - name: Run non-gating benchmark
       continue-on-error: true
       run: python tests/bench_python.py
+    - name: Compare Pixie render goldens
+      run: nim r --mm:arc --path:../pixie/src tests/test_no_gold_diff.nim

.github/workflows/zig.yml

@@ -45,3 +45,5 @@ jobs:
         cp ../../pixie/bindings/generated/pixie.zig generated/pixie.zig
         zig build-exe test_pixie_zig.zig -lc -L../../pixie/bindings/generated -lpixie
         LD_LIBRARY_PATH=$PWD/../../pixie/bindings/generated ./test_pixie_zig
+    - name: Compare Pixie render goldens
+      run: nim r --mm:arc --path:../pixie/src tests/test_no_gold_diff.nim

.gitignore

@@ -12,6 +12,7 @@
 nimcache
 __pycache__/
 tests/generated/*_native.c
+tests/generated/pixie_images/*.png
 *.pdb
 *.ilk
 .*

src/genny.nim

@@ -1,4 +1,4 @@
-import genny/internal, macros, strformat
+import genny/[common, internal], macros, strformat, strutils
 
 when defined(gennyC): import genny/languages/c
 when defined(gennyCpp): import genny/languages/cpp
@@ -86,6 +86,22 @@ proc fieldUntyped(clause, owner: NimNode): NimNode =
       obj: `owner`
       f = obj.`clause`
 
+proc objectFieldUntyped(clause: NimNode): NimNode =
+  result = emptyBlockStmt()
+  if clause.kind notin {nnkExprColonExpr, nnkCall}:
+    error("Object fields need explicit types, for example `x: float32`", clause)
+  let fieldName = clause[0]
+  let fieldType =
+    if clause.kind == nnkExprColonExpr:
+      clause[1]
+    elif clause.len == 2 and clause[1].kind == nnkStmtList and clause[1].len == 1:
+      clause[1][0]
+    else:
+      error("Object fields need explicit types, for example `x: float32`", clause)
+  let fieldVar = ident("gennyField_" & fieldName.repr)
+  result[1].add quote do:
+    var `fieldVar`: `fieldType`
+
 proc procUntyped(clause: NimNode): NimNode =
   result = emptyBlockStmt()
 
@@ -119,6 +135,25 @@ proc procTypedSym(entry: NimNode): NimNode =
       else:
         entry[1][^1][0][0]
 
+proc objectFieldsTyped(fieldsBlock: NimNode): seq[ObjectField] =
+  if fieldsBlock[1].len == 0:
+    return
+
+  for entry in fieldsBlock[1].asStmtList:
+    var fieldEntry = entry
+    if fieldEntry.kind == nnkBlockStmt:
+      fieldEntry = fieldEntry[1]
+    if fieldEntry.kind == nnkStmtList and fieldEntry.len == 1:
+      fieldEntry = fieldEntry[0]
+    if fieldEntry.kind != nnkVarSection:
+      error("Invalid generated object field entry", fieldEntry)
+    let identDefs = fieldEntry[0]
+    for i in 0 .. identDefs.len - 3:
+      let fieldSym = identDefs[i]
+      var fieldName = fieldSym.repr.split("`")[0]
+      fieldName.removePrefix("gennyField_")
+      result.add((fieldName, fieldSym.getTypeInst()))
+
 proc procTyped(
   entry: NimNode,
   owner: NimNode = nil,
@@ -156,6 +191,7 @@ macro exportObjectUntyped(sym, body: untyped) =
   result.add varSection
 
   var
+    fieldsBlock = emptyBlockStmt()
     constructorBlock = emptyBlockStmt()
     procsBlock = emptyBlockStmt()
 
@@ -164,6 +200,9 @@ macro exportObjectUntyped(sym, body: untyped) =
       continue
 
     case section[0].repr:
+    of "fields":
+      for field in section[1]:
+        fieldsBlock[1].add objectFieldUntyped(field)
     of "constructor":
       constructorBlock[1].add procUntyped(section[1][0])
     of "procs":
@@ -172,40 +211,57 @@ macro exportObjectUntyped(sym, body: untyped) =
     else:
       error("Invalid section", section)
 
+  result.add fieldsBlock
   result.add constructorBlock
   result.add procsBlock
 
 macro exportObjectTyped(body: typed) =
   let
-    sym = body[0][0][1]
-    constructorBlock = body[1]
-    procsBlock = body[2]
+    sym = body[0][0][0].getTypeInst()
+    typeExpr = body[0][0][1]
+    fieldsBlock = body[1]
+    constructorBlock = body[2]
+    procsBlock = body[3]
+    fields = objectFieldsTyped(fieldsBlock)
+    nimModule =
+      if typeExpr.kind == nnkDotExpr:
+        typeExpr[0].repr
+      else:
+        ""
 
   let constructor =
     if constructorBlock[1].len > 0:
       procTypedSym(constructorBlock[1])
     else:
       nil
 
-  exportObjectInternal(sym, constructor)
-  when defined(gennyNim): exportObjectNim(sym, constructor)
-  when defined(gennyPython): exportObjectPy(sym, constructor)
-  when defined(gennyPythonNative): exportObjectPyNative(sym, constructor)
-  when defined(gennyNode): exportObjectNode(sym, constructor)
-  when defined(gennyC): exportObjectC(sym, constructor)
-  when defined(gennyCpp): exportObjectCpp(sym, constructor)
-  when defined(gennyZig): exportObjectZig(sym, constructor)
+  registerValueObjectType(sym)
+  if constructor != nil:
+    let constructorType = constructor.getTypeInst()
+    if constructorType[0][0].kind != nnkEmpty:
+      registerValueObjectTypeAlias(sym.repr, constructorType[0][0])
+  exportObjectInternal(sym, fields, constructor, nimModule.len > 0)
+  when defined(gennyNim):
+    exportObjectNim(sym, fields, constructor, nimModule)
+  when defined(gennyPython): exportObjectPy(sym, fields, constructor)
+  when defined(gennyPythonNative): exportObjectPyNative(sym, fields, constructor)
+  when defined(gennyNode): exportObjectNode(sym, fields, constructor)
+  when defined(gennyC): exportObjectC(sym, fields, constructor)
+  when defined(gennyCpp): exportObjectCpp(sym, fields, constructor)
+  when defined(gennyZig): exportObjectZig(sym, fields, constructor)
 
   if procsBlock[1].len > 0:
     var procsSeen: seq[string]
     for entry in procsBlock[1].asStmtList:
       var
         procSym = procTypedSym(entry)
         prefixes: seq[NimNode]
+      let procType = procSym.getTypeInst()
+      if procType[0].len > 1:
+        registerValueObjectTypeAlias(sym.repr, procType[0][1][^2])
       if procSym.repr notin procsSeen:
         procsSeen.add procSym.repr
       else:
-        let procType = procSym.getTypeInst()
         if procType[0].len > 2:
           prefixes.add(procType[0][2][1])
       exportProcInternal(procSym, sym, prefixes)
@@ -217,6 +273,7 @@ macro exportObjectTyped(body: typed) =
       when defined(gennyCpp): exportProcCpp(procSym, sym, prefixes)
       when defined(gennyZig): exportProcZig(procSym, sym, prefixes)
 
+  when defined(gennyPython): exportCloseObjectPy(sym)
   when defined(gennyZig): exportCloseObjectZig()
   when defined(gennyCpp): exportCloseObjectCpp()
 

src/genny/common.nim

@@ -1,5 +1,11 @@
 import macros, strformat, strutils
 
+type ObjectField* = tuple[name: string, typ: NimNode]
+
+var
+  exportedValueTypes {.compiletime.}: seq[(string, NimNode)]
+  exportedValueTypeAliases {.compiletime.}: seq[(string, string)]
+
 const basicTypes* = [
   "bool",
   "int8",
@@ -68,6 +74,48 @@ proc toVarCase*(s: string): string =
   if i < s.len:
     result.add s[i .. ^1]
 
+proc stripSinkCommon(sym: NimNode): NimNode =
+  if sym.kind == nnkBracketExpr and sym[0].repr == "sink":
+    sym[1]
+  else:
+    sym
+
+proc normalizeTypeRepr(s: string): string =
+  s.replace("system.", "")
+
+proc registerValueObjectTypeAliasRepr*(name, alias: string) =
+  let normalizedAlias = alias.normalizeTypeRepr()
+  for (existingAlias, existingName) in exportedValueTypeAliases:
+    if existingAlias == normalizedAlias and existingName == name:
+      return
+  exportedValueTypeAliases.add((normalizedAlias, name))
+
+proc registerValueObjectTypeAlias*(name: string, typ: NimNode) =
+  registerValueObjectTypeAliasRepr(name, typ.repr)
+
+proc registerValueObjectType*(sym: NimNode) =
+  let name = sym.repr
+  for (existingName, _) in exportedValueTypes:
+    if existingName == name:
+      return
+  exportedValueTypes.add((name, sym))
+  var aliases = @[sym.repr, sym.getTypeInst().repr, sym.getType().repr]
+  let impl = sym.getImpl()
+  if impl.kind == nnkTypeDef:
+    aliases.add(impl[2].repr)
+  for alias in aliases:
+    registerValueObjectTypeAliasRepr(name, alias)
+
+proc exportedValueTypeName*(sym: NimNode): string =
+  let typ = sym.stripSinkCommon()
+  let typRepr = typ.repr.normalizeTypeRepr()
+  for (alias, name) in exportedValueTypeAliases:
+    if typRepr == alias:
+      return name
+  for (name, exportedType) in exportedValueTypes:
+    if typ.sameType(exportedType) or typ.sameType(exportedType.getTypeInst()):
+      return name
+
 proc getSeqName*(sym: NimNode): string =
   if sym.kind == nnkBracketExpr:
     result = &"Seq{sym[1]}"
@@ -76,11 +124,87 @@ proc getSeqName*(sym: NimNode): string =
   result[3] = toUpperAscii(result[3])
 
 proc getName*(sym: NimNode): string =
-  if sym.kind == nnkBracketExpr:
+  let valueName = sym.exportedValueTypeName()
+  if valueName.len > 0:
+    valueName
+  elif sym.kind == nnkBracketExpr:
     sym.getSeqName()
   else:
     sym.repr
 
+proc getParamName*(sym: NimNode): string =
+  sym.repr.split("`")[0]
+
+proc usePrefixName*(sym: NimNode): bool =
+  if sym.kind != nnkSym:
+    return true
+  let impl = sym.getImpl()
+  impl.kind != nnkNilLit and impl[2].kind != nnkEnumTy
+
+proc arrayCount*(sym: NimNode): int =
+  let bounds = sym[1].repr
+  if ".." in bounds:
+    let parts = bounds.split("..")
+    parseInt(parts[1].strip()) - parseInt(parts[0].strip()) + 1
+  else:
+    parseInt(bounds)
+
+proc normalizedOperatorName*(name: string): string =
+  result = name
+  result.removePrefix("`")
+  result.removeSuffix("`")
+
+proc isOperatorName*(name: string): bool =
+  name.normalizedOperatorName() in ["+", "-", "*", "/"]
+
+proc operatorProcName*(name: string): string =
+  case name.normalizedOperatorName()
+  of "+": "add"
+  of "-": "sub"
+  of "*": "mul"
+  of "/": "div"
+  else: name
+
+proc nimCallableName*(name: string): string =
+  let normalized = name.normalizedOperatorName()
+  if normalized.isOperatorName:
+    &"`{normalized}`"
+  else:
+    name
+
+proc cppOperatorName*(name: string): string =
+  "operator" & name.normalizedOperatorName()
+
+proc pythonOperatorName*(name: string): string =
+  case name.normalizedOperatorName()
+  of "+": "__add__"
+  of "-": "__sub__"
+  of "*": "__mul__"
+  of "/": "__truediv__"
+  else: name
+
+proc objectFieldName*(property: NimNode): string =
+  if property.kind == nnkPostfix:
+    property[1].repr
+  else:
+    property.repr
+
+proc objectFields*(sym: NimNode, explicitFields: seq[ObjectField]): seq[ObjectField] =
+  if explicitFields.len > 0:
+    return explicitFields
+
+  let typ = sym.getType()
+  if typ.len > 2:
+    for fieldSym in typ[2]:
+      result.add((fieldSym.repr, fieldSym.getTypeInst()))
+
+  if result.len == 0:
+    let impl = sym.getImpl()
+    if impl.kind == nnkTypeDef and impl[2].kind == nnkObjectTy:
+      for identDefs in impl[2][2]:
+        for property in identDefs[0 .. ^3]:
+          result.add((property.objectFieldName(), identDefs[^2]))
+
 proc raises*(procSym: NimNode): bool =
   for pragma in procSym.getImpl()[4]:
     if pragma.kind == nnkExprColonExpr and pragma[0].repr == "raises":