Starlark Experiment

bench/resources/starlark/allocation.py

@@ -0,0 +1,50 @@
+class ImmutableObj:
+    def __init__(self, f1, f2, f3, f4, f5, f6, f7, f8, f9, f10):
+        self.f1 = f1; self.f2 = f2; self.f3 = f3; self.f4 = f4; self.f5 = f5
+        self.f6 = f6; self.f7 = f7; self.f8 = f8; self.f9 = f9; self.f10 = f10
+
+    def withF1(self, v): return ImmutableObj(v, self.f2, self.f3, self.f4, self.f5, self.f6, self.f7, self.f8, self.f9, self.f10)
+    def withF2(self, v): return ImmutableObj(self.f1, v, self.f3, self.f4, self.f5, self.f6, self.f7, self.f8, self.f9, self.f10)
+    def withF3(self, v): return ImmutableObj(self.f1, self.f2, v, self.f4, self.f5, self.f6, self.f7, self.f8, self.f9, self.f10)
+    def withF4(self, v): return ImmutableObj(self.f1, self.f2, self.f3, v, self.f5, self.f6, self.f7, self.f8, self.f9, self.f10)
+    def withF5(self, v): return ImmutableObj(self.f1, self.f2, self.f3, self.f4, v, self.f6, self.f7, self.f8, self.f9, self.f10)
+    def withF6(self, v): return ImmutableObj(self.f1, self.f2, self.f3, self.f4, self.f5, v, self.f7, self.f8, self.f9, self.f10)
+    def withF7(self, v): return ImmutableObj(self.f1, self.f2, self.f3, self.f4, self.f5, self.f6, v, self.f8, self.f9, self.f10)
+    def withF8(self, v): return ImmutableObj(self.f1, self.f2, self.f3, self.f4, self.f5, self.f6, self.f7, v, self.f9, self.f10)
+    def withF9(self, v): return ImmutableObj(self.f1, self.f2, self.f3, self.f4, self.f5, self.f6, self.f7, self.f8, v, self.f10)
+    def withF10(self, v): return ImmutableObj(self.f1, self.f2, self.f3, self.f4, self.f5, self.f6, self.f7, self.f8, self.f9, v)
+
+class MutableObj:
+    def __init__(self, f1, f2, f3, f4, f5, f6, f7, f8, f9, f10):
+        self.f1 = f1; self.f2 = f2; self.f3 = f3; self.f4 = f4; self.f5 = f5
+        self.f6 = f6; self.f7 = f7; self.f8 = f8; self.f9 = f9; self.f10 = f10
+
+def benchmark_immutable(iterations):
+    obj = ImmutableObj(0,0,0,0,0,0,0,0,0,0)
+    for i in range(int(iterations)):
+        obj = obj.withF1(obj.f1 + i)
+        obj = obj.withF2(obj.f2 + i)
+        obj = obj.withF3(obj.f3 + i)
+        obj = obj.withF4(obj.f4 + i)
+        obj = obj.withF5(obj.f5 + i)
+        obj = obj.withF6(obj.f6 + i)
+        obj = obj.withF7(obj.f7 + i)
+        obj = obj.withF8(obj.f8 + i)
+        obj = obj.withF9(obj.f9 + i)
+        obj = obj.withF10(obj.f10 + i)
+    return obj.f1 + obj.f2 + obj.f3 + obj.f4 + obj.f5 + obj.f6 + obj.f7 + obj.f8 + obj.f9 + obj.f10
+
+def benchmark_mutable(iterations):
+    obj = MutableObj(0,0,0,0,0,0,0,0,0,0)
+    for i in range(int(iterations)):
+        obj.f1 += i
+        obj.f2 += i
+        obj.f3 += i
+        obj.f4 += i
+        obj.f5 += i
+        obj.f6 += i
+        obj.f7 += i
+        obj.f8 += i
+        obj.f9 += i
+        obj.f10 += i
+    return obj.f1 + obj.f2 + obj.f3 + obj.f4 + obj.f5 + obj.f6 + obj.f7 + obj.f8 + obj.f9 + obj.f10

bench/resources/starlark/benchmarks.py

@@ -0,0 +1,71 @@
+import math
+
+def makeArray(n, func):
+    return [func(i) for i in range(int(n))]
+
+def pow_bench(n):
+    n_int = int(n)
+    res = 0
+    for i in range(n_int):
+        res = 3 ** 2
+    return res
+
+def floor_bench(n):
+    n_int = int(n)
+    res = 0
+    for i in range(n_int):
+        res = math.floor(10.99999)
+    return res
+
+def ceil_bench(n):
+    n_int = int(n)
+    res = 0
+    for i in range(n_int):
+        res = math.ceil(10.99999)
+    return res
+
+def sqrt_bench(n):
+
+    n_int = int(n)
+
+    res = 0
+
+    for i in range(n_int):
+
+        res = math.sqrt(16)
+
+    return res
+
+
+
+def filter_bench(n):
+
+    n_int = int(n)
+
+    return [x for x in range(1, n_int + 1) if x % 2 == 0]
+
+
+
+def map_bench(n):
+
+    n_int = int(n)
+
+    return [x * x for x in range(1, n_int + 1)]
+
+
+
+
+
+def filter_bench(n):
+
+    n_int = int(n)
+
+    return [x for x in range(1, n_int + 1) if x % 2 == 0]
+
+
+
+def map_bench(n):
+
+    n_int = int(n)
+
+    return [x * x for x in range(1, n_int + 1)]

bench/resources/starlark/constants.star

@@ -0,0 +1,8 @@
+# constants.star
+MY_LIST = [1, 2, 3]
+
+def get_list():
+    return MY_LIST
+
+def create_fresh_list():
+    return [4, 5, 6]

bench/resources/starlark/expensive.py

@@ -0,0 +1,9 @@
+def compute():
+    res = 0
+    # A loop large enough to be measured, but small enough for a benchmark
+    for i in range(1000000):
+        res = (res + i) % 1000000
+    return res
+
+# Top-level execution
+X = compute()

bench/resources/starlark/test_fail.star

@@ -0,0 +1,9 @@
+load("constants.star", "get_list")
+
+def test_global_mutation():
+    g_list = get_list()
+    print("Attempting to mutate global list...")
+    g_list.append(4)
+    print("ERROR: Should not reach here!")
+
+test_global_mutation()

bench/resources/starlark/test_semantics.star

@@ -0,0 +1,23 @@
+load("constants.star", "get_list", "create_fresh_list")
+
+def test_semantics():
+    # 1. Test global list mutation (should fail)
+    g_list = get_list()
+    print("Global list:", g_list)
+    try:
+        g_list.append(4)
+        print("ERROR: Successfully mutated global list!")
+    except Error as e:
+        print("SUCCESS: Caught expected error when mutating global list:", e)
+
+    # 2. Test fresh list mutation (should succeed)
+    f_list = create_fresh_list()
+    print("Fresh list before:", f_list)
+    f_list.append(7)
+    print("Fresh list after:", f_list)
+    if f_list == [4, 5, 6, 7]:
+        print("SUCCESS: Successfully mutated fresh list.")
+    else:
+        print("ERROR: Fresh list mutation failed.")
+
+test_semantics()

bench/resources/starlark/test_success.star

@@ -0,0 +1,11 @@
+load("constants.star", "get_list", "create_fresh_list")
+
+def test_fresh_mutation():
+    f_list = create_fresh_list()
+    print("Fresh list before:", f_list)
+    f_list.append(7)
+    print("Fresh list after:", f_list)
+    if f_list == [4, 5, 6, 7]:
+        print("SUCCESS: Fresh list mutation worked as expected.")
+
+test_fresh_mutation()

bench/src/sjsonnet/bench/MainBenchmark.scala

@@ -33,8 +33,10 @@ object MainBenchmark {
       Map("var1" -> "\"test\"", "var2" -> """{"x": 1, "y": 2}"""),
       Map.empty[String, String],
       OsPath(wd),
-      importer = SjsonnetMainBase
-        .resolveImport(config.getOrderedJpaths.map(os.Path(_, wd)).map(OsPath(_)), None),
+      importer = new SjsonnetMainBase.SimpleImporter(
+        config.getOrderedJpaths.map(os.Path(_, wd)).map(OsPath(_)),
+        None
+      ),
       parseCache = parseCache
     )
     val renderer = new Renderer(new StringWriter, indent = 3)

bench/src/sjsonnet/bench/MaterializerBenchmark.scala

@@ -34,11 +34,10 @@ class MaterializerBenchmark {
       Map.empty[String, String],
       Map.empty[String, String],
       OsPath(wd),
-      importer = SjsonnetMainBase
-        .resolveImport(
-          config.getOrderedJpaths.map(os.Path(_, wd)).map(OsPath(_)).toIndexedSeq,
-          None
-        ),
+      importer = new SjsonnetMainBase.SimpleImporter(
+        config.getOrderedJpaths.map(os.Path(_, wd)).map(OsPath(_)),
+        None
+      ),
       parseCache = new DefaultParseCache
     )
     value = interp.evaluate(os.read(path), OsPath(path)).toOption.get

bench/src/sjsonnet/bench/RunProfiler.scala

@@ -18,8 +18,10 @@ object RunProfiler extends App {
     Map.empty[String, String],
     Map.empty[String, String],
     OsPath(wd),
-    importer = SjsonnetMainBase
-      .resolveImport(config.getOrderedJpaths.map(os.Path(_, wd)).map(OsPath(_)).toIndexedSeq, None),
+    importer = new SjsonnetMainBase.SimpleImporter(
+      config.getOrderedJpaths.map(os.Path(_, wd)).map(OsPath(_)),
+      None
+    ),
     parseCache = parseCache
   ) {
     override def createEvaluator(

bench/src/sjsonnet/bench/StarlarkAllocationBenchmark.scala

@@ -0,0 +1,81 @@
+package sjsonnet.bench
+
+import org.openjdk.jmh.annotations.*
+import org.openjdk.jmh.infra.*
+import sjsonnet.*
+import sjsonnet.starlark.*
+
+import java.util.concurrent.TimeUnit
+
+@BenchmarkMode(Array(Mode.AverageTime))
+@Fork(1)
+@Threads(1)
+@Warmup(iterations = 20, time = 1, timeUnit = TimeUnit.SECONDS)
+@Measurement(iterations = 15, time = 1, timeUnit = TimeUnit.SECONDS)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@State(Scope.Benchmark)
+class StarlarkAllocationBenchmark {
+
+  private val wd = {
+    var curr = os.pwd
+    while (curr.segmentCount > 0 && !os.exists(curr / "bench" / "resources" / "starlark")) {
+      curr = curr / os.up
+    }
+    OsPath(curr / "bench" / "resources" / "starlark")
+  }
+
+  private val importer = new Importer {
+    def resolve(docBase: Path, importName: String): Option[Path] = Some(docBase / importName)
+    def read(path: Path, binaryData: Boolean): Option[ResolvedFile] = {
+      val p = path.asInstanceOf[OsPath].p
+      if (os.exists(p)) Some(StaticResolvedFile(os.read(p))) else None
+    }
+  }
+
+  private var manager: StarlarkContextManager = _
+  private var interp: Interpreter = _
+
+  @Setup
+  def setup(): Unit = {
+    manager = Platform.makeStarlarkContextManager().get.asInstanceOf[StarlarkContextManager]
+    interp = new Interpreter(
+      extVars = Map.empty, tlaVars = Map.empty, wd = wd, importer = importer,
+      parseCache = new DefaultParseCache, settings = Settings.default,
+      variableResolver = {
+        case "importstarlark" => Some(Platform.makeStarlarkImportFunc(manager, importer))
+        case _ => None
+      }
+    )
+  }
+
+  @TearDown
+  def tearDown(): Unit = {
+    Platform.closeStarlarkContextManager(manager)
+  }
+
+  private def runJsonnet(code: String): ujson.Value = {
+    StarlarkEngine.currentManager.set(manager)
+    try {
+      interp.interpret(code, wd / "bench.jsonnet") match {
+        case Right(v) => v
+        case Left(err) => throw new RuntimeException(err)
+      }
+    } finally {
+      StarlarkEngine.currentManager.remove()
+    }
+  }
+
+  @Benchmark
+  def immutable_updates_10k(bh: Blackhole): Unit = {
+    // 10,000 loop iterations * 10 replacements = 100,000 allocations
+    val code = """local b = importstarlark("allocation.py"); b.benchmark_immutable(10000)"""
+    bh.consume(runJsonnet(code))
+  }
+
+  @Benchmark
+  def mutable_updates_10k(bh: Blackhole): Unit = {
+    // 10,000 loop iterations * 10 mutations = 1 object
+    val code = """local b = importstarlark("allocation.py"); b.benchmark_mutable(10000)"""
+    bh.consume(runJsonnet(code))
+  }
+}