Parser architecture rework

.gitignore

@@ -1,3 +1,3 @@
-test/*.json
-test/test
-test/*.pprof
+benchmarks/*.json
+benchmarks/test
+*.pprof

README.md

@@ -1,7 +1,5 @@
 # libjson
 
-> WARNING: libjson is currently a work in progress :)
-
 Fast and minimal JSON parser written in and for Go with a JIT query language
 
 ```go
@@ -13,16 +11,17 @@ import (
 
 func main() {
 	input := `{ "hello": {"world": ["hi"] } }`
-	jsonObj, _ := New(input) // or libjson.NewReader(r io.Reader)
+	jsonObj, _ := libjson.New([]byte(input)) // or libjson.NewReader(r io.Reader)
 
 	// accessing values
-	fmt.Println(Get[string](jsonObj, ".hello.world.0")) // hi, nil
+	fmt.Println(libjson.Get[string](jsonObj, ".hello.world.0")) // hi, nil
 }
 ```
 
 ## Features
 
-- [ECMA 404](https://ecma-international.org/wp-content/uploads/ECMA-404_2nd_edition_december_2017.pdf)
+- Parser consumes and mutates the input to make most operations zero copy and zero alloc
+- [ECMA 404](https://ecma-international.org/publications-and-standards/standards/ecma-404/)
   and [rfc8259](https://www.rfc-editor.org/rfc/rfc8259) compliant
   - tests against [JSONTestSuite](https://github.com/nst/JSONTestSuite), see
     [Parsing JSON is a Minefield
@@ -35,6 +34,15 @@ func main() {
 - caching of queries with `libjson.Compile`, just in time caching of queries
 - serialisation via `json.Marshal`
 
+## Why is it faster than encoding/json?
+
+- zero-copy strings
+- mutate input for string escaping instead of allocating a new one
+- no allocations for strings, views into the original input
+- no reflection
+- no copies for map keys
+- very simple lexer and parser
+
 ## Benchmarks
 
 ![libjson-vs-encodingjson](https://github.com/user-attachments/assets/b11bcce4-e7db-4c45-ab42-45a2042e2a51)

benchmarks/bench.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+echo "generating example data"
+python3 gen.py
+
+echo "building executable"
+rm ./test
+go build -o ./test ../cmd/lj.go
+
+for SIZE in 1MB 5MB 10MB 100MB; do
+    hyperfine \
+        --warmup 1 \
+        --runs 10 \
+        "./test -s ./${SIZE}.json" \
+        "./test -s -libjson=false ./${SIZE}.json"
+done

benchmarks/gen.py

@@ -0,0 +1,47 @@
+from os.path import exists
+import math
+import json
+
+sizes =[1,5,10,100]
+
+line = json.dumps({
+    "id": 12345,
+    "name": "very_long_string_with_escapes_and_unicode_abcdefghijklmnopqrstuvwxyz_0123456789",
+    "description": "This string contains\nmultiple\nlines\nand \"quotes\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"",
+    "nested": {
+        "level1": {
+            "level2": {
+                "level3": {
+                    "level4": {
+                        "array": [
+                            "short",
+                            "string_with_escape\\n",
+                            "another\\tvalue",
+                            "unicode\u2603",
+                            "escaped_quote_\"_and_backslash_\\",
+                            11234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,234567890,
+                            -1.2345e67,
+                            3.1415926535897932384626433832795028841971,
+                            True,
+                            False,
+                            None,
+                            "\u0041\u0042\u0043\u00A9\u20AC\u0041\u0042\u0043\u00A9\u20AC\u0041\u0042\u0043\u00A9\u20AC\u0041\u0042\u0043\u00A9\u20AC\u0041\u0042\u0043\u00A9\u20AC\u0041\u0042\u0043\u00A9\u20AC\u0041\u0042\u0043\u00A9\u20AC\u0041\u0042\u0043\u00A9\u20AC\u0041\u0042\u0043\u00A9\u20AC\u0041\u0042\u0043\u00A9\u20AC\u0041\u0042\u0043\u00A9\u20AC",
+                            "mix\\n\\t\\r\\\\\\\"end"
+                        ]
+                    }
+                }
+            }
+        }
+    }
+})
+
+def write_data(size: int): 
+    name = f"{size}MB.json"
+    if not exists(name):
+        with open(name, mode="w", encoding="utf8") as f:
+            f.write("[\n")
+            size = math.floor((size*1000000)/len(line))
+            f.write(",\n".join([line for _ in range(0, size)]))
+            f.write("\n]")
+
+[write_data(size) for size in sizes]

cmd/lj.go

@@ -1,9 +1,14 @@
 package main
 
 import (
+	"encoding/json"
+	"flag"
 	"fmt"
 	"log"
 	"os"
+	"path/filepath"
+	"runtime/debug"
+	"runtime/pprof"
 
 	"github.com/xnacly/libjson"
 )
@@ -16,17 +21,69 @@ func Must[T any](t T, err error) T {
 }
 
 func main() {
-	args := os.Args
+	noGc := flag.Bool("nogc", false, "disable the go garbage collector")
+	useLibjson := flag.Bool("libjson", true, "use libjson, if false use encoding/json")
+	usePprof := flag.Bool("pprof", false, "use pprof cpu tracing")
+	query := flag.String("q", ".", "query the parsed json")
+	silent := flag.Bool("s", false, "no stdoutput")
+	escape := flag.Bool("e", false, "escapes input with Gos '%#+v'")
+	flag.Parse()
+
+	if *noGc {
+		debug.SetGCPercent(-1)
+	}
+
+	args := flag.Args()
+
+	var filePath string
 	var file *os.File
 	if info, err := os.Stdin.Stat(); err != nil || info.Mode()&os.ModeCharDevice != 0 { // we are in a pipe
-		if len(args) == 1 {
-			log.Fatalln("Wanted a file as first argument, got nothing, exiting")
+		if len(args) == 0 {
+			log.Fatalln("Wanted a file as an argument, got nothing, exiting")
 		}
-		file = Must(os.Open(args[1]))
+		filePath = args[0]
+		file = Must(os.Open(filePath))
 	} else {
 		file = os.Stdin
+		filePath = "stdin"
+	}
+
+	if *usePprof {
+		f, err := os.Create(filepath.Base(filePath) + ".pprof")
+		if err != nil {
+			panic(err)
+		}
+		pprof.StartCPUProfile(f)
+		defer pprof.StopCPUProfile()
+	}
+
+	if *useLibjson {
+		out := Must(libjson.NewReader(file))
+		if !*silent {
+			out := Must(libjson.Get[any](&out, *query))
+			if *escape {
+				fmt.Printf("%#+v\n", out)
+			} else {
+				fmt.Println(out)
+			}
+		}
+	} else {
+		if *query != "." {
+			panic("With -libjson=false, there is no support for querying the json")
+		}
+
+		decoder := json.NewDecoder(file)
+		var out any
+		if err := decoder.Decode(&out); err != nil {
+			panic(err)
+		}
+
+		if !*silent {
+			if *escape {
+				fmt.Printf("%#+v\n", out)
+			} else {
+				fmt.Println(out)
+			}
+		}
 	}
-	query := os.Args[len(os.Args)-1]
-	json := Must(libjson.NewReader(file))
-	fmt.Printf("%+#v\n", Must(libjson.Get[any](&json, query)))
 }

go.mod

@@ -1,6 +1,6 @@
 module github.com/xnacly/libjson
 
-go 1.23.0
+go 1.26.0
 
 require github.com/stretchr/testify v1.9.0
 

hex.go

@@ -0,0 +1,54 @@
+package libjson
+
+import "errors"
+
+var invalid_hex_err = errors.New("invalid hex")
+
+var hexTable [256]byte
+
+func init() {
+	for i := 0; i < 256; i++ {
+		hexTable[i] = 0xFF
+	}
+	for i := byte('0'); i <= '9'; i++ {
+		hexTable[i] = i - '0'
+	}
+	for i := byte('a'); i <= 'f'; i++ {
+		hexTable[i] = i - 'a' + 10
+	}
+	for i := byte('A'); i <= 'F'; i++ {
+		hexTable[i] = i - 'A' + 10
+	}
+}
+
+// hex4 converts 4 ASCII hex bytes to a rune.
+// Returns an error if any byte is invalid.
+func hex4(b []byte) (r rune, err error) {
+	var v byte
+
+	v = hexTable[b[0]]
+	if v == 0xFF {
+		return 0, invalid_hex_err
+	}
+	r = rune(v) << 12
+
+	v = hexTable[b[1]]
+	if v == 0xFF {
+		return 0, invalid_hex_err
+	}
+	r |= rune(v) << 8
+
+	v = hexTable[b[2]]
+	if v == 0xFF {
+		return 0, invalid_hex_err
+	}
+	r |= rune(v) << 4
+
+	v = hexTable[b[3]]
+	if v == 0xFF {
+		return 0, invalid_hex_err
+	}
+	r |= rune(v)
+
+	return r, nil
+}

json.go

@@ -9,16 +9,17 @@ func NewReader(r io.Reader) (JSON, error) {
 	if err != nil {
 		return JSON{}, err
 	}
-	p := parser{l: lexer{data: data}}
+	p := parser{l: lexer{data: data, len: len(data)}}
 	obj, err := p.parse(data)
 	if err != nil {
 		return JSON{}, err
 	}
 	return JSON{obj}, nil
 }
 
+// data is consumed and possibly mutated, DO NOT REUSE
 func New(data []byte) (JSON, error) {
-	p := parser{l: lexer{data: data}}
+	p := parser{l: lexer{data: data, len: len(data)}}
 	obj, err := p.parse(data)
 	if err != nil {
 		return JSON{}, err