Add JSON translation of webgpu.yml (#561)

This is helpful for automated tooling using a different toolchain than
ours, so people don't have to deal with parsing YAML, but we can still
write in YAML.

Should simplify https://dawn-review.googlesource.com/c/dawn/+/264874
(and many other projects in the future).

Author: kainino0x

.gitattributes

@@ -1 +1,3 @@
 gen/** -linguist-detectable
+webgpu.json linguist-generated=true
+tests/extensions/extension.json linguist-generated=true

Makefile

@@ -1,40 +1,57 @@
 .PHONY: all all-help-message help check-schema fix gen gen-check doc
 
 # default target if you just type `make`
-all: all-help-message fix gen doc
+all: all-help-message gen fix doc
 
 # help message before starting `make all`
 all-help-message: help
-	@echo 'Running default targets: fix gen doc'
+	@echo 'Running default targets: gen fix doc'
 
 help:
-	@echo 'Targets are: all, help, check-schema, fix, gen, gen-check, doc'
+	@echo 'Targets are: all, help, gen, fix, gen-check, doc'
 
-check-schema: schema.json
-	go run ./gen -schema schema.json -yaml webgpu.yml -header webgpu.h -yaml tests/extensions/extension.yml -header tests/extensions/webgpu_extension.h
+# Validate schema and regenerate JSON and header from YAML
+gen: schema.json webgpu.yml tests/extensions/extension.yml
+	go run ./gen -schema schema.json \
+		-yaml webgpu.yml \
+		-out-json webgpu.json \
+		-out-header webgpu.h \
+		-yaml tests/extensions/extension.yml \
+		-out-json tests/extensions/extension.json \
+		-out-header tests/extensions/webgpu_extension.h
 
-fix: check-schema webgpu.yml tests/extensions/extension.yml
+# Regenerate files (to validate schema) then autoformat YAML
+fix: gen
 	go run ./fix -yaml webgpu.yml
 	go run ./fix -yaml tests/extensions/extension.yml
 
-gen: check-schema webgpu.yml tests/extensions/extension.yml
-
+# Verify that generation and autoformat are up to date (used for pull request checks)
 gen-check: fix gen
-	@git diff --quiet -- webgpu.h || {                                                      \
-		echo "error: The re-generated header from yml doesn't match the checked-in header"; \
-		git diff -- webgpu.h;                                                               \
-		exit 1;                                                                             \
+	@git diff --quiet -- webgpu.h || {                                                       \
+		echo "error: The re-generated header from yml doesn't match the checked-in version"; \
+		git diff -- webgpu.h;                                                                \
+		exit 1;                                                                              \
+	}
+	@git diff --quiet -- webgpu.json || {                                                                 \
+		echo "error: The re-translated webgpu.json from webgpu.yml doesn't match the checked-in version"; \
+		git diff -- webgpu.json;                                                                          \
+		exit 1;                                                                                           \
 	}
 	@git diff --quiet -- webgpu.yml || {                          \
 		echo "error: Please re-run 'make fix' to format the yml"; \
 		git diff -- webgpu.yml;                                   \
 		exit 1;                                                   \
 	}
-	@git diff --quiet -- tests/extensions/webgpu_extension.h || {                                       \
-		echo "error: The re-generated extensions header from yml doesn't match the checked-in header"; \
-		git diff -- tests/extensions/webgpu_extension.h;                                                \
+	@git diff --quiet -- tests/extensions/webgpu_extension.h || {                                      \
+		echo "error: The re-generated extension header from yml doesn't match the checked-in version"; \
+		git diff -- tests/extensions/webgpu_extension.h;                                               \
 		exit 1;                                                                                        \
 	}
+	@git diff --quiet -- tests/extensions/webgpu_extension.json || {                                  \
+		echo "error: The re-translated extension json from yml doesn't match the checked-in version"; \
+		git diff -- tests/extensions/webgpu_extension.json;                                           \
+		exit 1;                                                                                       \
+	}
 	@git diff --quiet -- tests/extensions/extension.yml || {                \
 		echo "error: Please re-run 'make fix' to format the extension yml"; \
 		git diff -- tests/extensions/extension.yml;                         \

README.md

@@ -27,23 +27,23 @@ Implementations of this header include:
  - [wgpu-native](https://github.com/gfx-rs/wgpu-native), C bindings to [wgpu](https://github.com/gfx-rs/wgpu), the Rust WebGPU implementation used in Firefox.
     - **wgpu-native does not yet implement the stable version of this header. Contributions needed!**
 
-## `webgpu.yml`
+## `webgpu.yml` and `webgpu.json`
 
-`webgpu.yml` is the main machine-readable source of truth for the C API and its documentation (in [YAML](https://yaml.org/) format). It is used to generate the official `webgpu.h` header present in this repository to generate the official documentation, and may be used by any other third party to design tools and wrappers around WebGPU-Native, especially bindings into other languages.
+`webgpu.yml` and `webgpu.json` (same contents, different formats) are the main machine-readable source of truth for the C API and its documentation. This data is used to generate the official `webgpu.h` header present in this repository to generate the official documentation, and may be used by any other third party to design tools and wrappers around WebGPU-Native, especially bindings into other languages.
 
 **If you are developing bindings of `webgpu.h` into another language and find that any additional high-level/semantic information would be useful in `webgpu.yml`, please contribute it!**
 
 ## Contributing to this project
 
-**Important:** When submitting a change, one must modify both the `webgpu.yml` and `webgpu.h` files in a consistent way. One should first edit `webgpu.yml` (the source of truth), then run `make gen` to update `webgpu.h` and finally commit both changes together.
+**Important:** When submitting a change, one must modify `webgpu.yml`, `webgpu.json`, and `webgpu.h` files in a consistent way. One should first edit `webgpu.yml` (the source of truth), then run `make gen` to update `webgpu.json` and `webgpu.h`, and finally commit all changes together.
 
 Here are some details about the structure of this repository.
 
 ### Main files
 
  - `webgpu.h` is the one and only header file that defines the WebGPU C API. Only this needs to be integrated in a C project that links against a WebGPU implementation. (But be sure to use the headers from your implementation if you need any extensions, or if the implementation doesn't match this header exactly.)
 
- - `webgpu.yml` - see above.
+ - `webgpu.yml` and `webgpu.json` - see above.
 
  - `schema.json` is the [JSON schema](https://json-schema.org/) that formally specifies the structure of `webgpu.yml`.
 

gen/main.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"encoding/json"
 	"flag"
 	"os"
 	"path/filepath"
@@ -16,19 +17,21 @@ import (
 var tmpl string
 
 var (
-	schemaPath  string
-	headerPaths StringListFlag
-	yamlPaths   StringListFlag
-	extPrefix   bool
+	schemaPath     string
+	yamlPaths      StringListFlag
+	outJsonPaths   StringListFlag
+	outHeaderPaths StringListFlag
+	extPrefix      bool
 )
 
 func main() {
 	flag.StringVar(&schemaPath, "schema", "", "path of the json schema")
 	flag.Var(&yamlPaths, "yaml", "path of the yaml spec")
-	flag.Var(&headerPaths, "header", "output path of the header")
-	flag.BoolVar(&extPrefix, "extprefix", true, "append prefix to extension identifiers")
+	flag.Var(&outJsonPaths, "out-json", "output path of the json version of the yaml")
+	flag.Var(&outHeaderPaths, "out-header", "output path of the header")
+	flag.BoolVar(&extPrefix, "ext-prefix", true, "append prefix to extension identifiers")
 	flag.Parse()
-	if schemaPath == "" || len(headerPaths) == 0 || len(yamlPaths) == 0 || len(headerPaths) != len(yamlPaths) {
+	if schemaPath == "" || len(yamlPaths) == 0 || len(outHeaderPaths) != len(yamlPaths) || len(outJsonPaths) != len(yamlPaths) {
 		flag.Usage()
 		os.Exit(1)
 	}
@@ -45,19 +48,23 @@ func main() {
 
 	// Generate the header files
 	for i, yamlPath := range yamlPaths {
-		headerPath := headerPaths[i]
-		headerFileName := filepath.Base(headerPath)
-		headerFileNameSplit := strings.Split(headerFileName, ".")
-		if len(headerFileNameSplit) != 2 {
-			panic("got invalid header file name: " + headerFileName)
+		outHeaderPath := outHeaderPaths[i]
+		outHeaderFileName := filepath.Base(outHeaderPath)
+		outHeaderFileNameSplit := strings.Split(outHeaderFileName, ".")
+		if len(outHeaderFileNameSplit) != 2 {
+			panic("got invalid out-header file name: " + outHeaderFileName)
 		}
 
+		outJsonPath := outJsonPaths[i]
+
 		src, err := os.ReadFile(yamlPath)
 		if err != nil {
 			panic(err)
 		}
 
-		dst, err := os.Create(headerPath)
+		ConvertToJSON(src, outJsonPath)
+
+		dst, err := os.Create(outHeaderPath)
 		if err != nil {
 			panic(err)
 		}
@@ -75,7 +82,7 @@ func main() {
 		}
 		g := &Generator{
 			Yml:        &yml,
-			HeaderName: headerFileNameSplit[0],
+			HeaderName: outHeaderFileNameSplit[0],
 			ExtPrefix:  prefix,
 		}
 		if err := g.Gen(dst); err != nil {
@@ -168,3 +175,50 @@ func SortAndTransform(yml *Yml) {
 		}
 	}
 }
+
+func ConvertToJSON(ymlString []byte, outJsonPath string) {
+	var body interface{}
+	if err := yaml.Unmarshal(ymlString, &body); err != nil {
+		panic(err)
+	}
+
+	switch b := body.(type) {
+	case map[string]interface{}:
+		// Insert an extra copy of the copyright that will get sorted at the
+		// top, and auto-gen warning.
+		b["__copyright"] = b["copyright"]
+		b["_comment"] = "AUTO-GENERATED FILE! Edit webgpu.yml instead."
+	default:
+		panic("unexpected")
+	}
+
+	body = convertToJSONHelper(body)
+
+	if outJson, err := json.MarshalIndent(body, "", "  "); err != nil {
+		panic(err)
+	} else {
+		outJson = append(outJson, '\n')
+		if err := os.WriteFile(outJsonPath, outJson, 0644); err != nil {
+			panic(err)
+		}
+	}
+}
+
+// From https://stackoverflow.com/a/40737676
+// Note the original key order is not preserved as Go's map isn't ordered.
+// This is fine because JSON keys are unordered per spec.
+func convertToJSONHelper(i interface{}) interface{} {
+	switch x := i.(type) {
+	case map[interface{}]interface{}:
+		m2 := map[string]interface{}{}
+		for k, v := range x {
+			m2[k.(string)] = convertToJSONHelper(v)
+		}
+		return m2
+	case []interface{}:
+		for i, v := range x {
+			x[i] = convertToJSONHelper(v)
+		}
+	}
+	return i
+}