feat: format off a stream of bytes instead of parsing (#30)

Cargo.toml

@@ -22,16 +22,19 @@ panic = "abort"
 
 [features]
 wasm = ["serde_json", "dprint-core/wasm"]
-tracing = ["dprint-core/tracing"]
+# The AST/IR generator (used only for trace output) is the only thing that
+# still needs a parser; the normal formatter works straight off the bytes.
+tracing = ["dprint-core/tracing", "dep:jsonc-parser", "dep:text_lines", "dep:dprint-core-macros"]
 
 [dependencies]
 anyhow = "1.0.64"
 dprint-core = { version = "0.67.4", features = ["formatting"] }
-dprint-core-macros = "0.1.0"
-jsonc-parser = { version = "0.32.4", features = ["error_unicode_width"] }
+dprint-core-macros = { version = "0.1.0", optional = true }
+jsonc-parser = { version = "0.32.4", features = ["error_unicode_width"], optional = true }
 serde = { version = "1.0.144", features = ["derive"] }
 serde_json = { version = "1.0", optional = true }
-text_lines = "0.6.0"
+text_lines = { version = "0.6.0", optional = true }
+unicode-width = "0.2.0"
 
 [dev-dependencies]
 debug-here = "0.2"

examples/bench.rs

@@ -0,0 +1,62 @@
+// Benchmark for issue #30. `format_text` now formats off a byte stream with no
+// parser at all — this measures its throughput.
+//
+// Run: cargo run --release --example bench
+use std::path::Path;
+use std::time::Instant;
+
+use dprint_core::configuration::*;
+use dprint_plugin_json::configuration::resolve_config;
+use dprint_plugin_json::format_text;
+
+fn gen_input(n: usize, pretty: bool) -> String {
+  // Array of n objects, each with a few keys of mixed types.
+  let mut s = String::from("[");
+  for i in 0..n {
+    if i > 0 {
+      s.push(',');
+    }
+    if pretty {
+      s.push_str("\n  ");
+    }
+    s.push_str(&format!(
+      r#"{{"id":{i},"name":"item-{i}","active":{},"tags":["a","b","c"],"score":{}.5,"nested":{{"x":1,"y":2,"z":[1,2,3,4,5]}}}}"#,
+      i % 2 == 0,
+      i
+    ));
+  }
+  if pretty {
+    s.push('\n');
+  }
+  s.push(']');
+  s
+}
+
+fn time<F: Fn()>(label: &str, iters: u32, f: F) -> f64 {
+  // warmup
+  f();
+  let start = Instant::now();
+  for _ in 0..iters {
+    f();
+  }
+  let per = start.elapsed().as_secs_f64() * 1000.0 / iters as f64;
+  println!("  {label:<28} {per:>8.3} ms/iter");
+  per
+}
+
+fn main() {
+  let global = GlobalConfiguration::default();
+  let config = resolve_config(ConfigKeyMap::new(), &global).config;
+  let path = Path::new("file.json");
+
+  for &(n, iters) in &[(1_000usize, 200u32), (20_000, 20)] {
+    for &pretty in &[false, true] {
+      let input = gen_input(n, pretty);
+      let kind = if pretty { "pretty" } else { "minified" };
+      println!("\n== n={n} {kind} ({} bytes) ==", input.len());
+      time("format_text (stream, no parse)", iters, || {
+        std::hint::black_box(format_text(path, &input, &config).unwrap());
+      });
+    }
+  }
+}

src/format_text.rs

@@ -2,15 +2,9 @@ use std::path::Path;
 
 use anyhow::Result;
 use anyhow::bail;
-use dprint_core::configuration::resolve_new_line_kind;
-use dprint_core::formatting::PrintOptions;
-use jsonc_parser::CollectOptions;
-use jsonc_parser::CommentCollectionStrategy;
-use jsonc_parser::ParseResult;
-use jsonc_parser::parse_to_ast;
 
 use super::configuration::Configuration;
-use super::generation::generate;
+use crate::streaming::format_streaming;
 
 pub fn format_text(path: &Path, text: &str, config: &Configuration) -> Result<Option<String>> {
   let result = format_text_inner(path, text, config)?;
@@ -19,54 +13,50 @@ pub fn format_text(path: &Path, text: &str, config: &Configuration) -> Result<Op
 
 fn format_text_inner(path: &Path, text: &str, config: &Configuration) -> Result<String> {
   let text = strip_bom(text);
-  let parse_result = parse(text)?;
   let is_jsonc = is_jsonc_file(path, config);
-  Ok(dprint_core::formatting::format(
-    || generate(parse_result, text, config, is_jsonc),
-    config_to_print_options(text, config),
-  ))
+  match format_streaming(text.as_bytes(), config, is_jsonc) {
+    // Input is valid UTF-8 (`&str`) and the formatter only rearranges/copies its
+    // bytes, so the output is always valid UTF-8.
+    Ok(bytes) => Ok(String::from_utf8(bytes).expect("formatted output is valid UTF-8")),
+    Err(err) => bail!(dprint_core::formatting::utils::string_utils::format_diagnostic(
+      Some((err.start, err.end)),
+      err.message,
+      text,
+    )),
+  }
 }
 
 #[cfg(feature = "tracing")]
 pub fn trace_file(text: &str, config: &Configuration) -> dprint_core::formatting::TracingResult {
-  let parse_result = parse(text).unwrap();
-
-  dprint_core::formatting::trace_printing(
-    || generate(parse_result, text, config),
-    config_to_print_options(text, config),
-  )
-}
-
-fn strip_bom(text: &str) -> &str {
-  text.strip_prefix("\u{FEFF}").unwrap_or(text)
-}
+  use dprint_core::configuration::resolve_new_line_kind;
+  use dprint_core::formatting::PrintOptions;
+  use jsonc_parser::CollectOptions;
+  use jsonc_parser::CommentCollectionStrategy;
+  use jsonc_parser::parse_to_ast;
 
-fn parse(text: &str) -> Result<ParseResult<'_>> {
   let parse_result = parse_to_ast(
     text,
     &CollectOptions {
       comments: CommentCollectionStrategy::Separate,
       tokens: true,
     },
     &Default::default(),
-  );
-  match parse_result {
-    Ok(result) => Ok(result),
-    Err(err) => bail!(dprint_core::formatting::utils::string_utils::format_diagnostic(
-      Some((err.range().start, err.range().end)),
-      &err.kind().to_string(),
-      text,
-    )),
-  }
+  )
+  .unwrap();
+
+  dprint_core::formatting::trace_printing(
+    || crate::generation::generate(parse_result, text, config, false),
+    PrintOptions {
+      indent_width: config.indent_width,
+      max_width: config.line_width,
+      use_tabs: config.use_tabs,
+      new_line_text: resolve_new_line_kind(text, config.new_line_kind),
+    },
+  )
 }
 
-fn config_to_print_options(text: &str, config: &Configuration) -> PrintOptions {
-  PrintOptions {
-    indent_width: config.indent_width,
-    max_width: config.line_width,
-    use_tabs: config.use_tabs,
-    new_line_text: resolve_new_line_kind(text, config.new_line_kind),
-  }
+fn strip_bom(text: &str) -> &str {
+  text.strip_prefix("\u{FEFF}").unwrap_or(text)
 }
 
 fn is_jsonc_file(path: &Path, config: &Configuration) -> bool {

src/lib.rs

@@ -1,8 +1,13 @@
 pub mod configuration;
 mod format_text;
+// Only the tracing path still uses the AST/IR generator; the normal formatter
+// is the streaming one.
+#[cfg(feature = "tracing")]
 mod generation;
+pub mod streaming;
 
 pub use format_text::format_text;
+pub use streaming::format_streaming;
 
 #[cfg(feature = "tracing")]
 pub use format_text::trace_file;