Move arena & helpers into their own crate

Cargo.toml

@@ -1,21 +1,13 @@
-[package]
-name = "edit"
-version = "1.2.1"
+[workspace]
+default-members = ["crates/edit"]
+members = ["crates/*"]
+resolver = "2"
+
+[workspace.package]
 edition = "2024"
-rust-version = "1.87"
-readme = "README.md"
-repository = "https://github.com/microsoft/edit"
-homepage = "https://github.com/microsoft/edit"
 license = "MIT"
-categories = ["text-editors"]
-build = "build/main.rs"
-
-[[bench]]
-name = "lib"
-harness = false
-
-[features]
-debug-latency = []
+repository = "https://github.com/microsoft/edit"
+rust-version = "1.88"
 
 # We use `opt-level = "s"` as it significantly reduces binary size.
 # We could then use the `#[optimize(speed)]` attribute for spot optimizations.
@@ -34,35 +26,7 @@ incremental = true          # Improves re-compile times
 codegen-units = 16          # Make compiling criterion faster (16 is the default, but profile.release sets it to 1)
 lto = "thin"                # Similarly, speed up linking by a ton
 
-[dependencies]
-
-[target.'cfg(unix)'.dependencies]
-libc = "0.2"
-
-[build-dependencies]
-# The default toml crate bundles its dependencies with bad compile times. Thanks.
-# Thankfully toml-span exists. FWIW the alternative is yaml-rust (without the 2 suffix).
-toml-span = { version = "0.5", default-features = false }
-
-[target.'cfg(windows)'.build-dependencies]
-winresource = { version = "0.1.22", default-features = false }
-
-[target.'cfg(windows)'.dependencies.windows-sys]
-version = "0.59"
-features = [
-    "Win32_Globalization",
-    "Win32_Security",
-    "Win32_Storage_FileSystem",
-    "Win32_System_Console",
-    "Win32_System_Diagnostics_Debug",
-    "Win32_System_IO",
-    "Win32_System_LibraryLoader",
-    "Win32_System_Memory",
-    "Win32_System_Threading",
-]
-
-[dev-dependencies]
-criterion = { version = "0.7", features = ["html_reports"] }
-serde = { version = "1.0", features = ["derive"] }
-serde_json = { version = "1.0" }
-zstd = { version = "0.13", default-features = false }
+[workspace.dependencies]
+edit = { path = "./crates/edit" }
+stdext = { path = "./crates/stdext" }
+unicode-gen = { path = "./crates/unicode-gen" }

crates/edit/Cargo.toml

@@ -0,0 +1,52 @@
+[package]
+name = "edit"
+version = "1.2.1"
+
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+rust-version.workspace = true
+
+build = "build/main.rs"
+categories = ["text-editors"]
+
+[[bench]]
+name = "lib"
+harness = false
+
+[features]
+debug-latency = []
+
+[dependencies]
+stdext.workspace = true
+
+[target.'cfg(unix)'.dependencies]
+libc = "0.2"
+
+[build-dependencies]
+stdext.workspace = true
+# The default toml crate bundles its dependencies with bad compile times. Thanks.
+# Thankfully toml-span exists. FWIW the alternative is yaml-rust (without the 2 suffix).
+toml-span = { version = "0.6", default-features = false }
+
+[target.'cfg(windows)'.build-dependencies]
+winresource = { version = "0.1", default-features = false }
+
+[target.'cfg(windows)'.dependencies.windows-sys]
+version = "0.61"
+features = [
+    "Win32_Globalization",
+    "Win32_Security",
+    "Win32_Storage_FileSystem",
+    "Win32_System_Console",
+    "Win32_System_Diagnostics_Debug",
+    "Win32_System_IO",
+    "Win32_System_LibraryLoader",
+    "Win32_System_Threading",
+]
+
+[dev-dependencies]
+criterion = { version = "0.7", features = ["html_reports"] }
+serde = { version = "1.0", features = ["derive"] }
+serde_json = { version = "1.0" }
+zstd = { version = "0.13", default-features = false }

benches/lib.rs → crates/edit/benches/lib.rs

@@ -8,8 +8,9 @@ use std::{mem, vec};
 use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main};
 use edit::helpers::*;
 use edit::simd::MemsetSafe;
-use edit::{arena, buffer, hash, oklab, simd, unicode};
+use edit::{buffer, hash, oklab, simd, unicode};
 use serde::Deserialize;
+use stdext::arena;
 
 #[derive(Deserialize)]
 pub struct EditingTracePatch(pub usize, pub usize, pub String);
@@ -29,7 +30,7 @@ pub struct EditingTraceData {
 }
 
 fn bench_buffer(c: &mut Criterion) {
-    let data = include_bytes!("../assets/editing-traces/rustcode.json.zst");
+    let data = include_bytes!("../../../assets/editing-traces/rustcode.json.zst");
     let data = zstd::decode_all(Cursor::new(data)).unwrap();
     let data: EditingTraceData = serde_json::from_slice(&data).unwrap();
     let mut patches_with_coords = Vec::new();

build/helpers.rs → crates/edit/build/helpers.rs

@@ -1,3 +1,6 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
 use std::env::VarError;
 
 pub fn env_opt(name: &str) -> String {

build/i18n.rs → crates/edit/build/i18n.rs

@@ -1,3 +1,6 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
 use std::collections::{BTreeMap, HashMap, HashSet};
 use std::fmt::Write as _;
 

build/main.rs → crates/edit/build/main.rs

@@ -16,6 +16,8 @@ enum TargetOs {
 }
 
 fn main() {
+    stdext::arena::init(128 * 1024 * 1024).unwrap();
+
     let target_os = match env_opt("CARGO_CFG_TARGET_OS").as_str() {
         "windows" => TargetOs::Windows,
         "macos" | "ios" => TargetOs::MacOS,
@@ -29,16 +31,16 @@ fn main() {
 }
 
 fn compile_i18n() {
-    const PATH: &str = "i18n/edit.toml";
+    let i18n_path = "../../i18n/edit.toml";
 
-    let i18n = std::fs::read_to_string(PATH).unwrap();
+    let i18n = std::fs::read_to_string(i18n_path).unwrap();
     let contents = i18n::generate(&i18n);
     let out_dir = env_opt("OUT_DIR");
     let path = format!("{out_dir}/i18n_edit.rs");
-    std::fs::write(path, contents).unwrap();
+    std::fs::write(&path, contents).unwrap();
 
     println!("cargo::rerun-if-env-changed=EDIT_CFG_LANGUAGES");
-    println!("cargo::rerun-if-changed={PATH}");
+    println!("cargo::rerun-if-changed={i18n_path}");
 }
 
 fn configure_icu(target_os: TargetOs) {
@@ -111,13 +113,16 @@ fn configure_windows_binary(target_os: TargetOs) {
         return;
     }
 
-    const PATH: &str = "src/bin/edit/edit.exe.manifest";
-    println!("cargo::rerun-if-changed={PATH}");
+    let manifest_path = "src/bin/edit/edit.exe.manifest";
+    let icon_path = "../../assets/edit.ico";
+
     winresource::WindowsResource::new()
-        .set_manifest_file(PATH)
+        .set_manifest_file(manifest_path)
         .set("FileDescription", "Microsoft Edit")
         .set("LegalCopyright", "© Microsoft Corporation. All rights reserved.")
-        .set_icon("assets/edit.ico")
+        .set_icon(icon_path)
         .compile()
         .unwrap();
+
+    println!("cargo::rerun-if-changed={manifest_path}");
 }

src/apperr.rs → crates/edit/src/apperr.rs

@@ -3,6 +3,7 @@
 
 //! Provides a transparent error type for edit.
 
+use std::alloc::AllocError;
 use std::{io, result};
 
 use crate::sys;
@@ -40,3 +41,11 @@ impl From<io::Error> for Error {
         sys::io_error_to_apperr(err)
     }
 }
+
+impl From<AllocError> for Error {
+    fn from(_: AllocError) -> Self {
+        // TODO: Technically this breaks if the AllocError isn't recent. By then, the errno may
+        // have been tained. But the stdlib AllocError is a bad type with no way to carry info.
+        sys::get_last_error()
+    }
+}

src/base64.rs → crates/edit/src/base64.rs

@@ -3,7 +3,7 @@
 
 //! Base64 facilities.
 
-use crate::arena::ArenaString;
+use stdext::arena::ArenaString;
 
 const CHARSET: [u8; 64] = *b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
 
@@ -79,8 +79,9 @@ pub fn encode(dst: &mut ArenaString, src: &[u8]) {
 
 #[cfg(test)]
 mod tests {
+    use stdext::arena::{Arena, ArenaString};
+
     use super::encode;
-    use crate::arena::{Arena, ArenaString};
 
     #[test]
     fn test_basic() {

src/bin/edit/draw_filepicker.rs → crates/edit/src/bin/edit/draw_filepicker.rs

@@ -5,12 +5,12 @@ use std::cmp::Ordering;
 use std::fs;
 use std::path::{Path, PathBuf};
 
-use edit::arena::scratch_arena;
 use edit::framebuffer::IndexedColor;
 use edit::helpers::*;
 use edit::input::{kbmod, vk};
 use edit::tui::*;
 use edit::{icu, path};
+use stdext::arena::scratch_arena;
 
 use crate::localization::*;
 use crate::state::*;

src/bin/edit/draw_menubar.rs → crates/edit/src/bin/edit/draw_menubar.rs

@@ -1,10 +1,10 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
-use edit::arena_format;
 use edit::helpers::*;
 use edit::input::{kbmod, vk};
 use edit::tui::*;
+use stdext::arena_format;
 
 use crate::localization::*;
 use crate::state::*;

src/bin/edit/draw_statusbar.rs → crates/edit/src/bin/edit/draw_statusbar.rs

@@ -1,13 +1,14 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
-use edit::arena::scratch_arena;
 use edit::framebuffer::{Attributes, IndexedColor};
 use edit::fuzzy::score_fuzzy;
 use edit::helpers::*;
+use edit::icu;
 use edit::input::vk;
 use edit::tui::*;
-use edit::{arena_format, icu};
+use stdext::arena::scratch_arena;
+use stdext::arena_format;
 
 use crate::localization::*;
 use crate::state::*;

src/bin/edit/localization.rs → crates/edit/src/bin/edit/localization.rs

@@ -1,9 +1,9 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
-use edit::arena::scratch_arena;
 use edit::helpers::AsciiStringHelpers;
 use edit::sys;
+use stdext::arena::scratch_arena;
 
 include!(concat!(env!("OUT_DIR"), "/i18n_edit.rs"));
 

src/bin/edit/main.rs → crates/edit/src/bin/edit/main.rs

@@ -22,16 +22,17 @@ use draw_editor::*;
 use draw_filepicker::*;
 use draw_menubar::*;
 use draw_statusbar::*;
-use edit::arena::{self, Arena, ArenaString, scratch_arena};
 use edit::framebuffer::{self, IndexedColor};
 use edit::helpers::{CoordType, KIBI, MEBI, MetricFormatter, Rect, Size};
 use edit::input::{self, kbmod, vk};
 use edit::oklab::StraightRgba;
 use edit::tui::*;
 use edit::vt::{self, Token};
-use edit::{apperr, arena_format, base64, path, sys, unicode};
+use edit::{apperr, base64, path, sys, unicode};
 use localization::*;
 use state::*;
+use stdext::arena::{self, Arena, ArenaString, scratch_arena};
+use stdext::arena_format;
 
 #[cfg(target_pointer_width = "32")]
 const SCRATCH_ARENA_CAPACITY: usize = 128 * MEBI;

src/buffer/gap_buffer.rs → crates/edit/src/buffer/gap_buffer.rs

@@ -5,9 +5,11 @@ use std::ops::Range;
 use std::ptr::{self, NonNull};
 use std::slice;
 
+use stdext::sys::{virtual_commit, virtual_release, virtual_reserve};
+
+use crate::apperr;
 use crate::document::{ReadableDocument, WriteableDocument};
 use crate::helpers::*;
-use crate::{apperr, sys};
 
 #[cfg(target_pointer_width = "32")]
 const LARGE_CAPACITY: usize = 128 * MEBI;
@@ -31,7 +33,7 @@ impl Drop for BackingBuffer {
     fn drop(&mut self) {
         unsafe {
             if let Self::VirtualMemory(ptr, reserve) = *self {
-                sys::virtual_release(ptr, reserve);
+                virtual_release(ptr, reserve);
             }
         }
     }
@@ -73,7 +75,7 @@ impl GapBuffer {
             buffer = BackingBuffer::Vec(Vec::new());
         } else {
             reserve = LARGE_CAPACITY;
-            text = unsafe { sys::virtual_reserve(reserve)? };
+            text = unsafe { virtual_reserve(reserve)? };
             buffer = BackingBuffer::VirtualMemory(text, reserve);
         }
 
@@ -195,7 +197,7 @@ impl GapBuffer {
 
             match &mut self.buffer {
                 BackingBuffer::VirtualMemory(ptr, _) => unsafe {
-                    if sys::virtual_commit(ptr.add(bytes_old), bytes_new - bytes_old).is_err() {
+                    if virtual_commit(ptr.add(bytes_old), bytes_new - bytes_old).is_err() {
                         return;
                     }
                 },

src/buffer/mod.rs → crates/edit/src/buffer/mod.rs

@@ -35,8 +35,8 @@ use std::rc::Rc;
 use std::str;
 
 pub use gap_buffer::GapBuffer;
+use stdext::arena::{Arena, ArenaString, scratch_arena};
 
-use crate::arena::{Arena, ArenaString, scratch_arena};
 use crate::cell::SemiRefCell;
 use crate::clipboard::Clipboard;
 use crate::document::{ReadableDocument, WriteableDocument};

src/document.rs → crates/edit/src/document.rs

@@ -8,7 +8,8 @@ use std::mem;
 use std::ops::Range;
 use std::path::PathBuf;
 
-use crate::arena::{ArenaString, scratch_arena};
+use stdext::arena::{ArenaString, scratch_arena};
+
 use crate::helpers::ReplaceRange as _;
 
 /// An abstraction over reading from text containers.