refactor string

Author: CppCXY

README.md

@@ -9,6 +9,8 @@
 
 A Lua 5.5 interpreter written in pure Rust. Faithfully ported from the official C Lua source code architecture — register-based VM, incremental/generational GC, string interning — and **passes the official Lua 5.5 test suite** (`all.lua` — 28/30 test files, 435 unit tests).
 
+Lua strings in luars are stored as byte strings with an optional UTF-8 text view. At the Lua level this matches Lua's byte-oriented string semantics; on the Rust side you can choose `as_str()` for text or `as_bytes()` / `create_bytes()` when exact bytes matter.
+
 ## Highlights
 
 - **Lua 5.5** — compiler, VM, and standard libraries implement the Lua 5.5 specification
@@ -79,6 +81,17 @@ vm.register_type_of::<Point>("Point")?;
 vm.register_enum::<Color>("Color")?;
 ```
 
+### Strings And Bytes
+
+```rust
+let text = vm.create_string("hello")?;
+assert_eq!(text.as_str(), Some("hello"));
+
+let raw = vm.create_bytes(&[0xff, 0x00, b'A'])?;
+assert_eq!(raw.as_str(), None);
+assert_eq!(raw.as_bytes(), Some(&[0xff, 0x00, b'A'][..]));
+```
+
 ### Tables
 
 ```rust
@@ -236,7 +249,7 @@ For the full list of behavioral differences, see [docs/Different.md](docs/Differ
 - **No C API / C module loading** — pure Rust, no `lua_State*` interface
 - **No debug hooks** — `debug.sethook` is a stub; `getinfo` / `getlocal` / `traceback` work
 - **Own bytecode format** — `string.dump` output is not compatible with C Lua
-- **UTF-8 strings** — no arbitrary binary bytes (use the separate `binary` type)
+- **Rust text view is explicit** — Lua strings are byte strings, but Rust-side `as_str()` only succeeds for valid UTF-8; use `as_bytes()` for exact byte access
 
 ## Building
 

crates/luars/src/compiler/expr_parser.rs

@@ -134,10 +134,7 @@ fn simpleexp(fs: &mut FuncState, v: &mut ExpDesc) -> Result<(), String> {
             let string_content = parse_string_token_value(text, fs.lexer.current_token());
             match string_content {
                 Ok(bytes) => {
-                    let string = match String::from_utf8(bytes.clone()) {
-                        Ok(s) => fs.vm.create_string(&s).unwrap(),
-                        Err(_) => fs.vm.create_binary(bytes).unwrap(),
-                    };
+                    let string = fs.vm.create_bytes(&bytes).unwrap();
                     // Create VKSTR expression (not VK!) - will convert to VK when needed
                     *v = ExpDesc::new_vkstr(string);
                 }

crates/luars/src/gc/gc_kind.rs

@@ -9,5 +9,4 @@ pub enum GcObjectKind {
     Upvalue = 5,
     Thread = 6,
     Userdata = 7,
-    Binary = 8,
 }

crates/luars/src/gc/gc_object.rs

@@ -305,7 +305,6 @@ impl<T> HasGcHeader for Gc<T> {
 }
 
 pub type GcString = Gc<LuaString>;
-pub type GcBinary = Gc<Vec<u8>>;
 pub type GcTable = Gc<LuaTable>;
 pub type GcFunction = Gc<LuaFunction>;
 pub type GcCClosure = Gc<CClosureFunction>;
@@ -401,7 +400,6 @@ impl<T: HasGcHeader> GcPtr<T> {
 }
 
 pub type UpvaluePtr = GcPtr<GcUpvalue>;
-pub type BinaryPtr = GcPtr<GcBinary>;
 pub type TablePtr = GcPtr<GcTable>;
 pub type StringPtr = GcPtr<GcString>;
 pub type FunctionPtr = GcPtr<GcFunction>;
@@ -454,8 +452,6 @@ impl GcObjectPtr {
     const TAG_UPVALUE: u64 = 5;
     const TAG_THREAD: u64 = 6;
     const TAG_USERDATA: u64 = 7;
-    const TAG_BINARY: u64 = 8;
-
     #[inline(always)]
     fn new_tagged(ptr: u64, tag: u64) -> Self {
         debug_assert!(
@@ -569,12 +565,6 @@ impl GcObjectPtr {
         UserdataPtr::from_raw(self.raw_ptr())
     }
 
-    #[inline(always)]
-    pub fn as_binary_ptr(&self) -> BinaryPtr {
-        debug_assert!(self.tag() == Self::TAG_BINARY as u8);
-        BinaryPtr::from_raw(self.raw_ptr())
-    }
-
     // ============ Pattern matching helpers (for code that still uses if-let) ============
 
     #[inline(always)]
@@ -616,11 +606,6 @@ impl GcObjectPtr {
     pub fn is_userdata(&self) -> bool {
         self.tag() == Self::TAG_USERDATA as u8
     }
-
-    #[inline(always)]
-    pub fn is_binary(&self) -> bool {
-        self.tag() == Self::TAG_BINARY as u8
-    }
 }
 
 impl From<StringPtr> for GcObjectPtr {
@@ -630,13 +615,6 @@ impl From<StringPtr> for GcObjectPtr {
     }
 }
 
-impl From<BinaryPtr> for GcObjectPtr {
-    #[inline(always)]
-    fn from(ptr: BinaryPtr) -> Self {
-        Self::new_tagged(ptr.as_u64(), Self::TAG_BINARY)
-    }
-}
-
 impl From<TablePtr> for GcObjectPtr {
     #[inline(always)]
     fn from(ptr: TablePtr) -> Self {
@@ -696,7 +674,6 @@ pub enum GcObjectOwner {
     Userdata(Box<GcUserdata>),
     CClosure(Box<GcCClosure>),
     RClosure(Box<GcRClosure>),
-    Binary(Box<GcBinary>),
 }
 
 impl GcObjectOwner {
@@ -706,7 +683,6 @@ impl GcObjectOwner {
     pub fn compute_size(&self) -> usize {
         match self {
             GcObjectOwner::String(s) => std::mem::size_of::<GcString>() + s.data.str.len(),
-            GcObjectOwner::Binary(b) => std::mem::size_of::<GcBinary>() + b.data.len(),
             GcObjectOwner::Table(t) => {
                 let base = std::mem::size_of::<GcTable>();
                 let asize = t.data.impl_table.asize as usize;
@@ -750,7 +726,6 @@ impl GcObjectOwner {
             GcObjectOwner::Upvalue(u) => &u.header,
             GcObjectOwner::Thread(t) => &t.header,
             GcObjectOwner::Userdata(u) => &u.header,
-            GcObjectOwner::Binary(b) => &b.header,
         }) as _
     }
 
@@ -764,7 +739,6 @@ impl GcObjectOwner {
             GcObjectOwner::Upvalue(u) => &mut u.header,
             GcObjectOwner::Thread(t) => &mut t.header,
             GcObjectOwner::Userdata(u) => &mut u.header,
-            GcObjectOwner::Binary(b) => &mut b.header,
         }) as _
     }
 
@@ -812,13 +786,6 @@ impl GcObjectOwner {
         }
     }
 
-    pub fn as_binary_ptr(&self) -> Option<BinaryPtr> {
-        match self {
-            GcObjectOwner::Binary(b) => Some(BinaryPtr::new(b.as_ref() as *const GcBinary)),
-            _ => None,
-        }
-    }
-
     pub fn as_closure_ptr(&self) -> Option<CClosurePtr> {
         match self {
             GcObjectOwner::CClosure(c) => Some(CClosurePtr::new(c.as_ref() as *const GcCClosure)),
@@ -853,9 +820,6 @@ impl GcObjectOwner {
             GcObjectOwner::Userdata(u) => {
                 GcObjectPtr::from(UserdataPtr::new(u.as_ref() as *const GcUserdata))
             }
-            GcObjectOwner::Binary(b) => {
-                GcObjectPtr::from(BinaryPtr::new(b.as_ref() as *const GcBinary))
-            }
             GcObjectOwner::CClosure(c) => {
                 GcObjectPtr::from(CClosurePtr::new(c.as_ref() as *const GcCClosure))
             }

crates/luars/src/gc/mod.rs

@@ -772,7 +772,7 @@ impl GC {
     /// For other objects: returns true if white (will be collected)
     fn is_cleared(&mut self, l: &mut LuaState, gc_ptr: GcObjectPtr) -> bool {
         match gc_ptr.kind() {
-            GcObjectKind::String | GcObjectKind::Binary => {
+            GcObjectKind::String => {
                 self.mark_object(l, gc_ptr);
                 false
             }
@@ -3383,8 +3383,8 @@ impl GC {
                             );
                         }
                     }
-                    // String/Binary have no GC references
-                    GcObjectOwner::String(_) | GcObjectOwner::Binary(_) => {}
+                    // Strings have no GC references
+                    GcObjectOwner::String(_) => {}
                 }
             }
         }
@@ -3543,7 +3543,6 @@ impl GC {
             GcObjectOwner::CClosure(b) => b.as_ref() as *const _ as u64,
             GcObjectOwner::RClosure(b) => b.as_ref() as *const _ as u64,
             GcObjectOwner::String(b) => b.as_ref() as *const _ as u64,
-            GcObjectOwner::Binary(b) => b.as_ref() as *const _ as u64,
             GcObjectOwner::Thread(b) => b.as_ref() as *const _ as u64,
             GcObjectOwner::Upvalue(b) => b.as_ref() as *const _ as u64,
             GcObjectOwner::Userdata(b) => b.as_ref() as *const _ as u64,
@@ -3969,7 +3968,7 @@ impl GC {
     /// ```.
     fn really_mark_object(&mut self, l: &mut LuaState, gc_ptr: GcObjectPtr) {
         self.gc_marked += 64; // fixed estimate; exact size unavailable from GcObjectPtr
-        if gc_ptr.is_string() || gc_ptr.is_binary() {
+        if gc_ptr.is_string() {
             gc_ptr.header_mut().unwrap().make_black(); // Leaves become black immediately
         } else if gc_ptr.is_upvalue() {
             let uv_ptr = gc_ptr.as_upvalue_ptr();

crates/luars/src/gc/object_allocator.rs

@@ -15,7 +15,7 @@ use crate::lua_value::{
 };
 use crate::lua_vm::{CFunction, LuaState};
 use crate::{
-    GC, GcBinary, GcCClosure, GcFunction, GcObjectOwner, GcRClosure, GcTable, GcThread, GcUpvalue,
+    GC, GcCClosure, GcFunction, GcObjectOwner, GcRClosure, GcTable, GcThread, GcUpvalue,
     GcUserdata, LuaFunction, LuaResult, LuaTable, LuaValue, StringPtr, UpvaluePtr,
 };
 use std::rc::Rc;
@@ -59,17 +59,18 @@ impl ObjectAllocator {
         self.strings.intern_owned(s, gc)
     }
 
-    /// Create a binary value from Vec<u8>
-    ///
+    /// Create a Lua string-like value from raw bytes.
+    /// All short byte strings are interned so Lua string equality keeps its fast path.
+    #[inline]
+    pub fn create_bytes(&mut self, gc: &mut GC, bytes: &[u8]) -> CreateResult {
+        self.strings.intern_bytes(bytes, gc)
+    }
+
+    /// Create a raw byte string from Vec<u8> without requiring UTF-8.
+    /// This compatibility path now uses the same byte-string interning rules as `create_bytes`.
     #[inline]
     pub fn create_binary(&mut self, gc: &mut GC, data: Vec<u8>) -> CreateResult {
-        let current_white = gc.current_white;
-        let size = (std::mem::size_of::<GcBinary>() + data.len()) as u32;
-        let gc_ptr = Box::new(GcBinary::new(data, current_white, size));
-        let gc_binary = GcObjectOwner::Binary(gc_ptr);
-        let ptr = gc_binary.as_binary_ptr().unwrap();
-        gc.trace_object(gc_binary)?;
-        Ok(LuaValue::binary(ptr))
+        self.strings.intern_bytes_owned(data, gc)
     }
 
     /// Create a substring from an existing string (optimized for string.sub)
@@ -84,14 +85,8 @@ impl ObjectAllocator {
         start: usize,
         end: usize,
     ) -> CreateResult {
-        // Get bytes - handle both string and binary types
-        let source_str = s_value.as_str();
-        let source_is_ascii = source_str.is_some_and(str::is_ascii);
-        let bytes = if let Some(s) = source_str {
-            s.as_bytes()
-        } else if let Some(b) = s_value.as_binary() {
-            b
-        } else {
+        let source_is_ascii = s_value.as_str().is_some_and(str::is_ascii);
+        let Some(bytes) = s_value.as_bytes() else {
             return self.create_string(gc, "");
         };
 
@@ -121,17 +116,7 @@ impl ObjectAllocator {
                 std::str::from_utf8_unchecked(substring_bytes)
             })
         } else {
-            match std::str::from_utf8(substring_bytes) {
-                Ok(valid_str) => {
-                    // Valid UTF-8 - intern as string
-                    self.create_string(gc, valid_str)
-                }
-                Err(_) => {
-                    // Invalid UTF-8 - create binary value to preserve original bytes
-                    // This is important for binary data like bytecode
-                    self.create_binary(gc, substring_bytes.to_vec())
-                }
-            }
+            self.create_bytes(gc, substring_bytes)
         }
     }