Use bytes crate instead of Vec<u8> for binary data

To avoid pack/unpack overhead, use magnus's TryConvert trait
implementation for bytes::Bytes.

Author: kateinoigakukun

Cargo.lock

@@ -10,8 +10,9 @@ publish = false
 crate-type = ["cdylib"]
 
 [dependencies]
-magnus = "0.6.2"
+magnus = { version = "0.6.2", features = ["bytes"] }
+bytes = "1"
 wizer = "4.0.0"
 wasi-vfs-cli = { git = "https://github.com/kateinoigakukun/wasi-vfs/", tag = "0.5.2" }
 structopt = "0.3.26"
-wit-component = { git = "https://github.com/bytecodealliance/wasm-tools", rev = "refs/pull/1490/head" }
+wit-component = "0.203.0"

ext/ruby_wasm/Cargo.toml

@@ -12,7 +12,7 @@ use wizer::Wizer;
 static RUBY_WASM: value::Lazy<RModule> =
     value::Lazy::new(|ruby| ruby.define_module("RubyWasmExt").unwrap());
 
-fn preinit(core_module: Vec<u8>) -> Result<Vec<u8>, Error> {
+fn preinit(core_module: bytes::Bytes) -> Result<bytes::Bytes, Error> {
     let rbwasm_error = eval("RubyWasmExt::Error")?;
     let rbwasm_error = ExceptionClass::from_value(rbwasm_error).unwrap();
     let mut wizer = Wizer::new();
@@ -26,6 +26,7 @@ fn preinit(core_module: Vec<u8>) -> Result<Vec<u8>, Error> {
     wizer
         .run(&core_module)
         .map_err(|e| Error::new(rbwasm_error, format!("failed to run wizer: {}", e)))
+        .map(|output| output.into())
 }
 
 struct WasiVfsInner {
@@ -53,14 +54,14 @@ impl WasiVfs {
         self.0.borrow_mut().map_dirs.push((guest_dir.into(), host_dir.into()));
     }
 
-    fn pack(&self, wasm_bytes: Vec<u8>) -> Result<Vec<u8>, Error> {
+    fn pack(&self, wasm_bytes: bytes::Bytes) -> Result<bytes::Bytes, Error> {
         let output_bytes = wasi_vfs_cli::pack(&wasm_bytes, self.0.borrow().map_dirs.clone()).map_err(|e| {
             Error::new(
                 exception::standard_error(),
                 format!("failed to pack wasi vfs: {}", e),
             )
         })?;
-        Ok(output_bytes)
+        Ok(output_bytes.into())
     }
 }
 
@@ -83,7 +84,7 @@ impl ComponentLink {
         Ok(())
     }
 
-    fn library(&self, name: String, module: Vec<u8>, dl_openable: bool) -> Result<(), Error> {
+    fn library(&self, name: String, module: bytes::Bytes, dl_openable: bool) -> Result<(), Error> {
         self.linker(|linker| {
             linker.library(&name, &module, dl_openable).map_err(|e| {
                 Error::new(
@@ -93,7 +94,7 @@ impl ComponentLink {
             })
         })
     }
-    fn adapter(&self, name: String, module: Vec<u8>) -> Result<(), Error> {
+    fn adapter(&self, name: String, module: bytes::Bytes) -> Result<(), Error> {
         self.linker(|linker| {
             linker.adapter(&name, &module).map_err(|e| {
                 Error::new(
@@ -123,7 +124,7 @@ impl ComponentLink {
             Ok(linker.use_built_in_libdl(use_libdl))
         })
     }
-    fn encode(&self) -> Result<Vec<u8>, Error> {
+    fn encode(&self) -> Result<bytes::Bytes, Error> {
         // Take the linker out of the cell and consume it
         let linker = self.0.borrow_mut().take().ok_or_else(|| {
             Error::new(
@@ -137,7 +138,7 @@ impl ComponentLink {
                 format!("failed to encode linker: {}", e),
             )
         })?;
-        Ok(encoded)
+        Ok(encoded.into())
     }
 }
 

ext/ruby_wasm/src/lib.rs

@@ -323,9 +323,9 @@ def do_build(executor, tmpdir, packager, options)
       RubyWasm.logger.info "Size: #{SizeFormatter.format(wasm_bytes.size)}"
       case options[:output]
       when "-"
-        @stdout.write wasm_bytes.pack("C*")
+        @stdout.write wasm_bytes
       else
-        File.binwrite(options[:output], wasm_bytes.pack("C*"))
+        File.binwrite(options[:output], wasm_bytes)
         RubyWasm.logger.debug "Wrote #{options[:output]}"
       end
     end

lib/ruby_wasm/cli.rb

@@ -38,6 +38,7 @@ def build(executor, options)
     end
 
     def build_and_link_exts(executor)
+      raise NotImplementedError
     end
 
     # Array of paths to extconf.rb files.
@@ -129,17 +130,15 @@ def link_exts(executor, build)
       libraries.each do |lib|
         # Non-DL openable libraries should be referenced as base name
         lib_name = File.basename(lib)
-        # @type var module_bytes: Array[Integer]
-        module_bytes = File.binread(lib).unpack("C*")
+        module_bytes = File.binread(lib)
         RubyWasm.logger.info "Linking #{lib_name} (#{module_bytes.size} bytes)"
         linker.library(lib_name, module_bytes, false)
       end
 
       dl_openable_libs.each do |lib|
         # DL openable lib_name should be a relative path from ruby_root
         lib_name = "/" + Pathname.new(lib).relative_path_from(Pathname.new(ruby_root)).to_s
-        # @type var module_bytes: Array[Integer]
-        module_bytes = File.binread(lib).unpack("C*")
+        module_bytes = File.binread(lib)
         RubyWasm.logger.info "Linking #{lib_name} (#{module_bytes.size} bytes)"
         linker.library(lib_name, module_bytes, true)
       end
@@ -148,8 +147,7 @@ def link_exts(executor, build)
         adapter_name = File.basename(adapter)
         # e.g. wasi_snapshot_preview1.command.wasm -> wasi_snapshot_preview1
         adapter_name = adapter_name.split(".")[0]
-        # @type var module_bytes: Array[Integer]
-        module_bytes = File.binread(adapter).unpack("C*")
+        module_bytes = File.binread(adapter)
         linker.adapter(adapter_name, module_bytes)
       end
       return linker.encode()

lib/ruby_wasm/packager/core.rb

@@ -1,5 +1,7 @@
 module RubyWasmExt
-  def self.preinitialize: (Array[Integer] module_bytes) -> Array[Integer]
+  type bytes = String
+
+  def self.preinitialize: (bytes module_bytes) -> bytes
 
   class WasiVfs
     def initialize: () -> void
@@ -8,17 +10,17 @@ module RubyWasmExt
 
     def map_dir: (String guest_path, String host_path) -> void
 
-    def pack: (Array[Integer] module_bytes) -> Array[Integer]
+    def pack: (bytes module_bytes) -> bytes
   end
 
   class ComponentLink
     def initialize: () -> void
-    def library: (String name, Array[Integer] | String module, bool dl_openable) -> void
-    def adapter: (String name, Array[Integer] | String module) -> void
+    def library: (String name, bytes module, bool dl_openable) -> void
+    def adapter: (String name, bytes module) -> void
     def validate: (bool) -> void
     def stack_size: (Integer) -> void
     def stub_missing_functions: (bool) -> void
     def use_built_in_libdl: (bool) -> void
-    def encode: () -> Array[Integer]
+    def encode: () -> bytes
   end
 end

sig/ruby_wasm/ext.rbs

@@ -20,12 +20,14 @@ class RubyWasm::Packager
 
   type build_config = Hash[untyped, untyped]
 
+  type bytes = String
+
   @definition: untyped
   @config: build_config
 
   def initialize: (string root, build_config?, untyped? definition) -> void
 
-  def package: (RubyWasm::BuildExecutor, string dest_dir, untyped options) -> Array[Integer]
+  def package: (RubyWasm::BuildExecutor, string dest_dir, untyped options) -> bytes
 
   @ruby_core_build: RubyWasm::Packager::Core?
   def ruby_core_build: () -> RubyWasm::Packager::Core
@@ -48,7 +50,7 @@ class RubyWasm::Packager
     @packager: RubyWasm::Packager
     def initialize: (RubyWasm::Packager) -> void
     def build: (RubyWasm::BuildExecutor, untyped options) -> String
-    def build_and_link_exts: (RubyWasm::BuildExecutor) -> Array[Integer]
+    def build_and_link_exts: (RubyWasm::BuildExecutor) -> bytes
 
     extend Forwardable
 
@@ -64,7 +66,7 @@ class RubyWasm::Packager
       def initialize: (RubyWasm::Packager) -> void
       def build: (RubyWasm::BuildExecutor, untyped options) -> String
       def specs_with_extensions: () -> Array[[untyped, Array[string]]]
-      def build_and_link_exts: (RubyWasm::BuildExecutor) -> void
+      def build_and_link_exts: (RubyWasm::BuildExecutor) -> bytes
     end
 
     class DynamicLinking < RubyWasm::Packager::Core::BuildStrategy
@@ -73,7 +75,7 @@ class RubyWasm::Packager
       def build_exts: (RubyWasm::BuildExecutor, RubyWasm::Build) -> void
       def name: () -> string
 
-      private def link_exts: (RubyWasm::BuildExecutor, RubyWasm::Build) -> Array[Integer]
+      private def link_exts: (RubyWasm::BuildExecutor, RubyWasm::Build) -> bytes
     end
 
     class StaticLinking < RubyWasm::Packager::Core::BuildStrategy