Skip to content

src: move all 1-byte encodings to native #61118

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ChALkeR wants to merge 1 commit into
base: main
Choose a base branch
from
+519 −160
Open

src: move all 1-byte encodings to native #61118

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 47 additions & 4 deletions lib/internal/encoding.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,13 @@
// https://encoding.spec.whatwg.org

const {
ArrayPrototypeMap,
Boolean,
ObjectDefineProperties,
ObjectGetOwnPropertyDescriptors,
ObjectSetPrototypeOf,
ObjectValues,
SafeArrayIterator,
SafeMap,
StringPrototypeSlice,
Symbol,
Expand All @@ -32,8 +34,6 @@ const kFatal = Symbol('kFatal');
const kUTF8FastPath = Symbol('kUTF8FastPath');
const kIgnoreBOM = Symbol('kIgnoreBOM');

const { isSinglebyteEncoding, createSinglebyteDecoder } = require('internal/encoding/single-byte');

const {
getConstructorOf,
customInspectSymbol: inspect,
Expand All @@ -58,6 +58,7 @@ const {
encodeIntoResults,
encodeUtf8String,
decodeUTF8,
decodeSingleByte,
} = binding;

function validateDecoder(obj) {
Expand All @@ -71,6 +72,47 @@ const CONVERTER_FLAGS_IGNORE_BOM = 0x4;

const empty = new FastBuffer();

// Has to be synced with src/
const encodingsSinglebyte = new SafeMap(new SafeArrayIterator(ArrayPrototypeMap([
'ibm866',
'koi8-r',
'koi8-u',
'macintosh',
'x-mac-cyrillic',
'iso-8859-2',
'iso-8859-3',
'iso-8859-4',
'iso-8859-5',
'iso-8859-6',
'iso-8859-7',
'iso-8859-8',
'iso-8859-8-i',
'iso-8859-10',
'iso-8859-13',
'iso-8859-14',
'iso-8859-15',
'iso-8859-16',
'windows-874',
'windows-1250',
'windows-1251',
'windows-1252',
'windows-1253',
'windows-1254',
'windows-1255',
'windows-1256',
'windows-1257',
'windows-1258',
'x-user-defined', // Has to be last, special case
], (e, i) => [e, i])));

const isSinglebyteEncoding = (enc) => encodingsSinglebyte.has(enc);

function createSinglebyteDecoder(encoding, fatal) {
const key = encodingsSinglebyte.get(encoding);
if (key === undefined) throw new ERR_ENCODING_NOT_SUPPORTED(encoding);
return (buf) => decodeSingleByte(buf, key, fatal);
}

const encodings = new SafeMap([
['unicode-1-1-utf-8', 'utf-8'],
['unicode11utf8', 'utf-8'],
Expand Down Expand Up @@ -462,7 +504,7 @@ function makeTextDecoderICU() {
validateDecoder(this);
validateObject(options, 'options', kValidateObjectAllowObjectsAndNull);

if (this[kMethod]) return this[kMethod](parseInput(input));
if (this[kMethod]) return this[kMethod](input);

this[kUTF8FastPath] &&= !(options?.stream);

Expand Down Expand Up @@ -532,11 +574,12 @@ function makeTextDecoderJS() {

decode(input = empty, options = kEmptyObject) {
validateDecoder(this);
input = parseInput(input);
validateObject(options, 'options', kValidateObjectAllowObjectsAndNull);

if (this[kMethod]) return this[kMethod](input);

input = parseInput(input);

if (this[kFlags] & CONVERTER_FLAGS_FLUSH) {
this[kBOMSeen] = false;
}
Expand Down
155 changes: 0 additions & 155 deletions lib/internal/encoding/single-byte.js
View file
Open in desktop

This file was deleted.

67 changes: 67 additions & 0 deletions src/encoding_binding.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
#include "encoding_binding.h"
#include "ada.h"
#include "encoding_singlebyte.h"
#include "env-inl.h"
#include "node_errors.h"
#include "node_external_reference.h"
Expand Down Expand Up @@ -389,6 +390,70 @@ void BindingData::DecodeUTF8(const FunctionCallbackInfo<Value>& args) {
}
}

void BindingData::DecodeSingleByte(const FunctionCallbackInfo<Value>& args) {
Environment* env = Environment::GetCurrent(args);

CHECK_GE(args.Length(), 2);
Isolate* isolate = env->isolate();

if (!(args[0]->IsArrayBuffer() || args[0]->IsSharedArrayBuffer() ||
args[0]->IsArrayBufferView())) {
return node::THROW_ERR_INVALID_ARG_TYPE(
isolate,
"The \"input\" argument must be an instance of SharedArrayBuffer, "
"ArrayBuffer or ArrayBufferView.");
}

static constexpr int kXUserDefined = 28; // Last one, see encoding.js

CHECK(args[1]->IsInt32());
const int encoding = args[1].As<v8::Int32>()->Value();
CHECK(encoding >= 0 && encoding <= kXUserDefined);

ArrayBufferViewContents<uint8_t> buffer(args[0]);
const uint8_t* data = buffer.data();
size_t length = buffer.length();

if (length == 0) return args.GetReturnValue().SetEmptyString();

const char* dataChar = reinterpret_cast<const char*>(data);
if (!simdutf::validate_ascii_with_errors(dataChar, length).error) {
Local<Value> ret;
if (StringBytes::Encode(isolate, dataChar, length, LATIN1).ToLocal(&ret)) {
args.GetReturnValue().Set(ret);
}
return;
}

uint16_t* dst = node::UncheckedMalloc<uint16_t>(length);
Copy link
Member

@gurgunday gurgunday Jan 17, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Here we can allocate a huge uint16_t[length] before any V8 string-length limit check, and if it ends up being higher than what V8 accepts, we would allocate for no reason

Maybe it would be better to guard this with sth like:

Suggested change
uint16_t* dst = node::UncheckedMalloc<uint16_t>(length);
if (length > static_cast<size_t>(v8::String::kMaxLength)) {
// throw
}
uint16_t* dst = node::UncheckedMalloc<uint16_t>(length);

Let me know if I'm missing something

if (dst == nullptr) return node::THROW_ERR_MEMORY_ALLOCATION_FAILED(isolate);

if (encoding == kXUserDefined) {
// x-user-defined
for (size_t i = 0; i < length; i++) {
dst[i] = data[i] >= 0x80 ? data[i] + 0xf700 : data[i];
}
} else {
bool has_fatal = args[2]->IsTrue();

const uint16_t* table = tSingleByteEncodings[encoding];
for (size_t i = 0; i < length; i++) dst[i] = table[data[i]];

const char16_t* dst16 = reinterpret_cast<char16_t*>(dst);
if (has_fatal && fSingleByteEncodings[encoding] &&
simdutf::find(dst16, dst16 + length, 0xfffd) != dst16 + length) {
free(dst);
return node::THROW_ERR_ENCODING_INVALID_ENCODED_DATA(
isolate, "The encoded data was not valid for this encoding");
}
}

Local<Value> ret;
if (StringBytes::Raw(isolate, dst, length).ToLocal(&ret)) {
args.GetReturnValue().Set(ret);
}
}

void BindingData::ToASCII(const FunctionCallbackInfo<Value>& args) {
Environment* env = Environment::GetCurrent(args);
CHECK_GE(args.Length(), 1);
Expand Down Expand Up @@ -421,6 +486,7 @@ void BindingData::CreatePerIsolateProperties(IsolateData* isolate_data,
SetMethod(isolate, target, "encodeInto", EncodeInto);
SetMethodNoSideEffect(isolate, target, "encodeUtf8String", EncodeUtf8String);
SetMethodNoSideEffect(isolate, target, "decodeUTF8", DecodeUTF8);
SetMethodNoSideEffect(isolate, target, "decodeSingleByte", DecodeSingleByte);
SetMethodNoSideEffect(isolate, target, "toASCII", ToASCII);
SetMethodNoSideEffect(isolate, target, "toUnicode", ToUnicode);
}
Expand All @@ -438,6 +504,7 @@ void BindingData::RegisterTimerExternalReferences(
registry->Register(EncodeInto);
registry->Register(EncodeUtf8String);
registry->Register(DecodeUTF8);
registry->Register(DecodeSingleByte);
registry->Register(ToASCII);
registry->Register(ToUnicode);
}
Expand Down
1 change: 1 addition & 0 deletions src/encoding_binding.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ class BindingData : public SnapshotableObject {
static void EncodeInto(const v8::FunctionCallbackInfo<v8::Value>& args);
static void EncodeUtf8String(const v8::FunctionCallbackInfo<v8::Value>& args);
static void DecodeUTF8(const v8::FunctionCallbackInfo<v8::Value>& args);
static void DecodeSingleByte(const v8::FunctionCallbackInfo<v8::Value>& args);

static void ToASCII(const v8::FunctionCallbackInfo<v8::Value>& args);
static void ToUnicode(const v8::FunctionCallbackInfo<v8::Value>& args);
Expand Down
Loading
Loading