From 062def70d0203c49397db0a9b30fde87c19fcb57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 11 Jul 2026 07:09:12 -0700 Subject: [PATCH 1/6] =?UTF-8?q?=E2=9A=A1=20perf(core):=20shrink=20tokenize?= =?UTF-8?q?r=20and=20DOM=20state?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Large flat documents paid for attribute data consumers did not use. DOM construction also repeated allocations for tags and per-node caches. Add an opt-in tag-and-text tokenizer path and compact shared storage. Preserve the default token contract while reducing parser allocations. --- docs/changelog/simple-parser.feature.rst | 1 + docs/development/bench/building-2.json | 6 +- docs/development/bench/building-4.json | 6 +- docs/development/bench/building.json | 6 +- docs/migration/bench/airium.json | 6 +- docs/migration/bench/beautifulsoup.json | 12 +- docs/migration/bench/dominate.json | 12 +- docs/migration/bench/fast-html.json | 6 +- docs/migration/bench/htbuilder.json | 12 +- docs/migration/bench/htpy.json | 12 +- docs/migration/bench/hyperpython.json | 12 +- docs/migration/bench/lxml.json | 18 +- docs/migration/bench/markyp.json | 12 +- docs/migration/bench/simple-html.json | 12 +- docs/migration/bench/yattag.json | 6 +- docs/tutorials/tokenizing.rst | 13 + src/turbohtml/_c/core/module.c | 5 +- src/turbohtml/_c/data/tag_atom.h | 327 +++++++++--------- src/turbohtml/_c/dom/element.c | 26 +- src/turbohtml/_c/dom/mutate.c | 23 +- src/turbohtml/_c/dom/nodes.h | 4 +- src/turbohtml/_c/dom/tree.c | 123 ++++--- src/turbohtml/_c/dom/tree.h | 6 +- src/turbohtml/_c/dom/tree_internal.h | 29 +- src/turbohtml/_c/query/methods.c | 18 + src/turbohtml/_c/serialize/document.c | 3 +- src/turbohtml/_c/serialize/lossless.c | 13 +- src/turbohtml/_c/tokenizer/rewrite.c | 2 +- src/turbohtml/_c/tokenizer/statemachine.c | 45 ++- src/turbohtml/_c/tokenizer/statemachine.h | 20 +- src/turbohtml/_c/tokenizer/statemachine_run.h | 84 +++-- src/turbohtml/_c/tokenizer/token.c | 1 + src/turbohtml/_c/tokenizer/tokenizer.c | 25 +- src/turbohtml/_c/tokenizer/xml.c | 2 +- src/turbohtml/_stubs/tokenizer.pyi | 13 +- tests/build/test_build.py | 4 + tests/tokenizer/test_tokenizer.py | 25 ++ tools/bench/operations.py | 1 + tools/generate_tags.py | 11 +- 39 files changed, 561 insertions(+), 401 deletions(-) create mode 100644 docs/changelog/simple-parser.feature.rst diff --git a/docs/changelog/simple-parser.feature.rst b/docs/changelog/simple-parser.feature.rst new file mode 100644 index 00000000..499f466a --- /dev/null +++ b/docs/changelog/simple-parser.feature.rst @@ -0,0 +1 @@ +Add ``capture_attributes=False`` for tag-and-text token streams, and reduce DOM allocation for large documents. diff --git a/docs/development/bench/building-2.json b/docs/development/bench/building-2.json index a5389a43..ff4346ae 100644 --- a/docs/development/bench/building-2.json +++ b/docs/development/bench/building-2.json @@ -9,19 +9,19 @@ "rows": [ [ "100 rows", - 4.5e-05, + 4.236423571531608e-05, 9e-05, 0.000262 ], [ "1000 rows", - 0.000482, + 0.0004373766555545444, 0.000952, 0.00255 ], [ "10000 rows", - 0.00442, + 0.0043080967928593355, 0.0094, 0.0253 ] diff --git a/docs/development/bench/building-4.json b/docs/development/bench/building-4.json index 24ef7fa4..9267b228 100644 --- a/docs/development/bench/building-4.json +++ b/docs/development/bench/building-4.json @@ -9,19 +9,19 @@ "rows": [ [ "100 rows", - 0.000117, + 0.00011428218567743897, 0.000438, 0.000147 ], [ "1000 rows", - 0.0012, + 0.0011681491435714027, 0.00442, 0.00136 ], [ "10000 rows", - 0.0143, + 0.014314786463122195, 0.047, 0.0137 ] diff --git a/docs/development/bench/building.json b/docs/development/bench/building.json index 9420968a..ed36be06 100644 --- a/docs/development/bench/building.json +++ b/docs/development/bench/building.json @@ -9,19 +9,19 @@ "rows": [ [ "100 rows", - 5.68e-05, + 5.5585856197618945e-05, 0.00013, 0.000756 ], [ "1000 rows", - 0.00054, + 0.0005315817325026728, 0.00132, 0.0074 ], [ "10000 rows", - 0.00534, + 0.0052333376722041676, 0.0132, 0.081 ] diff --git a/docs/migration/bench/airium.json b/docs/migration/bench/airium.json index 0c1fbafb..8218eb06 100644 --- a/docs/migration/bench/airium.json +++ b/docs/migration/bench/airium.json @@ -8,17 +8,17 @@ "rows": [ [ "100 rows", - 0.00011716102437731024, + 0.00011428218567743897, 0.0007652332342331647 ], [ "1k rows", - 0.001184829429500193, + 0.0011681491435714027, 0.007443119815434329 ], [ "10k rows", - 0.014274019122240134, + 0.014314786463122195, 0.07339981250697747 ] ] diff --git a/docs/migration/bench/beautifulsoup.json b/docs/migration/bench/beautifulsoup.json index cb3781e2..999c9be7 100644 --- a/docs/migration/bench/beautifulsoup.json +++ b/docs/migration/bench/beautifulsoup.json @@ -8,32 +8,32 @@ "rows": [ [ "build a list (constructors) — 100 rows", - 5.680004882909392e-05, + 5.5585856197618945e-05, 0.000755816298199837 ], [ "build a list (constructors) — 1k rows", - 0.0005429942487656566, + 0.0005315817325026728, 0.0073866467088616146 ], [ "build a list (constructors) — 10k rows", - 0.005344225261069369, + 0.0052333376722041676, 0.08085168067676325 ], [ "construct N elements (no serialize) — 100 rows", - 4.5046813956635866e-05, + 4.236423571531608e-05, 0.00026231168612866895 ], [ "construct N elements (no serialize) — 1k rows", - 0.00048195762762285693, + 0.0004373766555545444, 0.0025528541664243676 ], [ "construct N elements (no serialize) — 10k rows", - 0.004416420136597783, + 0.0043080967928593355, 0.025308875076007098 ], [ diff --git a/docs/migration/bench/dominate.json b/docs/migration/bench/dominate.json index ec8b4693..7a4d2ec4 100644 --- a/docs/migration/bench/dominate.json +++ b/docs/migration/bench/dominate.json @@ -8,32 +8,32 @@ "rows": [ [ "build a list (terse builders) — 100 rows", - 0.00011716102437731024, + 0.00011428218567743897, 0.000437955457073258 ], [ "build a list (terse builders) — 1k rows", - 0.001184829429500193, + 0.0011681491435714027, 0.004424150614795508 ], [ "build a list (terse builders) — 10k rows", - 0.014274019122240134, + 0.014314786463122195, 0.04696243749155352 ], [ "construct N elements (no serialize) — 100 rows", - 4.5046813956635866e-05, + 4.236423571531608e-05, 0.0002446207682472353 ], [ "construct N elements (no serialize) — 1k rows", - 0.00048195762762285693, + 0.0004373766555545444, 0.0024456197916151723 ], [ "construct N elements (no serialize) — 10k rows", - 0.004416420136597783, + 0.0043080967928593355, 0.024981239592307247 ], [ diff --git a/docs/migration/bench/fast-html.json b/docs/migration/bench/fast-html.json index 5791ed7e..5ea2cdfe 100644 --- a/docs/migration/bench/fast-html.json +++ b/docs/migration/bench/fast-html.json @@ -8,17 +8,17 @@ "rows": [ [ "100 rows", - 0.00011716102437731024, + 0.00011428218567743897, 0.0002738594563804024 ], [ "1k rows", - 0.001184829429500193, + 0.0011681491435714027, 0.0028063719685936426 ], [ "10k rows", - 0.014274019122240134, + 0.014314786463122195, 0.028608680586330593 ] ] diff --git a/docs/migration/bench/htbuilder.json b/docs/migration/bench/htbuilder.json index dc3c4aa8..47189a7f 100644 --- a/docs/migration/bench/htbuilder.json +++ b/docs/migration/bench/htbuilder.json @@ -8,32 +8,32 @@ "rows": [ [ "build a list (terse builders) — 100 rows", - 0.00011716102437731024, + 0.00011428218567743897, 0.00022387307361289763 ], [ "build a list (terse builders) — 1k rows", - 0.001184829429500193, + 0.0011681491435714027, 0.00231864908952654 ], [ "build a list (terse builders) — 10k rows", - 0.014274019122240134, + 0.014314786463122195, 0.02611050000026201 ], [ "construct N elements (no serialize) — 100 rows", - 4.5046813956635866e-05, + 4.236423571531608e-05, 0.00014903073470880676 ], [ "construct N elements (no serialize) — 1k rows", - 0.00048195762762285693, + 0.0004373766555545444, 0.0015199997839469386 ], [ "construct N elements (no serialize) — 10k rows", - 0.004416420136597783, + 0.0043080967928593355, 0.015160777751589194 ], [ diff --git a/docs/migration/bench/htpy.json b/docs/migration/bench/htpy.json index c6d1f3ee..4267e4b4 100644 --- a/docs/migration/bench/htpy.json +++ b/docs/migration/bench/htpy.json @@ -8,32 +8,32 @@ "rows": [ [ "build a list (terse builders) — 100 rows", - 0.00011716102437731024, + 0.00011428218567743897, 0.0003595937233361231 ], [ "build a list (terse builders) — 1k rows", - 0.001184829429500193, + 0.0011681491435714027, 0.0035140655430344245 ], [ "build a list (terse builders) — 10k rows", - 0.014274019122240134, + 0.014314786463122195, 0.03526455908043621 ], [ "construct N elements (no serialize) — 100 rows", - 4.5046813956635866e-05, + 4.236423571531608e-05, 0.00024182763672797591 ], [ "construct N elements (no serialize) — 1k rows", - 0.00048195762762285693, + 0.0004373766555545444, 0.002418691400331833 ], [ "construct N elements (no serialize) — 10k rows", - 0.004416420136597783, + 0.0043080967928593355, 0.024709956623458613 ], [ diff --git a/docs/migration/bench/hyperpython.json b/docs/migration/bench/hyperpython.json index 5c83a118..a52ca348 100644 --- a/docs/migration/bench/hyperpython.json +++ b/docs/migration/bench/hyperpython.json @@ -8,32 +8,32 @@ "rows": [ [ "build a list (terse builders) — 100 rows", - 0.00011716102437731024, + 0.00011428218567743897, 0.00034247941077107197 ], [ "build a list (terse builders) — 1k rows", - 0.001184829429500193, + 0.0011681491435714027, 0.0034413402718200814 ], [ "build a list (terse builders) — 10k rows", - 0.014274019122240134, + 0.014314786463122195, 0.03530683333519846 ], [ "construct N elements (no serialize) — 100 rows", - 4.5046813956635866e-05, + 4.236423571531608e-05, 0.0001921427409191286 ], [ "construct N elements (no serialize) — 1k rows", - 0.00048195762762285693, + 0.0004373766555545444, 0.0019511121875742294 ], [ "construct N elements (no serialize) — 10k rows", - 0.004416420136597783, + 0.0043080967928593355, 0.01924277607758995 ], [ diff --git a/docs/migration/bench/lxml.json b/docs/migration/bench/lxml.json index 2faedfa7..72b48cf8 100644 --- a/docs/migration/bench/lxml.json +++ b/docs/migration/bench/lxml.json @@ -8,47 +8,47 @@ "rows": [ [ "build a list (constructors) — 100 rows", - 5.680004882909392e-05, + 5.5585856197618945e-05, 0.00012973849804135776 ], [ "build a list (constructors) — 1k rows", - 0.0005429942487656566, + 0.0005315817325026728, 0.0013196478930694866 ], [ "build a list (constructors) — 10k rows", - 0.005344225261069369, + 0.0052333376722041676, 0.013233920122729614 ], [ "build a list (terse builders) — 100 rows", - 0.00011716102437731024, + 0.00011428218567743897, 0.00019785382748220096 ], [ "build a list (terse builders) — 1k rows", - 0.001184829429500193, + 0.0011681491435714027, 0.0019465998284431407 ], [ "build a list (terse builders) — 10k rows", - 0.014274019122240134, + 0.014314786463122195, 0.02056280729205658 ], [ "construct N elements (no serialize) — 100 rows", - 4.5046813956635866e-05, + 4.236423571531608e-05, 9.010871709354736e-05 ], [ "construct N elements (no serialize) — 1k rows", - 0.00048195762762285693, + 0.0004373766555545444, 0.0009515415598192097 ], [ "construct N elements (no serialize) — 10k rows", - 0.004416420136597783, + 0.0043080967928593355, 0.009378886289293101 ], [ diff --git a/docs/migration/bench/markyp.json b/docs/migration/bench/markyp.json index 81ebf417..7ca5f052 100644 --- a/docs/migration/bench/markyp.json +++ b/docs/migration/bench/markyp.json @@ -8,32 +8,32 @@ "rows": [ [ "build a list (terse builders) — 100 rows", - 0.00011716102437731024, + 0.00011428218567743897, 0.00010226232908886838 ], [ "build a list (terse builders) — 1k rows", - 0.001184829429500193, + 0.0011681491435714027, 0.0009990138878492871 ], [ "build a list (terse builders) — 10k rows", - 0.014274019122240134, + 0.014314786463122195, 0.010889369771272564 ], [ "construct N elements (no serialize) — 100 rows", - 4.5046813956635866e-05, + 4.236423571531608e-05, 2.3579376261295693e-05 ], [ "construct N elements (no serialize) — 1k rows", - 0.00048195762762285693, + 0.0004373766555545444, 0.00023230053640569773 ], [ "construct N elements (no serialize) — 10k rows", - 0.004416420136597783, + 0.0043080967928593355, 0.0023480466564554567 ], [ diff --git a/docs/migration/bench/simple-html.json b/docs/migration/bench/simple-html.json index 8c286e37..10e02103 100644 --- a/docs/migration/bench/simple-html.json +++ b/docs/migration/bench/simple-html.json @@ -8,32 +8,32 @@ "rows": [ [ "build a list (terse builders) — 100 rows", - 0.00011716102437731024, + 0.00011428218567743897, 4.539174730666673e-05 ], [ "build a list (terse builders) — 1k rows", - 0.001184829429500193, + 0.0011681491435714027, 0.00045403689318845863 ], [ "build a list (terse builders) — 10k rows", - 0.014274019122240134, + 0.014314786463122195, 0.004933736094244523 ], [ "construct N elements (no serialize) — 100 rows", - 4.5046813956635866e-05, + 4.236423571531608e-05, 3.591008504599813e-05 ], [ "construct N elements (no serialize) — 1k rows", - 0.00048195762762285693, + 0.0004373766555545444, 0.000367746582090452 ], [ "construct N elements (no serialize) — 10k rows", - 0.004416420136597783, + 0.0043080967928593355, 0.003697205727803521 ], [ diff --git a/docs/migration/bench/yattag.json b/docs/migration/bench/yattag.json index d3fdfc7a..fd8bec65 100644 --- a/docs/migration/bench/yattag.json +++ b/docs/migration/bench/yattag.json @@ -8,17 +8,17 @@ "rows": [ [ "100 rows", - 0.00011716102437731024, + 0.00011428218567743897, 0.0001471504046624735 ], [ "1k rows", - 0.001184829429500193, + 0.0011681491435714027, 0.0013629659297293983 ], [ "10k rows", - 0.014274019122240134, + 0.014314786463122195, 0.013686433997160444 ] ] diff --git a/docs/tutorials/tokenizing.rst b/docs/tutorials/tokenizing.rst index 4f399cda..36e0c230 100644 --- a/docs/tutorials/tokenizing.rst +++ b/docs/tutorials/tokenizing.rst @@ -70,6 +70,19 @@ remains: The incomplete ``')) + print(start.tag, start.attrs) + +.. testoutput:: + + a [] + The tokenizer tracks the offset of every construct as it runs, and :func:`turbohtml.parse` can carry those offsets onto the tree it builds. Pass ``source_locations=True`` and each element's :attr:`~turbohtml.Node.source_location` gives the span of its start tag, its end tag, and each attribute -- the same information as parse5's ``sourceCodeLocationInfo``, diff --git a/src/turbohtml/_c/core/module.c b/src/turbohtml/_c/core/module.c index b36c66fa..1ef83c96 100644 --- a/src/turbohtml/_c/core/module.c +++ b/src/turbohtml/_c/core/module.c @@ -63,13 +63,16 @@ PyDoc_STRVAR(markup_soft_str_doc, "soft_str(s, /)\n--\n\n" ":param s: value to coerce to text.\n" ":returns: s unchanged when it is already a str, otherwise str(s)."); -PyDoc_STRVAR(tokenize_doc, "tokenize(s, /, *, resolve_references=True, capture_source=False)\n--\n\n" +PyDoc_STRVAR(tokenize_doc, + "tokenize(s, /, *, resolve_references=True, capture_source=False, capture_attributes=True)\n--\n\n" "Tokenize a whole HTML string following the WHATWG tokenization algorithm.\n\n" ":param s: the HTML to tokenize.\n" ":param resolve_references: fold character references into the surrounding text\n" " run; when False each one becomes its own CHARACTER_REFERENCE token.\n" ":param capture_source: record each markup token's verbatim source on\n" " Token.source.\n" + ":param capture_attributes: retain tag attributes; disable this when only tag names and\n" + " text are needed.\n" ":returns: an iterator of Token objects in document order.\n" ":raises TypeError: if s is not a str."); diff --git a/src/turbohtml/_c/data/tag_atom.h b/src/turbohtml/_c/data/tag_atom.h index 5d64bd40..9b7ddc03 100644 --- a/src/turbohtml/_c/data/tag_atom.h +++ b/src/turbohtml/_c/data/tag_atom.h @@ -176,179 +176,178 @@ typedef struct { uint8_t name_len; uint16_t atom; uint8_t flags; + uint16_t wide_offset; } th_tag_entry; static const int th_tag_count = 152; static const th_tag_entry th_tag_table[] = { - {"a", 1u, TH_TAG_A, TH_TAG_FORMATTING}, - {"abbr", 4u, TH_TAG_ABBR, 0}, - {"address", 7u, TH_TAG_ADDRESS, TH_TAG_SPECIAL}, - {"annotation-xml", 14u, TH_TAG_ANNOTATION_XML, 0}, - {"applet", 6u, TH_TAG_APPLET, TH_TAG_SPECIAL | TH_TAG_SCOPING}, - {"area", 4u, TH_TAG_AREA, TH_TAG_SPECIAL}, - {"article", 7u, TH_TAG_ARTICLE, TH_TAG_SPECIAL}, - {"aside", 5u, TH_TAG_ASIDE, TH_TAG_SPECIAL}, - {"audio", 5u, TH_TAG_AUDIO, 0}, - {"b", 1u, TH_TAG_B, TH_TAG_FORMATTING}, - {"base", 4u, TH_TAG_BASE, TH_TAG_SPECIAL}, - {"basefont", 8u, TH_TAG_BASEFONT, TH_TAG_SPECIAL}, - {"bdi", 3u, TH_TAG_BDI, 0}, - {"bdo", 3u, TH_TAG_BDO, 0}, - {"bgsound", 7u, TH_TAG_BGSOUND, TH_TAG_SPECIAL}, - {"big", 3u, TH_TAG_BIG, TH_TAG_FORMATTING}, - {"blockquote", 10u, TH_TAG_BLOCKQUOTE, TH_TAG_SPECIAL}, - {"body", 4u, TH_TAG_BODY, TH_TAG_SPECIAL}, - {"br", 2u, TH_TAG_BR, TH_TAG_SPECIAL}, - {"button", 6u, TH_TAG_BUTTON, TH_TAG_SPECIAL}, - {"canvas", 6u, TH_TAG_CANVAS, 0}, - {"caption", 7u, TH_TAG_CAPTION, TH_TAG_SPECIAL | TH_TAG_SCOPING}, - {"center", 6u, TH_TAG_CENTER, TH_TAG_SPECIAL}, - {"cite", 4u, TH_TAG_CITE, 0}, - {"code", 4u, TH_TAG_CODE, TH_TAG_FORMATTING}, - {"col", 3u, TH_TAG_COL, TH_TAG_SPECIAL}, - {"colgroup", 8u, TH_TAG_COLGROUP, TH_TAG_SPECIAL}, - {"command", 7u, TH_TAG_COMMAND, TH_TAG_SPECIAL}, - {"data", 4u, TH_TAG_DATA, 0}, - {"datalist", 8u, TH_TAG_DATALIST, 0}, - {"dd", 2u, TH_TAG_DD, TH_TAG_SPECIAL}, - {"del", 3u, TH_TAG_DEL, 0}, - {"desc", 4u, TH_TAG_DESC, 0}, - {"details", 7u, TH_TAG_DETAILS, TH_TAG_SPECIAL}, - {"dfn", 3u, TH_TAG_DFN, 0}, - {"dialog", 6u, TH_TAG_DIALOG, 0}, - {"dir", 3u, TH_TAG_DIR, TH_TAG_SPECIAL}, - {"div", 3u, TH_TAG_DIV, TH_TAG_SPECIAL}, - {"dl", 2u, TH_TAG_DL, TH_TAG_SPECIAL}, - {"dt", 2u, TH_TAG_DT, TH_TAG_SPECIAL}, - {"em", 2u, TH_TAG_EM, TH_TAG_FORMATTING}, - {"embed", 5u, TH_TAG_EMBED, TH_TAG_SPECIAL}, - {"fieldset", 8u, TH_TAG_FIELDSET, TH_TAG_SPECIAL}, - {"figcaption", 10u, TH_TAG_FIGCAPTION, TH_TAG_SPECIAL}, - {"figure", 6u, TH_TAG_FIGURE, TH_TAG_SPECIAL}, - {"font", 4u, TH_TAG_FONT, TH_TAG_FORMATTING}, - {"footer", 6u, TH_TAG_FOOTER, TH_TAG_SPECIAL}, - {"foreignobject", 13u, TH_TAG_FOREIGNOBJECT, 0}, - {"form", 4u, TH_TAG_FORM, TH_TAG_SPECIAL}, - {"frame", 5u, TH_TAG_FRAME, TH_TAG_SPECIAL}, - {"frameset", 8u, TH_TAG_FRAMESET, TH_TAG_SPECIAL}, - {"g", 1u, TH_TAG_G, 0}, - {"h1", 2u, TH_TAG_H1, TH_TAG_SPECIAL}, - {"h2", 2u, TH_TAG_H2, TH_TAG_SPECIAL}, - {"h3", 2u, TH_TAG_H3, TH_TAG_SPECIAL}, - {"h4", 2u, TH_TAG_H4, TH_TAG_SPECIAL}, - {"h5", 2u, TH_TAG_H5, TH_TAG_SPECIAL}, - {"h6", 2u, TH_TAG_H6, TH_TAG_SPECIAL}, - {"head", 4u, TH_TAG_HEAD, TH_TAG_SPECIAL}, - {"header", 6u, TH_TAG_HEADER, TH_TAG_SPECIAL}, - {"hgroup", 6u, TH_TAG_HGROUP, TH_TAG_SPECIAL}, - {"hr", 2u, TH_TAG_HR, TH_TAG_SPECIAL}, - {"html", 4u, TH_TAG_HTML, TH_TAG_SPECIAL | TH_TAG_SCOPING}, - {"i", 1u, TH_TAG_I, TH_TAG_FORMATTING}, - {"iframe", 6u, TH_TAG_IFRAME, TH_TAG_SPECIAL | TH_TAG_RAWTEXT}, - {"image", 5u, TH_TAG_IMAGE, TH_TAG_SPECIAL}, - {"img", 3u, TH_TAG_IMG, TH_TAG_SPECIAL}, - {"input", 5u, TH_TAG_INPUT, TH_TAG_SPECIAL}, - {"ins", 3u, TH_TAG_INS, 0}, - {"isindex", 7u, TH_TAG_ISINDEX, TH_TAG_SPECIAL}, - {"kbd", 3u, TH_TAG_KBD, 0}, - {"keygen", 6u, TH_TAG_KEYGEN, TH_TAG_SPECIAL}, - {"label", 5u, TH_TAG_LABEL, 0}, - {"legend", 6u, TH_TAG_LEGEND, 0}, - {"li", 2u, TH_TAG_LI, TH_TAG_SPECIAL}, - {"link", 4u, TH_TAG_LINK, TH_TAG_SPECIAL}, - {"listing", 7u, TH_TAG_LISTING, TH_TAG_SPECIAL}, - {"main", 4u, TH_TAG_MAIN, TH_TAG_SPECIAL}, - {"malignmark", 10u, TH_TAG_MALIGNMARK, 0}, - {"map", 3u, TH_TAG_MAP, 0}, - {"mark", 4u, TH_TAG_MARK, 0}, - {"marquee", 7u, TH_TAG_MARQUEE, TH_TAG_SPECIAL | TH_TAG_SCOPING}, - {"math", 4u, TH_TAG_MATH, 0}, - {"menu", 4u, TH_TAG_MENU, TH_TAG_SPECIAL}, - {"meta", 4u, TH_TAG_META, TH_TAG_SPECIAL}, - {"meter", 5u, TH_TAG_METER, 0}, - {"mglyph", 6u, TH_TAG_MGLYPH, 0}, - {"mi", 2u, TH_TAG_MI, 0}, - {"mn", 2u, TH_TAG_MN, 0}, - {"mo", 2u, TH_TAG_MO, 0}, - {"ms", 2u, TH_TAG_MS, 0}, - {"mtext", 5u, TH_TAG_MTEXT, 0}, - {"nav", 3u, TH_TAG_NAV, TH_TAG_SPECIAL}, - {"nobr", 4u, TH_TAG_NOBR, TH_TAG_FORMATTING}, - {"noembed", 7u, TH_TAG_NOEMBED, TH_TAG_SPECIAL | TH_TAG_RAWTEXT}, - {"noframes", 8u, TH_TAG_NOFRAMES, TH_TAG_SPECIAL | TH_TAG_RAWTEXT}, - {"noscript", 8u, TH_TAG_NOSCRIPT, TH_TAG_SPECIAL | TH_TAG_RAWTEXT}, - {"object", 6u, TH_TAG_OBJECT, TH_TAG_SPECIAL | TH_TAG_SCOPING}, - {"ol", 2u, TH_TAG_OL, TH_TAG_SPECIAL}, - {"optgroup", 8u, TH_TAG_OPTGROUP, 0}, - {"option", 6u, TH_TAG_OPTION, 0}, - {"output", 6u, TH_TAG_OUTPUT, 0}, - {"p", 1u, TH_TAG_P, TH_TAG_SPECIAL}, - {"param", 5u, TH_TAG_PARAM, TH_TAG_SPECIAL}, - {"path", 4u, TH_TAG_PATH, 0}, - {"picture", 7u, TH_TAG_PICTURE, 0}, - {"plaintext", 9u, TH_TAG_PLAINTEXT, TH_TAG_SPECIAL}, - {"pre", 3u, TH_TAG_PRE, TH_TAG_SPECIAL}, - {"progress", 8u, TH_TAG_PROGRESS, 0}, - {"q", 1u, TH_TAG_Q, 0}, - {"rb", 2u, TH_TAG_RB, 0}, - {"rp", 2u, TH_TAG_RP, 0}, - {"rt", 2u, TH_TAG_RT, 0}, - {"rtc", 3u, TH_TAG_RTC, 0}, - {"ruby", 4u, TH_TAG_RUBY, 0}, - {"s", 1u, TH_TAG_S, TH_TAG_FORMATTING}, - {"samp", 4u, TH_TAG_SAMP, 0}, - {"script", 6u, TH_TAG_SCRIPT, TH_TAG_SPECIAL}, - {"search", 6u, TH_TAG_SEARCH, TH_TAG_SPECIAL}, - {"section", 7u, TH_TAG_SECTION, TH_TAG_SPECIAL}, - {"select", 6u, TH_TAG_SELECT, TH_TAG_SPECIAL | TH_TAG_SCOPING}, - {"selectedcontent", 15u, TH_TAG_SELECTEDCONTENT, 0}, - {"slot", 4u, TH_TAG_SLOT, 0}, - {"small", 5u, TH_TAG_SMALL, TH_TAG_FORMATTING}, - {"source", 6u, TH_TAG_SOURCE, TH_TAG_SPECIAL}, - {"span", 4u, TH_TAG_SPAN, 0}, - {"strike", 6u, TH_TAG_STRIKE, TH_TAG_FORMATTING}, - {"strong", 6u, TH_TAG_STRONG, TH_TAG_FORMATTING}, - {"style", 5u, TH_TAG_STYLE, TH_TAG_SPECIAL | TH_TAG_RAWTEXT}, - {"sub", 3u, TH_TAG_SUB, 0}, - {"summary", 7u, TH_TAG_SUMMARY, TH_TAG_SPECIAL}, - {"sup", 3u, TH_TAG_SUP, 0}, - {"svg", 3u, TH_TAG_SVG, 0}, - {"table", 5u, TH_TAG_TABLE, TH_TAG_SPECIAL | TH_TAG_SCOPING}, - {"tbody", 5u, TH_TAG_TBODY, TH_TAG_SPECIAL}, - {"td", 2u, TH_TAG_TD, TH_TAG_SPECIAL | TH_TAG_SCOPING}, - {"template", 8u, TH_TAG_TEMPLATE, TH_TAG_SPECIAL | TH_TAG_SCOPING}, - {"textarea", 8u, TH_TAG_TEXTAREA, TH_TAG_SPECIAL | TH_TAG_RCDATA}, - {"tfoot", 5u, TH_TAG_TFOOT, TH_TAG_SPECIAL}, - {"th", 2u, TH_TAG_TH, TH_TAG_SPECIAL | TH_TAG_SCOPING}, - {"thead", 5u, TH_TAG_THEAD, TH_TAG_SPECIAL}, - {"time", 4u, TH_TAG_TIME, 0}, - {"title", 5u, TH_TAG_TITLE, TH_TAG_SPECIAL | TH_TAG_RCDATA}, - {"tr", 2u, TH_TAG_TR, TH_TAG_SPECIAL}, - {"track", 5u, TH_TAG_TRACK, TH_TAG_SPECIAL}, - {"tt", 2u, TH_TAG_TT, TH_TAG_FORMATTING}, - {"u", 1u, TH_TAG_U, TH_TAG_FORMATTING}, - {"ul", 2u, TH_TAG_UL, TH_TAG_SPECIAL}, - {"var", 3u, TH_TAG_VAR, 0}, - {"video", 5u, TH_TAG_VIDEO, 0}, - {"wbr", 3u, TH_TAG_WBR, TH_TAG_SPECIAL}, - {"xmp", 3u, TH_TAG_XMP, TH_TAG_SPECIAL | TH_TAG_RAWTEXT}, + {"a", 1u, TH_TAG_A, TH_TAG_FORMATTING, 0u}, + {"abbr", 4u, TH_TAG_ABBR, 0, 1u}, + {"address", 7u, TH_TAG_ADDRESS, TH_TAG_SPECIAL, 5u}, + {"annotation-xml", 14u, TH_TAG_ANNOTATION_XML, 0, 12u}, + {"applet", 6u, TH_TAG_APPLET, TH_TAG_SPECIAL | TH_TAG_SCOPING, 26u}, + {"area", 4u, TH_TAG_AREA, TH_TAG_SPECIAL, 32u}, + {"article", 7u, TH_TAG_ARTICLE, TH_TAG_SPECIAL, 36u}, + {"aside", 5u, TH_TAG_ASIDE, TH_TAG_SPECIAL, 43u}, + {"audio", 5u, TH_TAG_AUDIO, 0, 48u}, + {"b", 1u, TH_TAG_B, TH_TAG_FORMATTING, 53u}, + {"base", 4u, TH_TAG_BASE, TH_TAG_SPECIAL, 54u}, + {"basefont", 8u, TH_TAG_BASEFONT, TH_TAG_SPECIAL, 58u}, + {"bdi", 3u, TH_TAG_BDI, 0, 66u}, + {"bdo", 3u, TH_TAG_BDO, 0, 69u}, + {"bgsound", 7u, TH_TAG_BGSOUND, TH_TAG_SPECIAL, 72u}, + {"big", 3u, TH_TAG_BIG, TH_TAG_FORMATTING, 79u}, + {"blockquote", 10u, TH_TAG_BLOCKQUOTE, TH_TAG_SPECIAL, 82u}, + {"body", 4u, TH_TAG_BODY, TH_TAG_SPECIAL, 92u}, + {"br", 2u, TH_TAG_BR, TH_TAG_SPECIAL, 96u}, + {"button", 6u, TH_TAG_BUTTON, TH_TAG_SPECIAL, 98u}, + {"canvas", 6u, TH_TAG_CANVAS, 0, 104u}, + {"caption", 7u, TH_TAG_CAPTION, TH_TAG_SPECIAL | TH_TAG_SCOPING, 110u}, + {"center", 6u, TH_TAG_CENTER, TH_TAG_SPECIAL, 117u}, + {"cite", 4u, TH_TAG_CITE, 0, 123u}, + {"code", 4u, TH_TAG_CODE, TH_TAG_FORMATTING, 127u}, + {"col", 3u, TH_TAG_COL, TH_TAG_SPECIAL, 131u}, + {"colgroup", 8u, TH_TAG_COLGROUP, TH_TAG_SPECIAL, 134u}, + {"command", 7u, TH_TAG_COMMAND, TH_TAG_SPECIAL, 142u}, + {"data", 4u, TH_TAG_DATA, 0, 149u}, + {"datalist", 8u, TH_TAG_DATALIST, 0, 153u}, + {"dd", 2u, TH_TAG_DD, TH_TAG_SPECIAL, 161u}, + {"del", 3u, TH_TAG_DEL, 0, 163u}, + {"desc", 4u, TH_TAG_DESC, 0, 166u}, + {"details", 7u, TH_TAG_DETAILS, TH_TAG_SPECIAL, 170u}, + {"dfn", 3u, TH_TAG_DFN, 0, 177u}, + {"dialog", 6u, TH_TAG_DIALOG, 0, 180u}, + {"dir", 3u, TH_TAG_DIR, TH_TAG_SPECIAL, 186u}, + {"div", 3u, TH_TAG_DIV, TH_TAG_SPECIAL, 189u}, + {"dl", 2u, TH_TAG_DL, TH_TAG_SPECIAL, 192u}, + {"dt", 2u, TH_TAG_DT, TH_TAG_SPECIAL, 194u}, + {"em", 2u, TH_TAG_EM, TH_TAG_FORMATTING, 196u}, + {"embed", 5u, TH_TAG_EMBED, TH_TAG_SPECIAL, 198u}, + {"fieldset", 8u, TH_TAG_FIELDSET, TH_TAG_SPECIAL, 203u}, + {"figcaption", 10u, TH_TAG_FIGCAPTION, TH_TAG_SPECIAL, 211u}, + {"figure", 6u, TH_TAG_FIGURE, TH_TAG_SPECIAL, 221u}, + {"font", 4u, TH_TAG_FONT, TH_TAG_FORMATTING, 227u}, + {"footer", 6u, TH_TAG_FOOTER, TH_TAG_SPECIAL, 231u}, + {"foreignobject", 13u, TH_TAG_FOREIGNOBJECT, 0, 237u}, + {"form", 4u, TH_TAG_FORM, TH_TAG_SPECIAL, 250u}, + {"frame", 5u, TH_TAG_FRAME, TH_TAG_SPECIAL, 254u}, + {"frameset", 8u, TH_TAG_FRAMESET, TH_TAG_SPECIAL, 259u}, + {"g", 1u, TH_TAG_G, 0, 267u}, + {"h1", 2u, TH_TAG_H1, TH_TAG_SPECIAL, 268u}, + {"h2", 2u, TH_TAG_H2, TH_TAG_SPECIAL, 270u}, + {"h3", 2u, TH_TAG_H3, TH_TAG_SPECIAL, 272u}, + {"h4", 2u, TH_TAG_H4, TH_TAG_SPECIAL, 274u}, + {"h5", 2u, TH_TAG_H5, TH_TAG_SPECIAL, 276u}, + {"h6", 2u, TH_TAG_H6, TH_TAG_SPECIAL, 278u}, + {"head", 4u, TH_TAG_HEAD, TH_TAG_SPECIAL, 280u}, + {"header", 6u, TH_TAG_HEADER, TH_TAG_SPECIAL, 284u}, + {"hgroup", 6u, TH_TAG_HGROUP, TH_TAG_SPECIAL, 290u}, + {"hr", 2u, TH_TAG_HR, TH_TAG_SPECIAL, 296u}, + {"html", 4u, TH_TAG_HTML, TH_TAG_SPECIAL | TH_TAG_SCOPING, 298u}, + {"i", 1u, TH_TAG_I, TH_TAG_FORMATTING, 302u}, + {"iframe", 6u, TH_TAG_IFRAME, TH_TAG_SPECIAL | TH_TAG_RAWTEXT, 303u}, + {"image", 5u, TH_TAG_IMAGE, TH_TAG_SPECIAL, 309u}, + {"img", 3u, TH_TAG_IMG, TH_TAG_SPECIAL, 314u}, + {"input", 5u, TH_TAG_INPUT, TH_TAG_SPECIAL, 317u}, + {"ins", 3u, TH_TAG_INS, 0, 322u}, + {"isindex", 7u, TH_TAG_ISINDEX, TH_TAG_SPECIAL, 325u}, + {"kbd", 3u, TH_TAG_KBD, 0, 332u}, + {"keygen", 6u, TH_TAG_KEYGEN, TH_TAG_SPECIAL, 335u}, + {"label", 5u, TH_TAG_LABEL, 0, 341u}, + {"legend", 6u, TH_TAG_LEGEND, 0, 346u}, + {"li", 2u, TH_TAG_LI, TH_TAG_SPECIAL, 352u}, + {"link", 4u, TH_TAG_LINK, TH_TAG_SPECIAL, 354u}, + {"listing", 7u, TH_TAG_LISTING, TH_TAG_SPECIAL, 358u}, + {"main", 4u, TH_TAG_MAIN, TH_TAG_SPECIAL, 365u}, + {"malignmark", 10u, TH_TAG_MALIGNMARK, 0, 369u}, + {"map", 3u, TH_TAG_MAP, 0, 379u}, + {"mark", 4u, TH_TAG_MARK, 0, 382u}, + {"marquee", 7u, TH_TAG_MARQUEE, TH_TAG_SPECIAL | TH_TAG_SCOPING, 386u}, + {"math", 4u, TH_TAG_MATH, 0, 393u}, + {"menu", 4u, TH_TAG_MENU, TH_TAG_SPECIAL, 397u}, + {"meta", 4u, TH_TAG_META, TH_TAG_SPECIAL, 401u}, + {"meter", 5u, TH_TAG_METER, 0, 405u}, + {"mglyph", 6u, TH_TAG_MGLYPH, 0, 410u}, + {"mi", 2u, TH_TAG_MI, 0, 416u}, + {"mn", 2u, TH_TAG_MN, 0, 418u}, + {"mo", 2u, TH_TAG_MO, 0, 420u}, + {"ms", 2u, TH_TAG_MS, 0, 422u}, + {"mtext", 5u, TH_TAG_MTEXT, 0, 424u}, + {"nav", 3u, TH_TAG_NAV, TH_TAG_SPECIAL, 429u}, + {"nobr", 4u, TH_TAG_NOBR, TH_TAG_FORMATTING, 432u}, + {"noembed", 7u, TH_TAG_NOEMBED, TH_TAG_SPECIAL | TH_TAG_RAWTEXT, 436u}, + {"noframes", 8u, TH_TAG_NOFRAMES, TH_TAG_SPECIAL | TH_TAG_RAWTEXT, 443u}, + {"noscript", 8u, TH_TAG_NOSCRIPT, TH_TAG_SPECIAL | TH_TAG_RAWTEXT, 451u}, + {"object", 6u, TH_TAG_OBJECT, TH_TAG_SPECIAL | TH_TAG_SCOPING, 459u}, + {"ol", 2u, TH_TAG_OL, TH_TAG_SPECIAL, 465u}, + {"optgroup", 8u, TH_TAG_OPTGROUP, 0, 467u}, + {"option", 6u, TH_TAG_OPTION, 0, 475u}, + {"output", 6u, TH_TAG_OUTPUT, 0, 481u}, + {"p", 1u, TH_TAG_P, TH_TAG_SPECIAL, 487u}, + {"param", 5u, TH_TAG_PARAM, TH_TAG_SPECIAL, 488u}, + {"path", 4u, TH_TAG_PATH, 0, 493u}, + {"picture", 7u, TH_TAG_PICTURE, 0, 497u}, + {"plaintext", 9u, TH_TAG_PLAINTEXT, TH_TAG_SPECIAL, 504u}, + {"pre", 3u, TH_TAG_PRE, TH_TAG_SPECIAL, 513u}, + {"progress", 8u, TH_TAG_PROGRESS, 0, 516u}, + {"q", 1u, TH_TAG_Q, 0, 524u}, + {"rb", 2u, TH_TAG_RB, 0, 525u}, + {"rp", 2u, TH_TAG_RP, 0, 527u}, + {"rt", 2u, TH_TAG_RT, 0, 529u}, + {"rtc", 3u, TH_TAG_RTC, 0, 531u}, + {"ruby", 4u, TH_TAG_RUBY, 0, 534u}, + {"s", 1u, TH_TAG_S, TH_TAG_FORMATTING, 538u}, + {"samp", 4u, TH_TAG_SAMP, 0, 539u}, + {"script", 6u, TH_TAG_SCRIPT, TH_TAG_SPECIAL, 543u}, + {"search", 6u, TH_TAG_SEARCH, TH_TAG_SPECIAL, 549u}, + {"section", 7u, TH_TAG_SECTION, TH_TAG_SPECIAL, 555u}, + {"select", 6u, TH_TAG_SELECT, TH_TAG_SPECIAL | TH_TAG_SCOPING, 562u}, + {"selectedcontent", 15u, TH_TAG_SELECTEDCONTENT, 0, 568u}, + {"slot", 4u, TH_TAG_SLOT, 0, 583u}, + {"small", 5u, TH_TAG_SMALL, TH_TAG_FORMATTING, 587u}, + {"source", 6u, TH_TAG_SOURCE, TH_TAG_SPECIAL, 592u}, + {"span", 4u, TH_TAG_SPAN, 0, 598u}, + {"strike", 6u, TH_TAG_STRIKE, TH_TAG_FORMATTING, 602u}, + {"strong", 6u, TH_TAG_STRONG, TH_TAG_FORMATTING, 608u}, + {"style", 5u, TH_TAG_STYLE, TH_TAG_SPECIAL | TH_TAG_RAWTEXT, 614u}, + {"sub", 3u, TH_TAG_SUB, 0, 619u}, + {"summary", 7u, TH_TAG_SUMMARY, TH_TAG_SPECIAL, 622u}, + {"sup", 3u, TH_TAG_SUP, 0, 629u}, + {"svg", 3u, TH_TAG_SVG, 0, 632u}, + {"table", 5u, TH_TAG_TABLE, TH_TAG_SPECIAL | TH_TAG_SCOPING, 635u}, + {"tbody", 5u, TH_TAG_TBODY, TH_TAG_SPECIAL, 640u}, + {"td", 2u, TH_TAG_TD, TH_TAG_SPECIAL | TH_TAG_SCOPING, 645u}, + {"template", 8u, TH_TAG_TEMPLATE, TH_TAG_SPECIAL | TH_TAG_SCOPING, 647u}, + {"textarea", 8u, TH_TAG_TEXTAREA, TH_TAG_SPECIAL | TH_TAG_RCDATA, 655u}, + {"tfoot", 5u, TH_TAG_TFOOT, TH_TAG_SPECIAL, 663u}, + {"th", 2u, TH_TAG_TH, TH_TAG_SPECIAL | TH_TAG_SCOPING, 668u}, + {"thead", 5u, TH_TAG_THEAD, TH_TAG_SPECIAL, 670u}, + {"time", 4u, TH_TAG_TIME, 0, 675u}, + {"title", 5u, TH_TAG_TITLE, TH_TAG_SPECIAL | TH_TAG_RCDATA, 679u}, + {"tr", 2u, TH_TAG_TR, TH_TAG_SPECIAL, 684u}, + {"track", 5u, TH_TAG_TRACK, TH_TAG_SPECIAL, 686u}, + {"tt", 2u, TH_TAG_TT, TH_TAG_FORMATTING, 691u}, + {"u", 1u, TH_TAG_U, TH_TAG_FORMATTING, 693u}, + {"ul", 2u, TH_TAG_UL, TH_TAG_SPECIAL, 694u}, + {"var", 3u, TH_TAG_VAR, 0, 696u}, + {"video", 5u, TH_TAG_VIDEO, 0, 699u}, + {"wbr", 3u, TH_TAG_WBR, TH_TAG_SPECIAL, 704u}, + {"xmp", 3u, TH_TAG_XMP, TH_TAG_SPECIAL | TH_TAG_RAWTEXT, 707u}, }; +static const uint32_t th_tag_wide_names[] = { + 97u, 97u, 98u, 98u, 114u, 97u, 100u, 100u, 114u, 101u, 115u, 115u, 97u, 110u, 110u, 111u, 116u, 97u, 116u, 105u, 111u, 110u, 45u, 120u, 109u, 108u, 97u, 112u, 112u, 108u, 101u, 116u, 97u, 114u, 101u, 97u, 97u, 114u, 116u, 105u, 99u, 108u, 101u, 97u, 115u, 105u, 100u, 101u, 97u, 117u, 100u, 105u, 111u, 98u, 98u, 97u, 115u, 101u, 98u, 97u, 115u, 101u, 102u, 111u, 110u, 116u, 98u, 100u, 105u, 98u, 100u, 111u, 98u, 103u, 115u, 111u, 117u, 110u, 100u, 98u, 105u, 103u, 98u, 108u, 111u, 99u, 107u, 113u, 117u, 111u, 116u, 101u, 98u, 111u, 100u, 121u, 98u, 114u, 98u, 117u, 116u, 116u, 111u, 110u, 99u, 97u, 110u, 118u, 97u, 115u, 99u, 97u, 112u, 116u, 105u, 111u, 110u, 99u, 101u, 110u, 116u, 101u, 114u, 99u, 105u, 116u, 101u, 99u, 111u, 100u, 101u, 99u, 111u, 108u, 99u, 111u, 108u, 103u, 114u, 111u, 117u, 112u, 99u, 111u, 109u, 109u, 97u, 110u, 100u, 100u, 97u, 116u, 97u, 100u, 97u, 116u, 97u, 108u, 105u, 115u, 116u, 100u, 100u, 100u, 101u, 108u, 100u, 101u, 115u, 99u, 100u, 101u, 116u, 97u, 105u, 108u, 115u, 100u, 102u, 110u, 100u, 105u, 97u, 108u, 111u, 103u, 100u, 105u, 114u, 100u, 105u, 118u, 100u, 108u, 100u, 116u, 101u, 109u, 101u, 109u, 98u, 101u, 100u, 102u, 105u, 101u, 108u, 100u, 115u, 101u, 116u, 102u, 105u, 103u, 99u, 97u, 112u, 116u, 105u, 111u, 110u, 102u, 105u, 103u, 117u, 114u, 101u, 102u, 111u, 110u, 116u, 102u, 111u, 111u, 116u, 101u, 114u, 102u, 111u, 114u, 101u, 105u, 103u, 110u, 111u, 98u, 106u, 101u, 99u, 116u, 102u, 111u, 114u, 109u, 102u, 114u, 97u, 109u, 101u, 102u, 114u, 97u, 109u, 101u, 115u, 101u, 116u, 103u, 104u, 49u, 104u, 50u, 104u, 51u, 104u, 52u, 104u, 53u, 104u, 54u, 104u, 101u, 97u, 100u, 104u, 101u, 97u, 100u, 101u, 114u, 104u, 103u, 114u, 111u, 117u, 112u, 104u, 114u, 104u, 116u, 109u, 108u, 105u, 105u, 102u, 114u, 97u, 109u, 101u, 105u, 109u, 97u, 103u, 101u, 105u, 109u, 103u, 105u, 110u, 112u, 117u, 116u, 105u, 110u, 115u, 105u, 115u, 105u, 110u, 100u, 101u, 120u, 107u, 98u, 100u, 107u, 101u, 121u, 103u, 101u, 110u, 108u, 97u, 98u, 101u, 108u, 108u, 101u, 103u, 101u, 110u, 100u, 108u, 105u, 108u, 105u, 110u, 107u, 108u, 105u, 115u, 116u, 105u, 110u, 103u, 109u, 97u, 105u, 110u, 109u, 97u, 108u, 105u, 103u, 110u, 109u, 97u, 114u, 107u, 109u, 97u, 112u, 109u, 97u, 114u, 107u, 109u, 97u, 114u, 113u, 117u, 101u, 101u, 109u, 97u, 116u, 104u, 109u, 101u, 110u, 117u, 109u, 101u, 116u, 97u, 109u, 101u, 116u, 101u, 114u, 109u, 103u, 108u, 121u, 112u, 104u, 109u, 105u, 109u, 110u, 109u, 111u, 109u, 115u, 109u, 116u, 101u, 120u, 116u, 110u, 97u, 118u, 110u, 111u, 98u, 114u, 110u, 111u, 101u, 109u, 98u, 101u, 100u, 110u, 111u, 102u, 114u, 97u, 109u, 101u, 115u, 110u, 111u, 115u, 99u, 114u, 105u, 112u, 116u, 111u, 98u, 106u, 101u, 99u, 116u, 111u, 108u, 111u, 112u, 116u, 103u, 114u, 111u, 117u, 112u, 111u, 112u, 116u, 105u, 111u, 110u, 111u, 117u, 116u, 112u, 117u, 116u, 112u, 112u, 97u, 114u, 97u, 109u, 112u, 97u, 116u, 104u, 112u, 105u, 99u, 116u, 117u, 114u, 101u, 112u, 108u, 97u, 105u, 110u, 116u, 101u, 120u, 116u, 112u, 114u, 101u, 112u, 114u, 111u, 103u, 114u, 101u, 115u, 115u, 113u, 114u, 98u, 114u, 112u, 114u, 116u, 114u, 116u, 99u, 114u, 117u, 98u, 121u, 115u, 115u, 97u, 109u, 112u, 115u, 99u, 114u, 105u, 112u, 116u, 115u, 101u, 97u, 114u, 99u, 104u, 115u, 101u, 99u, 116u, 105u, 111u, 110u, 115u, 101u, 108u, 101u, 99u, 116u, 115u, 101u, 108u, 101u, 99u, 116u, 101u, 100u, 99u, 111u, 110u, 116u, 101u, 110u, 116u, 115u, 108u, 111u, 116u, 115u, 109u, 97u, 108u, 108u, 115u, 111u, 117u, 114u, 99u, 101u, 115u, 112u, 97u, 110u, 115u, 116u, 114u, 105u, 107u, 101u, 115u, 116u, 114u, 111u, 110u, 103u, 115u, 116u, 121u, 108u, 101u, 115u, 117u, 98u, 115u, 117u, 109u, 109u, 97u, 114u, 121u, 115u, 117u, 112u, 115u, 118u, 103u, 116u, 97u, 98u, 108u, 101u, 116u, 98u, 111u, 100u, 121u, 116u, 100u, 116u, 101u, 109u, 112u, 108u, 97u, 116u, 101u, 116u, 101u, 120u, 116u, 97u, 114u, 101u, 97u, 116u, 102u, 111u, 111u, 116u, 116u, 104u, 116u, 104u, 101u, 97u, 100u, 116u, 105u, 109u, 101u, 116u, 105u, 116u, 108u, 101u, 116u, 114u, 116u, 114u, 97u, 99u, 107u, 116u, 116u, 117u, 117u, 108u, 118u, 97u, 114u, 118u, 105u, 100u, 101u, 111u, 119u, 98u, 114u, 120u, 109u, 112u +}; + +static inline const uint32_t *th_tag_wide_name(uint16_t atom) { + return th_tag_wide_names + th_tag_table[atom - 1].wide_offset; +} + /* th_tag_first[c] is the first table index whose name starts with a byte >= c, so the entries beginning with byte c are [th_tag_first[c], th_tag_first[c + 1]). */ static const uint16_t th_tag_first[257] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 20, 28, 40, 42, 51, 52, 63, 70, 70, 72, 77, - 92, 97, 102, 109, 110, 115, 133, 146, 148, 150, 151, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, - 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, - 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, - 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, - 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, - 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, - 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152}; + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 20, 28, 40, 42, 51, 52, 63, 70, 70, 72, 77, 92, 97, 102, 109, 110, 115, 133, 146, 148, 150, 151, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152 +}; #endif /* TURBOHTML_TAG_ATOM_H */ diff --git a/src/turbohtml/_c/dom/element.c b/src/turbohtml/_c/dom/element.c index d558c1bd..65f94638 100644 --- a/src/turbohtml/_c/dom/element.c +++ b/src/turbohtml/_c/dom/element.c @@ -1863,19 +1863,6 @@ PyObject *make_element(PyTypeObject *type, PyObject *tag, PyObject *attrs) { Py_XDECREF(keys); /* GCOVR_EXCL_LINE: allocation-failure path */ return PyErr_NoMemory(); /* GCOVR_EXCL_LINE: allocation-failure path */ } - Py_UCS4 *tag_points = PyUnicode_AsUCS4Copy(tag); - if (tag_points == NULL) { /* GCOVR_EXCL_BR_LINE: allocation failure cannot be forced from a test */ - th_tree_free(tree); /* GCOVR_EXCL_LINE: allocation-failure path */ - Py_XDECREF(keys); /* GCOVR_EXCL_LINE: allocation-failure path */ - return NULL; /* GCOVR_EXCL_LINE: allocation-failure path */ - } - /* tag and attribute names are ASCII-lowercased to match what the parser - stores, so a constructed and a parsed element compare and serialize alike */ - for (Py_ssize_t index = 0; index < tag_len; index++) { - if (tag_points[index] >= 'A' && tag_points[index] <= 'Z') { - tag_points[index] += 32; - } - } Py_ssize_t utf8_len; const char *utf8 = PyUnicode_AsUTF8AndSize(tag, &utf8_len); uint16_t atom = TH_TAG_UNKNOWN; @@ -1888,6 +1875,19 @@ PyObject *make_element(PyTypeObject *type, PyObject *tag, PyObject *attrs) { } else { PyErr_Clear(); /* a surrogate or very long custom tag is not in the table */ } + Py_UCS4 *tag_points = atom == TH_TAG_UNKNOWN ? PyUnicode_AsUCS4Copy(tag) : NULL; + if (atom == TH_TAG_UNKNOWN && tag_points == NULL) { /* GCOVR_EXCL_BR_LINE: allocation failure */ + th_tree_free(tree); /* GCOVR_EXCL_LINE: allocation-failure path */ + Py_XDECREF(keys); /* GCOVR_EXCL_LINE: allocation-failure path */ + return NULL; /* GCOVR_EXCL_LINE: allocation-failure path */ + } + /* Unknown tag names are ASCII-lowercased to match what the parser stores. + Known names already point at their lowercase generated entry. */ + for (Py_ssize_t index = 0; index < tag_len && tag_points != NULL; index++) { + if (tag_points[index] >= 'A' && tag_points[index] <= 'Z') { + tag_points[index] += 32; + } + } th_node *node = th_tree_make_element(tree, tag_points, tag_len, atom, attr_count); PyMem_Free(tag_points); if (node == NULL) { /* GCOVR_EXCL_BR_LINE: allocation failure cannot be forced from a test */ diff --git a/src/turbohtml/_c/dom/mutate.c b/src/turbohtml/_c/dom/mutate.c index 30e1e384..e467788c 100644 --- a/src/turbohtml/_c/dom/mutate.c +++ b/src/turbohtml/_c/dom/mutate.c @@ -147,8 +147,8 @@ static uint32_t th_attr_intern_utf8(th_tree *tree, const char *bytes, Py_ssize_t return intern_attr_dynamic(tree, bytes, len); } -/* Construct an element node owning a copy of the tag name, with attr_count empty - attribute slots to fill with th_tree_set_attr. */ +/* Construct an element node, with attr_count empty attribute slots to fill with + th_tree_set_attr. A NULL known tag selects the generated immutable spelling. */ th_node *th_tree_make_element(th_tree *tree, const Py_UCS4 *tag, Py_ssize_t tag_len, uint16_t atom, Py_ssize_t attr_count) { th_node *node = node_new(tree, TH_NODE_ELEMENT); @@ -157,13 +157,20 @@ th_node *th_tree_make_element(th_tree *tree, const Py_UCS4 *tag, Py_ssize_t tag_ } node->atom = atom; node->tag_flags = th_tag_flags(atom); /* so a constructed/unpickled raw-text element serializes literally */ - Py_UCS4 *owned = arena_alloc(tree, tag_len * (Py_ssize_t)sizeof(Py_UCS4)); - if (owned == NULL) { /* GCOVR_EXCL_BR_LINE: allocation failure cannot be forced from a test */ - return NULL; /* GCOVR_EXCL_LINE: allocation-failure path */ + if (atom == TH_TAG_UNKNOWN || tag != NULL) { + if (tag_len > 0) { + Py_UCS4 *owned = arena_alloc(tree, tag_len * (Py_ssize_t)sizeof(Py_UCS4)); + if (owned == NULL) { /* GCOVR_EXCL_BR_LINE: allocation failure cannot be forced from a test */ + return NULL; /* GCOVR_EXCL_LINE: allocation-failure path */ + } + memcpy(owned, tag, (size_t)tag_len * sizeof(Py_UCS4)); + node->text = owned; + } + node->text_len = tag_len; + } else { + node->text = (Py_UCS4 *)th_tag_wide_name(atom); + node->text_len = th_tag_table[atom - 1].name_len; } - memcpy(owned, tag, (size_t)tag_len * sizeof(Py_UCS4)); - node->text = owned; - node->text_len = tag_len; if (attr_count > 0) { node->attrs = arena_alloc(tree, attr_count * (Py_ssize_t)sizeof(th_node_attr)); if (node->attrs == NULL) { /* GCOVR_EXCL_BR_LINE: allocation failure cannot be forced from a test */ diff --git a/src/turbohtml/_c/dom/nodes.h b/src/turbohtml/_c/dom/nodes.h index fadc4639..6f802272 100644 --- a/src/turbohtml/_c/dom/nodes.h +++ b/src/turbohtml/_c/dom/nodes.h @@ -85,9 +85,9 @@ typedef struct { and always 0, when encoding is None. */ int encoding_certain; int index_built; - sel_cache_entry sel_cache[SEL_CACHE_CAP]; + sel_cache_entry *sel_cache; int sel_cache_len; - xpath_cache_entry xpath_cache[XPATH_CACHE_CAP]; + xpath_cache_entry *xpath_cache; int xpath_cache_len; } HandleObject; diff --git a/src/turbohtml/_c/dom/tree.c b/src/turbohtml/_c/dom/tree.c index cf877cfe..810dd319 100644 --- a/src/turbohtml/_c/dom/tree.c +++ b/src/turbohtml/_c/dom/tree.c @@ -25,16 +25,7 @@ #include "dom/quirks.h" #include "dom/foreign.h" -/* Copy a th_buf's code points into an arena UCS4 array. */ -static Py_UCS4 *buf_to_ucs4(th_tree *tree, const th_buf *buf, Py_ssize_t *out_len) { - *out_len = buf->len; - if (buf->len == 0) { - return NULL; - } - Py_UCS4 *out = arena_alloc(tree, buf->len * (Py_ssize_t)sizeof(Py_UCS4)); - if (out == NULL) { /* GCOVR_EXCL_BR_LINE: allocation failure cannot be forced from a test */ - return NULL; /* GCOVR_EXCL_LINE: allocation-failure path, unreachable from a test */ - } +static void buf_write_ucs4(Py_UCS4 *out, const th_buf *buf) { /* Hoist the width dispatch out of the loop: a 1-byte buffer (every ASCII tag name) widens in a tight loop, a 4-byte one is a memcpy. */ if (buf->kind == PyUnicode_1BYTE_KIND) { @@ -50,6 +41,19 @@ static Py_UCS4 *buf_to_ucs4(th_tree *tree, const th_buf *buf, Py_ssize_t *out_le out[index] = units[index]; } } +} + +/* Copy a th_buf's code points into an arena UCS4 array. */ +static Py_UCS4 *buf_to_ucs4(th_tree *tree, const th_buf *buf, Py_ssize_t *out_len) { + *out_len = buf->len; + if (buf->len == 0) { + return NULL; + } + Py_UCS4 *out = arena_alloc(tree, buf->len * (Py_ssize_t)sizeof(Py_UCS4)); + if (out == NULL) { /* GCOVR_EXCL_BR_LINE: allocation failure cannot be forced from a test */ + return NULL; /* GCOVR_EXCL_LINE: allocation-failure path, unreachable from a test */ + } + buf_write_ucs4(out, buf); return out; } @@ -648,7 +652,9 @@ static void insertion_location(th_tree *tree, th_node **parent, th_node **before left absent until the source closes the element (record_end_tag_location). NULL on allocation failure. */ static th_src_loc *build_source_location(th_tree *tree, th_node *node, const th_token *token) { - th_src_loc *loc = arena_alloc(tree, (Py_ssize_t)sizeof(th_src_loc)); + th_src_loc *loc = arena_alloc( + tree, (Py_ssize_t)sizeof(th_src_loc) + token->attr_count * (Py_ssize_t)sizeof(th_src_attr) + ); if (loc == NULL) { /* GCOVR_EXCL_BR_LINE: allocation failure cannot be forced from a test */ return NULL; /* GCOVR_EXCL_LINE: allocation-failure path, unreachable from a test */ } @@ -658,17 +664,15 @@ static th_src_loc *build_source_location(th_tree *tree, th_node *node, const th_ loc->has_end_tag = 0; loc->start_dirty = 0; loc->attr_count = token->attr_count; - loc->attrs = NULL; + loc->attrs = token->attr_count > 0 ? (th_src_attr *)(loc + 1) : NULL; if (token->attr_count > 0) { - loc->attrs = arena_alloc(tree, token->attr_count * (Py_ssize_t)sizeof(th_src_attr)); - if (loc->attrs == NULL) { /* GCOVR_EXCL_BR_LINE: allocation failure cannot be forced from a test */ - return NULL; /* GCOVR_EXCL_LINE: allocation-failure path, unreachable from a test */ - } for (Py_ssize_t index = 0; index < token->attr_count; index++) { const th_attr *src = &token->attrs[index]; + const th_attr_loc *src_loc = &token->attr_locs[index]; loc->attrs[index].name_atom = intern_attr(tree, &src->name); /* idempotent with insert_element's */ loc->attrs[index].span = - (th_src_span){src->name_line, src->name_col, src->name_off, src->end_line, src->end_col, src->end_off}; + (th_src_span){src_loc->name_line, src_loc->name_col, src_loc->name_off, src_loc->end_line, + src_loc->end_col, src_loc->end_off}; } } *node_loc(node) = loc; @@ -698,20 +702,33 @@ static th_node *build_element(th_tree *tree, const th_token *token) { return NULL; /* GCOVR_EXCL_LINE: allocation-failure path */ } } - node->text = buf_to_ucs4(tree, &token->name, &node->text_len); + if (node->atom == TH_TAG_UNKNOWN) { + node->text = buf_to_ucs4(tree, &token->name, &node->text_len); + } else { + node->text = (Py_UCS4 *)th_tag_wide_name(node->atom); + node->text_len = th_tag_table[node->atom - 1].name_len; + } if (token->attr_count > 0) { - node->attrs = arena_alloc(tree, token->attr_count * (Py_ssize_t)sizeof(th_node_attr)); + Py_ssize_t value_len = 0; + for (Py_ssize_t index = 0; index < token->attr_count; index++) { + value_len += token->attrs[index].has_value ? token->attrs[index].value.len : 0; + } + Py_ssize_t attrs_size = token->attr_count * (Py_ssize_t)sizeof(th_node_attr); + node->attrs = arena_alloc(tree, attrs_size + value_len * (Py_ssize_t)sizeof(Py_UCS4)); if (node->attrs == NULL) { /* GCOVR_EXCL_BR_LINE: allocation failure cannot be forced from a test */ return NULL; /* GCOVR_EXCL_LINE: allocation-failure path, unreachable from a test */ } + Py_UCS4 *value = (Py_UCS4 *)((char *)node->attrs + attrs_size); node->attr_count = token->attr_count; for (Py_ssize_t index = 0; index < token->attr_count; index++) { const th_attr *src = &token->attrs[index]; th_node_attr *dst = &node->attrs[index]; dst->name_atom = intern_attr(tree, &src->name); - dst->value = src->has_value ? buf_to_ucs4(tree, &src->value, &dst->value_len) : NULL; - if (!src->has_value) { - dst->value_len = 0; + dst->value_len = src->has_value ? src->value.len : 0; + dst->value = dst->value_len > 0 ? value : NULL; + if (dst->value != NULL) { + buf_write_ucs4(value, &src->value); + value += dst->value_len; } } } @@ -1180,9 +1197,8 @@ static void insert_text_span(th_tree *tree, Py_ssize_t off, Py_ssize_t len) { if (node == NULL) { /* GCOVR_EXCL_BR_LINE: allocation failure cannot be forced from a test */ return; /* GCOVR_EXCL_LINE: allocation-failure path, unreachable from a test */ } - node->text = NULL; /* span marker */ + text_set_span(node, off); node->text_len = len; - node->attr_count = off; /* the span's start offset into the input */ node_insert_before(parent, node, before); } @@ -1999,22 +2015,15 @@ static int content_model_for(uint16_t atom, uint8_t flags, int scripting) { } /* Forward declaration: a fabricated start tag for implicit html/head/body. */ -static th_node *insert_implicit(th_tree *tree, const char *name, uint16_t atom, uint8_t flags); +static th_node *insert_implicit(th_tree *tree, uint16_t atom, uint8_t flags); -static th_node *insert_implicit(th_tree *tree, const char *name, uint16_t atom, uint8_t flags) { +static th_node *insert_implicit(th_tree *tree, uint16_t atom, uint8_t flags) { th_node *node = node_new(tree, TH_NODE_ELEMENT); if (node == NULL) { /* GCOVR_EXCL_BR_LINE: allocation failure cannot be forced from a test */ return NULL; /* GCOVR_EXCL_LINE: allocation-failure path, unreachable from a test */ } - Py_ssize_t len = (Py_ssize_t)strlen(name); - node->text = arena_alloc(tree, len * (Py_ssize_t)sizeof(Py_UCS4)); - if (node->text == NULL) { /* GCOVR_EXCL_BR_LINE: allocation failure cannot be forced from a test */ - return NULL; /* GCOVR_EXCL_LINE: allocation-failure path, unreachable from a test */ - } - for (Py_ssize_t index = 0; index < len; index++) { - node->text[index] = (Py_UCS4)name[index]; - } - node->text_len = len; + node->text = (Py_UCS4 *)th_tag_wide_name(atom); + node->text_len = th_tag_table[atom - 1].name_len; node->atom = atom; node->tag_flags = flags; th_node *parent, *before; @@ -2119,7 +2128,7 @@ static enum th_drain drain_before_html(th_tree *tree, th_token *tok, th_insert * } } { - th_node *html = insert_implicit(tree, "html", TH_TAG_HTML, TH_TAG_SPECIAL | TH_TAG_SCOPING); + th_node *html = insert_implicit(tree, TH_TAG_HTML, TH_TAG_SPECIAL | TH_TAG_SCOPING); if (html != NULL) { /* GCOVR_EXCL_BR_LINE: NULL only on alloc failure */ stack_push(tree, html); } @@ -2169,7 +2178,7 @@ static enum th_drain drain_before_head(th_tree *tree, th_token *tok, th_insert * return TH_DRAIN_NEXT; /* any other end tag before head is a parse error and ignored */ } } - tree->head = insert_implicit(tree, "head", TH_TAG_HEAD, TH_TAG_SPECIAL); + tree->head = insert_implicit(tree, TH_TAG_HEAD, TH_TAG_SPECIAL); stack_push(tree, tree->head); dc->mode = M_IN_HEAD; return TH_DRAIN_REPROCESS; @@ -2420,7 +2429,7 @@ static enum th_drain drain_after_head(th_tree *tree, th_token *tok, th_insert *d } } { - th_node *body = insert_implicit(tree, "body", TH_TAG_BODY, TH_TAG_SPECIAL); + th_node *body = insert_implicit(tree, TH_TAG_BODY, TH_TAG_SPECIAL); if (body != NULL) { /* GCOVR_EXCL_BR_LINE: NULL only on alloc failure */ stack_push(tree, body); } @@ -2862,14 +2871,8 @@ static enum th_drain drain_in_body(th_tree *tree, th_token *tok, th_insert *dc) th_node *node = insert_element(tree, tok); if (node != NULL) { /* GCOVR_EXCL_BR_LINE: NULL only on alloc failure */ node->atom = TH_TAG_IMG; - static const char img[] = "img"; - node->text = arena_alloc(tree, 3 * (Py_ssize_t)sizeof(Py_UCS4)); - if (node->text != NULL) { /* GCOVR_EXCL_BR_LINE: NULL only on alloc failure */ - for (int index = 0; index < 3; index++) { - node->text[index] = (Py_UCS4)img[index]; - } - node->text_len = 3; - } + node->text = (Py_UCS4 *)th_tag_wide_name(TH_TAG_IMG); + node->text_len = th_tag_table[TH_TAG_IMG - 1].name_len; } return TH_DRAIN_NEXT; /* img is void */ } @@ -2999,7 +3002,7 @@ static enum th_drain drain_in_body(th_tree *tree, th_token *tok, th_insert *dc) } if (atom == TH_TAG_P) { if (!has_in_button_scope(tree, TH_TAG_P)) { - insert_implicit(tree, "p", TH_TAG_P, TH_TAG_SPECIAL); /* empty p, immediately closed */ + insert_implicit(tree, TH_TAG_P, TH_TAG_SPECIAL); /* empty p, immediately closed */ } else { generate_implied_end_tags(tree, TH_TAG_P); pop_until_atom(tree, TH_TAG_P); @@ -3096,7 +3099,7 @@ static enum th_drain drain_in_body(th_tree *tree, th_token *tok, th_insert *dc) if (atom == TH_TAG_BR) { /*
acts as a
start tag (attributes dropped) */ reconstruct_afe(tree); - insert_implicit(tree, "br", TH_TAG_BR, TH_TAG_SPECIAL); + insert_implicit(tree, TH_TAG_BR, TH_TAG_SPECIAL); tree->frameset_ok = 0; return TH_DRAIN_NEXT; } @@ -3184,7 +3187,7 @@ static enum th_drain drain_in_table(th_tree *tree, th_token *tok, th_insert *dc) if (atom == TH_TAG_COL) { clear_to(tree, TH_TAG_TABLE, TH_TAG_TABLE, TH_TAG_TABLE); { - th_node *cg = insert_implicit(tree, "colgroup", TH_TAG_COLGROUP, TH_TAG_SPECIAL); + th_node *cg = insert_implicit(tree, TH_TAG_COLGROUP, TH_TAG_SPECIAL); if (cg != NULL) { /* GCOVR_EXCL_BR_LINE: NULL only on alloc failure */ stack_push(tree, cg); } @@ -3204,7 +3207,7 @@ static enum th_drain drain_in_table(th_tree *tree, th_token *tok, th_insert *dc) if (atom == TH_TAG_TD || atom == TH_TAG_TH || atom == TH_TAG_TR) { clear_to(tree, TH_TAG_TABLE, TH_TAG_TABLE, TH_TAG_TABLE); { - th_node *tb = insert_implicit(tree, "tbody", TH_TAG_TBODY, TH_TAG_SPECIAL); + th_node *tb = insert_implicit(tree, TH_TAG_TBODY, TH_TAG_SPECIAL); if (tb != NULL) { /* GCOVR_EXCL_BR_LINE: NULL only on alloc failure */ stack_push(tree, tb); } @@ -3379,7 +3382,7 @@ static enum th_drain drain_in_table_body(th_tree *tree, th_token *tok, th_insert if (atom == TH_TAG_TD || atom == TH_TAG_TH) { clear_to(tree, TH_TAG_TBODY, TH_TAG_TFOOT, TH_TAG_THEAD); { - th_node *row = insert_implicit(tree, "tr", TH_TAG_TR, TH_TAG_SPECIAL); + th_node *row = insert_implicit(tree, TH_TAG_TR, TH_TAG_SPECIAL); if (row != NULL) { /* GCOVR_EXCL_BR_LINE: NULL only on alloc failure */ stack_push(tree, row); } @@ -3861,21 +3864,15 @@ static th_node *child_with_atom(th_node *parent, uint16_t atom) { } /* Create a fabricated structural element with the given lowercase name/atom. */ -static th_node *make_named(th_tree *tree, const char *name, Py_ssize_t len, uint16_t atom, uint8_t flags) { +static th_node *make_named(th_tree *tree, uint16_t atom, uint8_t flags) { th_node *node = node_new(tree, TH_NODE_ELEMENT); if (node == NULL) { /* GCOVR_EXCL_BR_LINE: allocation failure cannot be forced from a test */ return NULL; /* GCOVR_EXCL_LINE: allocation-failure path, unreachable from a test */ } node->atom = atom; node->tag_flags = flags; - node->text = arena_alloc(tree, len * (Py_ssize_t)sizeof(Py_UCS4)); - if (node->text == NULL) { /* GCOVR_EXCL_BR_LINE: allocation failure cannot be forced from a test */ - return NULL; /* GCOVR_EXCL_LINE: allocation-failure path, unreachable from a test */ - } - for (Py_ssize_t index = 0; index < len; index++) { - node->text[index] = (Py_UCS4)name[index]; - } - node->text_len = len; + node->text = (Py_UCS4 *)th_tag_wide_name(atom); + node->text_len = th_tag_table[atom - 1].name_len; return node; } @@ -3889,7 +3886,7 @@ static void ensure_head_body(th_tree *tree, th_node *html) { } th_node *head = child_with_atom(html, TH_TAG_HEAD); if (head == NULL) { - head = make_named(tree, "head", 4, TH_TAG_HEAD, TH_TAG_SPECIAL); + head = make_named(tree, TH_TAG_HEAD, TH_TAG_SPECIAL); if (head == NULL) { /* GCOVR_EXCL_BR_LINE: allocation failure cannot be forced from a test */ return; /* GCOVR_EXCL_LINE: allocation-failure path, unreachable from a test */ } @@ -3897,7 +3894,7 @@ static void ensure_head_body(th_tree *tree, th_node *html) { node_append(html, head); } if (child_with_atom(html, TH_TAG_BODY) == NULL) { - th_node *body = make_named(tree, "body", 4, TH_TAG_BODY, TH_TAG_SPECIAL); + th_node *body = make_named(tree, TH_TAG_BODY, TH_TAG_SPECIAL); if (body == NULL) { /* GCOVR_EXCL_BR_LINE: allocation failure cannot be forced from a test */ return; /* GCOVR_EXCL_LINE: allocation-failure path, unreachable from a test */ } @@ -3910,7 +3907,7 @@ static void ensure_head_body(th_tree *tree, th_node *html) { static void finalize_document(th_tree *tree) { th_node *html = child_with_atom(tree->document, TH_TAG_HTML); if (html == NULL) { - html = make_named(tree, "html", 4, TH_TAG_HTML, TH_TAG_SPECIAL | TH_TAG_SCOPING); + html = make_named(tree, TH_TAG_HTML, TH_TAG_SPECIAL | TH_TAG_SCOPING); if (html == NULL) { /* GCOVR_EXCL_BR_LINE: allocation failure cannot be forced from a test */ return; /* GCOVR_EXCL_LINE: allocation-failure path, unreachable from a test */ } diff --git a/src/turbohtml/_c/dom/tree.h b/src/turbohtml/_c/dom/tree.h index 04f6560f..9aa657f8 100644 --- a/src/turbohtml/_c/dom/tree.h +++ b/src/turbohtml/_c/dom/tree.h @@ -251,11 +251,7 @@ th_node *th_tree_build_shell(th_tree *tree, const Py_UCS4 *lang, Py_ssize_t lang int th_node_attr_set(th_tree *tree, th_node *node, const char *name, Py_ssize_t name_len, const Py_UCS4 *value, Py_ssize_t value_len, int has_value); -/* An element's attribute array and count; (NULL, 0) for every non-element node. Only - an element carries attribute storage: a text-node span reuses attr_count to hold a - source offset with attrs == NULL, so a reader that walks arbitrary nodes (the XPath - attribute axis, lang()) must gate on the element type -- which it does by reaching - the storage only through this accessor (issues #401, #422). */ +/* An element's attribute array and count; (NULL, 0) for every non-element node. */ static inline Py_ssize_t th_node_attributes(const th_node *node, th_node_attr **attrs) { if (node->type == TH_NODE_ELEMENT) { *attrs = node->attrs; diff --git a/src/turbohtml/_c/dom/tree_internal.h b/src/turbohtml/_c/dom/tree_internal.h index a4135121..902ae61a 100644 --- a/src/turbohtml/_c/dom/tree_internal.h +++ b/src/turbohtml/_c/dom/tree_internal.h @@ -9,6 +9,7 @@ #include "dom/tree.h" +#include #include #define ARENA_BLOCK ((Py_ssize_t)64 * 1024) @@ -155,12 +156,23 @@ static inline Py_UCS4 *copy_input_span(th_tree *tree, Py_ssize_t off, Py_ssize_t return out; } -/* A TEXT node is a zero-copy span when text == NULL && text_len > 0: its content - is input[attr_count .. attr_count + text_len] (the attr_count field is unused - on text nodes). Realize it into the arena on first read. */ +/* A parsed TEXT node tags its source offset into the otherwise-aligned text + pointer. This leaves the element-only attrs fields outside its allocation. */ +static inline int text_is_span(const th_node *node) { + return node->type == TH_NODE_TEXT && ((uintptr_t)node->text & 1u) != 0; +} + +static inline Py_ssize_t text_span_offset(const th_node *node) { + return (Py_ssize_t)((uintptr_t)node->text >> 1); +} + +static inline void text_set_span(th_node *node, Py_ssize_t offset) { + node->text = (Py_UCS4 *)(((uintptr_t)offset << 1) | 1u); +} + static inline Py_UCS4 *need_text(th_tree *tree, th_node *node) { - if (node->text == NULL && node->text_len > 0) { /* GCOVR_EXCL_BR_LINE: a text span always has positive length */ - node->text = copy_input_span(tree, node->attr_count, node->text_len); + if (text_is_span(node)) { + node->text = copy_input_span(tree, text_span_offset(node), node->text_len); } return node->text; } @@ -213,13 +225,14 @@ static inline th_src_loc **node_loc(th_node *node) { static inline th_node *node_new(th_tree *tree, enum th_node_type type) { int positioned = tree->track_positions && type == TH_NODE_ELEMENT; int located = tree->track_locations && type == TH_NODE_ELEMENT; - Py_ssize_t size = (Py_ssize_t)sizeof(th_node) + (positioned ? 2 * (Py_ssize_t)sizeof(uint32_t) : 0) + - (located ? (Py_ssize_t)sizeof(th_src_loc *) : 0); + Py_ssize_t size = type == TH_NODE_TEXT ? (Py_ssize_t)offsetof(th_node, attrs) : (Py_ssize_t)sizeof(th_node); + size += (positioned ? 2 * (Py_ssize_t)sizeof(uint32_t) : 0) + + (located ? (Py_ssize_t)sizeof(th_src_loc *) : 0); th_node *node = arena_alloc(tree, size); if (node == NULL) { /* GCOVR_EXCL_BR_LINE: allocation failure cannot be forced from a test */ return NULL; /* GCOVR_EXCL_LINE: allocation-failure path, unreachable from a test */ } - memset(node, 0, sizeof(*node)); + memset(node, 0, (size_t)size); if (positioned) { node_pos(node)[0] = 0; /* line; 0 = no source until insert_element sets it */ node_pos(node)[1] = 0; /* col */ diff --git a/src/turbohtml/_c/query/methods.c b/src/turbohtml/_c/query/methods.c index 091c4455..aa6bc5b4 100644 --- a/src/turbohtml/_c/query/methods.c +++ b/src/turbohtml/_c/query/methods.c @@ -17,6 +17,8 @@ void handle_clear_caches(HandleObject *handle) { xp_free(handle->xpath_cache[index].prog); Py_DECREF(handle->xpath_cache[index].key); } + PyMem_Free(handle->sel_cache); + PyMem_Free(handle->xpath_cache); } /* The tag atom every alternative of compiled selects as its subject (rightmost @@ -82,6 +84,14 @@ static sel_compiled *cached_compile(PyObject *selector_error, HandleObject *hand if (compiled == NULL) { return NULL; } + if (handle->sel_cache == NULL) { + handle->sel_cache = PyMem_New(sel_cache_entry, SEL_CACHE_CAP); + if (handle->sel_cache == NULL) { /* GCOVR_EXCL_BR_LINE: allocation failure cannot be forced from a test */ + selector_free(compiled); /* GCOVR_EXCL_LINE: allocation-failure path */ + PyErr_NoMemory(); /* GCOVR_EXCL_LINE: allocation-failure path */ + return NULL; /* GCOVR_EXCL_LINE: allocation-failure path */ + } + } if (handle->sel_cache_len == SEL_CACHE_CAP) { sel_cache_entry *evicted = &handle->sel_cache[SEL_CACHE_CAP - 1]; selector_free(evicted->compiled); @@ -578,6 +588,14 @@ static xp_program *cached_xpath_compile(HandleObject *handle, PyObject *arg) { PyErr_SetString(PyExc_ValueError, err); return NULL; } + if (handle->xpath_cache == NULL) { + handle->xpath_cache = PyMem_New(xpath_cache_entry, XPATH_CACHE_CAP); + if (handle->xpath_cache == NULL) { /* GCOVR_EXCL_BR_LINE: allocation failure cannot be forced from a test */ + xp_free(prog); /* GCOVR_EXCL_LINE: allocation-failure path */ + PyErr_NoMemory(); /* GCOVR_EXCL_LINE: allocation-failure path */ + return NULL; /* GCOVR_EXCL_LINE: allocation-failure path */ + } + } if (handle->xpath_cache_len == XPATH_CACHE_CAP) { xpath_cache_entry *evicted = &handle->xpath_cache[XPATH_CACHE_CAP - 1]; xp_free(evicted->prog); diff --git a/src/turbohtml/_c/serialize/document.c b/src/turbohtml/_c/serialize/document.c index 88a81711..000bc107 100644 --- a/src/turbohtml/_c/serialize/document.c +++ b/src/turbohtml/_c/serialize/document.c @@ -89,8 +89,7 @@ static void serialize_node_line(sbuf *out, th_tree *tree, th_node *node, int dep } sbuf_putc(out, '\n'); /* attributes: each on its own deeper line, output in lexicographic name - order (the html5lib #document format sorts them). Only elements have - attributes; a text node's attr_count field holds a span offset. */ + order (the html5lib #document format sorts them). */ Py_ssize_t order[MAX_SORTED_ATTRS]; Py_ssize_t count = node->type == TH_NODE_ELEMENT ? (node->attr_count < MAX_SORTED_ATTRS ? node->attr_count : MAX_SORTED_ATTRS) : 0; diff --git a/src/turbohtml/_c/serialize/lossless.c b/src/turbohtml/_c/serialize/lossless.c index 48153135..b59a3bc5 100644 --- a/src/turbohtml/_c/serialize/lossless.c +++ b/src/turbohtml/_c/serialize/lossless.c @@ -48,20 +48,21 @@ static const th_serialize_opts lossless_opts = {TH_FMT_WHATWG, 0, 0, NULL, 0, 0, the parse left (character references and raw ampersands preserved), or the WHATWG escaping of its current code points once a read realized it or an edit replaced it. */ static void lossless_put_text(sbuf *out, th_tree *tree, th_node *node) { - if (node->text == NULL && node->text_len > 0) { - sbuf_put_source(out, tree, node->attr_count, node->attr_count + node->text_len); + if (text_is_span(node)) { + Py_ssize_t offset = text_span_offset(node); + sbuf_put_source(out, tree, offset, offset + node->text_len); } else { sbuf_put_text(out, node->text, node->text_len, 0, TH_FMT_WHATWG); } } /* Emit a raw-text element's child: its verbatim source span while it is still the - zero-copy slice the parse left (text NULL implies a positive-length span, since the - builder never inserts an empty text node), else its current code points literally + zero-copy slice the parse left, else its current code points literally (raw-text content is never escaped, so both paths emit the bytes unchanged). */ static void lossless_put_rawtext(sbuf *out, th_tree *tree, th_node *node) { - if (node->text == NULL) { - sbuf_put_source(out, tree, node->attr_count, node->attr_count + node->text_len); + if (text_is_span(node)) { + Py_ssize_t offset = text_span_offset(node); + sbuf_put_source(out, tree, offset, offset + node->text_len); } else { sbuf_put_ucs4(out, node->text, node->text_len); } diff --git a/src/turbohtml/_c/tokenizer/rewrite.c b/src/turbohtml/_c/tokenizer/rewrite.c index daa855b3..2f5307bf 100644 --- a/src/turbohtml/_c/tokenizer/rewrite.c +++ b/src/turbohtml/_c/tokenizer/rewrite.c @@ -1401,7 +1401,7 @@ PyObject *turbohtml_rewrite(PyObject *module, PyObject *args) { rw_ctx_clear(&ctx); /* GCOVR_EXCL_LINE: allocation-failure path */ return PyErr_NoMemory(); /* GCOVR_EXCL_LINE: allocation-failure path */ } - th_tok_set_options(ctx.sm, 0, 1); + th_tok_set_options(ctx.sm, 0, 1, 1); Py_ssize_t length = PyUnicode_GET_LENGTH(source); th_tok_feed(ctx.sm, PyUnicode_KIND(source), PyUnicode_DATA(source), length); th_tok_close(ctx.sm); diff --git a/src/turbohtml/_c/tokenizer/statemachine.c b/src/turbohtml/_c/tokenizer/statemachine.c index 6347ef14..bd5b82b2 100644 --- a/src/turbohtml/_c/tokenizer/statemachine.c +++ b/src/turbohtml/_c/tokenizer/statemachine.c @@ -208,6 +208,7 @@ struct th_tokenizer { int resolve_references; /* fold references into text (1) or split them into TH_CHARREF tokens (0) */ int capture_source; /* record the verbatim source span of markup tokens */ + int capture_attributes; /* retain tag attributes for emitted tokens */ int capture_locations; /* stamp the granular tag/attribute source spans */ int building; /* a markup token is mid-construction, so its source span's opening '<' must survive input compaction */ @@ -233,7 +234,9 @@ struct th_tokenizer { th_token tok; /* tag/comment/doctype under construction */ th_attr *attr; /* attribute under construction (points into tok.attrs) */ + th_attr_loc *attr_loc; th_attr oom_attr; /* writable sink for attribute data after an allocation failure */ + th_attr_loc oom_attr_loc; th_attr_slot *attr_seen; /* per-tag open-addressing set of kept attribute names */ Py_ssize_t attr_seen_cap; /* power-of-two slot count, 0 until the first attribute */ @@ -307,6 +310,8 @@ static void token_free(th_token *tok) { } PyMem_Free(tok->attrs); tok->attrs = NULL; + PyMem_Free(tok->attr_locs); + tok->attr_locs = NULL; tok->attr_cap = 0; tok->attr_count = 0; buf_free(&tok->public_id); @@ -321,13 +326,15 @@ th_tokenizer *th_tok_new(void) { memset(self, 0, sizeof(*self)); self->resolve_references = 1; self->capture_source = 0; + self->capture_attributes = 1; th_tok_reset(self); return self; } -void th_tok_set_options(th_tokenizer *self, int resolve_references, int capture_source) { +void th_tok_set_options(th_tokenizer *self, int resolve_references, int capture_source, int capture_attributes) { self->resolve_references = resolve_references; self->capture_source = capture_source; + self->capture_attributes = capture_attributes; } void th_tok_capture_locations(th_tokenizer *self, int on) { @@ -394,6 +401,7 @@ void th_tok_reset(th_tokenizer *self) { self->state = ST_DATA; self->oom = 0; self->attr = NULL; + self->attr_loc = NULL; buf_reset(&self->input); self->pos = 0; self->last_cr = 0; @@ -701,6 +709,11 @@ static void start_tag(th_tokenizer *self, int end_tag, Py_UCS4 first) { oom_attr sink keeps subsequent appends writing into valid storage until the sticky flag is reported. */ static void new_attr(th_tokenizer *self) { + if (!self->capture_attributes) { + self->attr = &self->oom_attr; + self->attr_loc = &self->oom_attr_loc; + return; + } th_token *tok = &self->tok; if (tok->attr_count == tok->attr_cap) { size_t cap; @@ -709,12 +722,14 @@ static void new_attr(th_tokenizer *self) { if (!grew) { /* GCOVR_EXCL_BR_LINE: size overflow needs a length no allocation could hold */ self->oom = 1; /* GCOVR_EXCL_LINE: size-overflow path, unreachable from a test */ self->attr = &self->oom_attr; /* GCOVR_EXCL_LINE: size-overflow path, unreachable from a test */ + self->attr_loc = &self->oom_attr_loc; /* GCOVR_EXCL_LINE: size-overflow path */ return; /* GCOVR_EXCL_LINE: size-overflow path, unreachable from a test */ } th_attr *grown = PyMem_Realloc(tok->attrs, bytes); if (grown == NULL) { /* GCOVR_EXCL_BR_LINE: allocation failure cannot be forced from a test */ self->oom = 1; /* GCOVR_EXCL_LINE: out-of-memory path, unreachable from a test */ self->attr = &self->oom_attr; /* GCOVR_EXCL_LINE: out-of-memory path, unreachable from a test */ + self->attr_loc = &self->oom_attr_loc; /* GCOVR_EXCL_LINE: out-of-memory path */ return; /* GCOVR_EXCL_LINE: allocation-failure path, unreachable from a test */ } for (Py_ssize_t index = tok->attr_cap; index < (Py_ssize_t)cap; index++) { @@ -723,6 +738,16 @@ static void new_attr(th_tokenizer *self) { grown[index].has_value = 0; } tok->attrs = grown; + if (self->capture_locations) { + th_attr_loc *grown_locs = PyMem_Realloc(tok->attr_locs, cap * sizeof(th_attr_loc)); + if (grown_locs == NULL) { /* GCOVR_EXCL_BR_LINE: allocation failure cannot be forced from a test */ + self->oom = 1; /* GCOVR_EXCL_LINE: out-of-memory path, unreachable from a test */ + self->attr = &self->oom_attr; /* GCOVR_EXCL_LINE: out-of-memory path */ + self->attr_loc = &self->oom_attr_loc; /* GCOVR_EXCL_LINE: out-of-memory path */ + return; /* GCOVR_EXCL_LINE: allocation-failure path */ + } + tok->attr_locs = grown_locs; + } tok->attr_cap = cap; } th_attr *attr = &tok->attrs[tok->attr_count++]; @@ -732,9 +757,11 @@ static void new_attr(th_tokenizer *self) { if (self->capture_locations) { /* the cursor sits on the first name code point (or the '=' a name may open with); the end span is filled as the name and any value complete */ - attr->name_line = self->line; - attr->name_col = self->col; - attr->name_off = self->pos; + th_attr_loc *loc = &tok->attr_locs[tok->attr_count - 1]; + loc->name_line = self->line; + loc->name_col = self->col; + loc->name_off = self->pos; + self->attr_loc = loc; } self->attr = attr; } @@ -817,9 +844,9 @@ static int attr_seen_probe(th_tokenizer *self, Py_ssize_t cur) { value delimiter, so the span ends past the last name or value code point. */ static void attr_end_here(th_tokenizer *self) { if (self->capture_locations) { - self->attr->end_line = self->line; - self->attr->end_col = self->col; - self->attr->end_off = self->pos; + self->attr_loc->end_line = self->line; + self->attr_loc->end_col = self->col; + self->attr_loc->end_off = self->pos; } } @@ -830,6 +857,9 @@ static void attr_end_here(th_tokenizer *self) { storage. The first attribute of each tag bumps the epoch, which clears the duplicate index in O(1) without walking the table. */ static void finish_attr_name(th_tokenizer *self) { + if (!self->capture_attributes) { + return; + } attr_end_here(self); /* a valueless attribute ends here; a value overwrites it */ th_token *tok = &self->tok; Py_ssize_t cur = tok->attr_count - 1; @@ -843,6 +873,7 @@ static void finish_attr_name(th_tokenizer *self) { tok_error(self, "duplicate-attribute"); tok->attr_count = cur; self->attr = &self->oom_attr; + self->attr_loc = &self->oom_attr_loc; } } diff --git a/src/turbohtml/_c/tokenizer/statemachine.h b/src/turbohtml/_c/tokenizer/statemachine.h index 497aab69..e3c64605 100644 --- a/src/turbohtml/_c/tokenizer/statemachine.h +++ b/src/turbohtml/_c/tokenizer/statemachine.h @@ -59,13 +59,13 @@ typedef struct { th_buf value; int has_value; /* 0 for a valueless attribute (maps to None in Python) */ uint32_t name_hash; /* FNV-1a of the name, for O(1) duplicate detection */ - /* Source span of the whole attribute -- the first name code point through the - last value code point (the name's end for a valueless one) -- filled only - while capture_locations is on. Line is 1-based, col 0-based, and off a - code-point index into the newline-normalized input, matching th_token.line/col. */ +} th_attr; + +/* Source span of one attribute, allocated only while capture_locations is on. */ +typedef struct { Py_ssize_t name_line, name_col, name_off; Py_ssize_t end_line, end_col, end_off; -} th_attr; +} th_attr_loc; /* One completed token. Buffers are owned and reused across tokens via reset; the Python layer copies what it needs out of them when it builds a Token. @@ -86,6 +86,7 @@ typedef struct { uint8_t tag_flags; th_buf text; /* TEXT run or COMMENT data */ th_attr *attrs; + th_attr_loc *attr_locs; Py_ssize_t attr_count; Py_ssize_t attr_cap; int self_closing; @@ -191,11 +192,10 @@ void th_tok_free(th_tokenizer *self); void th_tok_reset(th_tokenizer *self); /* Select the output surface. resolve_references off splits each character - reference in text into a TH_CHARREF token; capture_source on records the - verbatim source span of every markup token. Both default off-equivalent - (resolve on, capture off) and survive th_tok_reset, since they are - configuration rather than input state. */ -void th_tok_set_options(th_tokenizer *self, int resolve_references, int capture_source); + reference in text into a TH_CHARREF token; capture_source records each markup + token's verbatim span; capture_attributes retains tag attributes. The options + survive th_tok_reset because they are configuration rather than input state. */ +void th_tok_set_options(th_tokenizer *self, int resolve_references, int capture_source, int capture_attributes); /* Record the granular source spans parse5's sourceCodeLocationInfo exposes: each tag's end position (past its '>') and each attribute's name/value span. Off by diff --git a/src/turbohtml/_c/tokenizer/statemachine_run.h b/src/turbohtml/_c/tokenizer/statemachine_run.h index 4a6563ab..ce5a4f3b 100644 --- a/src/turbohtml/_c/tokenizer/statemachine_run.h +++ b/src/turbohtml/_c/tokenizer/statemachine_run.h @@ -7,6 +7,12 @@ #define TH_READ(i) ((Py_UCS4)((const TH_CHAR *)self->input.data)[(i)]) +static inline void TH_NAME(charref_push)(th_tokenizer *self, th_buf *dest, Py_UCS4 ch) { + if (dest != NULL) { + push(self, dest, ch); + } +} + /* Resolve a character reference starting at the '&' under self->pos, appending the decoded code points to dest. in_attr selects the legacy attribute rule (a named reference without a trailing ';' is left literal when followed by @@ -30,7 +36,7 @@ static Py_ssize_t TH_NAME(consume_charref)(th_tokenizer *self, th_buf *dest, int if (!self->eof) { return -1; } - push(self, dest, '&'); + TH_NAME(charref_push)(self, dest, '&'); return 1; } @@ -74,10 +80,10 @@ static Py_ssize_t TH_NAME(consume_charref)(th_tokenizer *self, th_buf *dest, int if (cursor == first_digit) { /* "&#" or "&#x" with no digits is not a reference: emit it literally */ tok_error_at(self, "absence-of-digits-in-numeric-character-reference", self->col + (hex ? 3 : 2)); - push(self, dest, '&'); - push(self, dest, '#'); + TH_NAME(charref_push)(self, dest, '&'); + TH_NAME(charref_push)(self, dest, '#'); if (hex) { - push(self, dest, TH_READ(after_amp + 1)); + TH_NAME(charref_push)(self, dest, TH_READ(after_amp + 1)); } return hex ? 3 : 2; } @@ -94,13 +100,13 @@ static Py_ssize_t TH_NAME(consume_charref)(th_tokenizer *self, th_buf *dest, int Py_UCS4 replacement; if (num == 0) { tok_error_at(self, "null-character-reference", end_col); - push(self, dest, REPLACEMENT); + TH_NAME(charref_push)(self, dest, REPLACEMENT); } else if (overflow || num > 0x10FFFF) { tok_error_at(self, "character-reference-outside-unicode-range", end_col); - push(self, dest, REPLACEMENT); + TH_NAME(charref_push)(self, dest, REPLACEMENT); } else if (num >= 0xD800 && num <= 0xDFFF) { tok_error_at(self, "surrogate-character-reference", end_col); - push(self, dest, REPLACEMENT); + TH_NAME(charref_push)(self, dest, REPLACEMENT); } else { if ((num >= 0xFDD0 && num <= 0xFDEF) || (num & 0xFFFE) == 0xFFFE) { tok_error_at(self, "noncharacter-character-reference", end_col); @@ -110,9 +116,9 @@ static Py_ssize_t TH_NAME(consume_charref)(th_tokenizer *self, th_buf *dest, int tok_error_at(self, "control-character-reference", end_col); } if (charref_find_invalid(num, &replacement)) { - push(self, dest, replacement); + TH_NAME(charref_push)(self, dest, replacement); } else { - push(self, dest, num); + TH_NAME(charref_push)(self, dest, num); } } if (is_reference != NULL) { @@ -123,7 +129,7 @@ static Py_ssize_t TH_NAME(consume_charref)(th_tokenizer *self, th_buf *dest, int if (!is_ascii_alpha(first) && !(first >= '0' && first <= '9')) { /* "&" not followed by '#' or a name start is a literal ampersand */ - push(self, dest, '&'); + TH_NAME(charref_push)(self, dest, '&'); return 1; } @@ -191,9 +197,9 @@ static Py_ssize_t TH_NAME(consume_charref)(th_tokenizer *self, th_buf *dest, int if (named_semicolon) { tok_error_at(self, "unknown-named-character-reference", self->col + (ambiguous_end - amp)); } - push(self, dest, '&'); + TH_NAME(charref_push)(self, dest, '&'); for (int index = 0; index < name_len; index++) { - push(self, dest, chars[index]); + TH_NAME(charref_push)(self, dest, chars[index]); } return 1 + name_len; } @@ -202,9 +208,9 @@ static Py_ssize_t TH_NAME(consume_charref)(th_tokenizer *self, th_buf *dest, int Py_UCS4 after = (match_len < name_len) ? chars[match_len] : (cursor < len ? TH_READ(cursor) : 0); if (after == '=' || is_ascii_alpha(after) || (after >= '0' && after <= '9')) { /* legacy rule: leave the reference literal inside an attribute */ - push(self, dest, '&'); + TH_NAME(charref_push)(self, dest, '&'); for (int index = 0; index < name_len; index++) { - push(self, dest, chars[index]); + TH_NAME(charref_push)(self, dest, chars[index]); } return 1 + name_len; } @@ -213,13 +219,13 @@ static Py_ssize_t TH_NAME(consume_charref)(th_tokenizer *self, th_buf *dest, int if (!match_semicolon) { tok_error_at(self, "missing-semicolon-after-character-reference", self->col + 1 + match_len); } - push(self, dest, entity->cp0); + TH_NAME(charref_push)(self, dest, entity->cp0); if (entity->cp1) { - push(self, dest, entity->cp1); + TH_NAME(charref_push)(self, dest, entity->cp1); } /* characters consumed past the matched name are emitted literally */ for (int index = match_len; index < name_len; index++) { - push(self, dest, chars[index]); + TH_NAME(charref_push)(self, dest, chars[index]); } if (is_reference != NULL) { *is_reference = 1; @@ -1033,7 +1039,9 @@ static enum run_result TH_NAME(run)(th_tokenizer *self) { new_attr(self); if (ch == '=') { tok_error(self, "unexpected-equals-sign-before-attribute-name"); - push(self, &self->attr->name, ch); + if (self->capture_attributes) { + push(self, &self->attr->name, ch); + } CONSUME(); } self->state = ST_ATTR_NAME; @@ -1056,7 +1064,9 @@ static enum run_result TH_NAME(run)(th_tokenizer *self) { } else if (ch == 0) { tok_error(self, "unexpected-null-character"); } - push(self, &self->attr->name, ch == 0 ? REPLACEMENT : lower_ascii(ch)); + if (self->capture_attributes) { + push(self, &self->attr->name, ch == 0 ? REPLACEMENT : lower_ascii(ch)); + } CONSUME(); continue; @@ -1092,7 +1102,9 @@ static enum run_result TH_NAME(run)(th_tokenizer *self) { CONSUME(); continue; } - self->attr->has_value = 1; + if (self->capture_attributes) { + self->attr->has_value = 1; + } if (!at_eof && ch == '"') { CONSUME(); self->state = ST_ATTR_VALUE_DQ; @@ -1120,7 +1132,9 @@ static enum run_result TH_NAME(run)(th_tokenizer *self) { /* bulk-copy the ordinary value run; '\n' is a stop so line/col stay accurate, and &/" /NUL keep their dedicated handling */ Py_ssize_t stop = TH_SCAN(self, self->pos, '"', '&', '\n', 0); - buf_append_input(self, &self->attr->value, self->pos, stop - self->pos); + if (self->capture_attributes) { + buf_append_input(self, &self->attr->value, self->pos, stop - self->pos); + } self->col += stop - self->pos; self->pos = stop; continue; @@ -1132,7 +1146,9 @@ static enum run_result TH_NAME(run)(th_tokenizer *self) { continue; } if (ch == '&') { - Py_ssize_t consumed = TH_NAME(consume_charref)(self, &self->attr->value, 1, NULL); + Py_ssize_t consumed = TH_NAME(consume_charref)( + self, self->capture_attributes ? &self->attr->value : NULL, 1, NULL + ); if (consumed == -1) { return RUN_NEED_MORE; } @@ -1143,7 +1159,9 @@ static enum run_result TH_NAME(run)(th_tokenizer *self) { if (ch == 0) { tok_error(self, "unexpected-null-character"); } - push(self, &self->attr->value, ch == 0 ? REPLACEMENT : ch); + if (self->capture_attributes) { + push(self, &self->attr->value, ch == 0 ? REPLACEMENT : ch); + } CONSUME(); continue; @@ -1153,7 +1171,9 @@ static enum run_result TH_NAME(run)(th_tokenizer *self) { } if (ch != '\'' && ch != '&' && ch != '\n' && ch != 0) { Py_ssize_t stop = TH_SCAN(self, self->pos, '\'', '&', '\n', 0); - buf_append_input(self, &self->attr->value, self->pos, stop - self->pos); + if (self->capture_attributes) { + buf_append_input(self, &self->attr->value, self->pos, stop - self->pos); + } self->col += stop - self->pos; self->pos = stop; continue; @@ -1165,7 +1185,9 @@ static enum run_result TH_NAME(run)(th_tokenizer *self) { continue; } if (ch == '&') { - Py_ssize_t consumed = TH_NAME(consume_charref)(self, &self->attr->value, 1, NULL); + Py_ssize_t consumed = TH_NAME(consume_charref)( + self, self->capture_attributes ? &self->attr->value : NULL, 1, NULL + ); if (consumed == -1) { return RUN_NEED_MORE; } @@ -1176,7 +1198,9 @@ static enum run_result TH_NAME(run)(th_tokenizer *self) { if (ch == 0) { tok_error(self, "unexpected-null-character"); } - push(self, &self->attr->value, ch == 0 ? REPLACEMENT : ch); + if (self->capture_attributes) { + push(self, &self->attr->value, ch == 0 ? REPLACEMENT : ch); + } CONSUME(); continue; @@ -1191,7 +1215,9 @@ static enum run_result TH_NAME(run)(th_tokenizer *self) { continue; } if (ch == '&') { - Py_ssize_t consumed = TH_NAME(consume_charref)(self, &self->attr->value, 1, NULL); + Py_ssize_t consumed = TH_NAME(consume_charref)( + self, self->capture_attributes ? &self->attr->value : NULL, 1, NULL + ); if (consumed == -1) { return RUN_NEED_MORE; } @@ -1210,7 +1236,9 @@ static enum run_result TH_NAME(run)(th_tokenizer *self) { } else if (ch == 0) { tok_error(self, "unexpected-null-character"); } - push(self, &self->attr->value, ch == 0 ? REPLACEMENT : ch); + if (self->capture_attributes) { + push(self, &self->attr->value, ch == 0 ? REPLACEMENT : ch); + } CONSUME(); continue; diff --git a/src/turbohtml/_c/tokenizer/token.c b/src/turbohtml/_c/tokenizer/token.c index 0497a79b..0a4ad8a1 100644 --- a/src/turbohtml/_c/tokenizer/token.c +++ b/src/turbohtml/_c/tokenizer/token.c @@ -64,6 +64,7 @@ static char *token_pack(th_token *dst, const th_token *src) { } *dst = *src; dst->attr_cap = src->attr_count; + dst->attr_locs = NULL; char *cursor = arena; dst->attrs = (th_attr *)cursor; cursor += src->attr_count * (Py_ssize_t)sizeof(th_attr); diff --git a/src/turbohtml/_c/tokenizer/tokenizer.c b/src/turbohtml/_c/tokenizer/tokenizer.c index e64cdcc2..3b7664f6 100644 --- a/src/turbohtml/_c/tokenizer/tokenizer.c +++ b/src/turbohtml/_c/tokenizer/tokenizer.c @@ -142,12 +142,14 @@ static PyType_Spec iter_spec = { }; static PyObject *tokenizer_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { - static char *keywords[] = {"resolve_references", "capture_source", NULL}; + static char *keywords[] = {"resolve_references", "capture_source", "capture_attributes", NULL}; int resolve_references = 1; int capture_source = 0; - /* a streaming tokenizer starts empty; the two options are keyword-only and + int capture_attributes = 1; + /* a streaming tokenizer starts empty; its options are keyword-only and reject any other positional or keyword argument */ - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|$pp:Tokenizer", keywords, &resolve_references, &capture_source)) { + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|$ppp:Tokenizer", keywords, &resolve_references, &capture_source, + &capture_attributes)) { return NULL; } TokenizerObject *self = (TokenizerObject *)type->tp_alloc(type, 0); @@ -161,7 +163,7 @@ static PyObject *tokenizer_new(PyTypeObject *type, PyObject *args, PyObject *kwd Py_DECREF(self); /* GCOVR_EXCL_LINE: allocation-failure path */ return PyErr_NoMemory(); /* GCOVR_EXCL_LINE: allocation-failure path */ } - th_tok_set_options(self->sm, resolve_references, capture_source); + th_tok_set_options(self->sm, resolve_references, capture_source, capture_attributes); return (PyObject *)self; } @@ -270,7 +272,7 @@ static PyMethodDef tokenizer_methods[] = { {NULL, NULL, 0, NULL}, }; -PyDoc_STRVAR(tokenizer_doc, "Tokenizer(*, resolve_references=True, capture_source=False)\n--\n\n" +PyDoc_STRVAR(tokenizer_doc, "Tokenizer(*, resolve_references=True, capture_source=False, capture_attributes=True)\n--\n\n" "Streaming HTML tokenizer. Feed markup with feed() and iterate the\n" "returned iterators; call close() at the end, or use the tokenizer as a\n" "context manager so leaving the with block signals end of input, then\n" @@ -281,7 +283,9 @@ PyDoc_STRVAR(tokenizer_doc, "Tokenizer(*, resolve_references=True, capture_sourc " token (its data the resolved value, its source the verbatim reference).\n" " Attribute-value references are always resolved.\n" ":param capture_source: record each markup token's verbatim source slice,\n" - " available as Token.source."); + " available as Token.source.\n" + ":param capture_attributes: retain tag attributes; disable this when only tag names and\n" + " text are needed to skip attribute storage and copying."); static PyType_Slot tokenizer_slots[] = { {Py_tp_doc, (void *)tokenizer_doc}, @@ -302,12 +306,13 @@ static PyType_Spec tokenizer_spec = { // NOLINTNEXTLINE(misc-use-internal-linkage): declared in core/common.h and called from core/module.c PyObject *turbohtml_tokenize(PyObject *module, PyObject *args, PyObject *kwargs) { - static char *keywords[] = {"", "resolve_references", "capture_source", NULL}; + static char *keywords[] = {"", "resolve_references", "capture_source", "capture_attributes", NULL}; PyObject *arg; int resolve_references = 1; int capture_source = 0; - if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|$pp:tokenize", keywords, &arg, &resolve_references, - &capture_source)) { + int capture_attributes = 1; + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|$ppp:tokenize", keywords, &arg, &resolve_references, + &capture_source, &capture_attributes)) { return NULL; } if (!PyUnicode_Check(arg)) { @@ -320,7 +325,7 @@ PyObject *turbohtml_tokenize(PyObject *module, PyObject *args, PyObject *kwargs) return NULL; /* GCOVR_EXCL_LINE: allocation-failure path */ } th_tokenizer *sm = ((TokenizerObject *)tokenizer)->sm; - th_tok_set_options(sm, resolve_references, capture_source); + th_tok_set_options(sm, resolve_references, capture_source, capture_attributes); Py_ssize_t length = PyUnicode_GET_LENGTH(arg); if (PyUnicode_FindChar(arg, '\r', 0, length, 1) == -1) { /* nothing to normalize: borrow the string's storage instead of diff --git a/src/turbohtml/_c/tokenizer/xml.c b/src/turbohtml/_c/tokenizer/xml.c index 9b3f1a2d..29ceba41 100644 --- a/src/turbohtml/_c/tokenizer/xml.c +++ b/src/turbohtml/_c/tokenizer/xml.c @@ -501,7 +501,7 @@ static int consume_text(xml_parser *parser) { if (text == NULL) { /* GCOVR_EXCL_BR_LINE: allocation failure cannot be forced from a test */ return -1; /* GCOVR_EXCL_LINE: allocation-failure path */ } - text->attr_count = start; /* zero-copy span: input[start .. start+text_len] */ + text_set_span(text, start); text->text_len = scan - start; node_append(current(parser), text); parser->pos = scan; diff --git a/src/turbohtml/_stubs/tokenizer.pyi b/src/turbohtml/_stubs/tokenizer.pyi index 505f8053..fdc746d9 100644 --- a/src/turbohtml/_stubs/tokenizer.pyi +++ b/src/turbohtml/_stubs/tokenizer.pyi @@ -41,7 +41,9 @@ class Token: @final class Tokenizer: - def __init__(self, *, resolve_references: bool = ..., capture_source: bool = ...) -> None: ... + def __init__( + self, *, resolve_references: bool = ..., capture_source: bool = ..., capture_attributes: bool = ... + ) -> None: ... def feed(self, data: str) -> Iterator[Token]: ... def close(self) -> Iterator[Token]: ... def reset(self) -> None: ... @@ -49,7 +51,14 @@ class Tokenizer: def __exit__(self, *exc: object) -> None: ... def __iter__(self) -> Iterator[Token]: ... -def tokenize(s: str, /, *, resolve_references: bool = ..., capture_source: bool = ...) -> Iterator[Token]: ... +def tokenize( + s: str, + /, + *, + resolve_references: bool = ..., + capture_source: bool = ..., + capture_attributes: bool = ..., +) -> Iterator[Token]: ... @final class _RewriteHandle: diff --git a/tests/build/test_build.py b/tests/build/test_build.py index c2c7ebb5..700c4213 100644 --- a/tests/build/test_build.py +++ b/tests/build/test_build.py @@ -154,6 +154,10 @@ def test_constructor_accepts_a_children_tuple() -> None: assert Element("div", None, (Text("a"), Text("b"))).serialize() == "
ab
" +def test_constructor_lowercases_a_custom_tag() -> None: + assert Element("CUSTOM-TAG").tag == "custom-tag" + + def test_constructor_children_must_be_iterable() -> None: with pytest.raises(TypeError, match="children must be iterable"): Element("div", None, 42) # ty: ignore[invalid-argument-type] # children must be an iterable of nodes diff --git a/tests/tokenizer/test_tokenizer.py b/tests/tokenizer/test_tokenizer.py index f2b1151f..d9a4039a 100644 --- a/tests/tokenizer/test_tokenizer.py +++ b/tests/tokenizer/test_tokenizer.py @@ -95,6 +95,31 @@ def test_tag_attrs(document: str, tag_name: str, attrs: list[tuple[str, str | No assert tag.attrs == attrs +@pytest.mark.parametrize("mode", ["whole", "streaming"]) +def test_capture_attributes_can_be_disabled(mode: str) -> None: + document = 'text' + if mode == "streaming": + tokenizer = Tokenizer(capture_attributes=False) + tokens = [token for char in document for token in tokenizer.feed(char)] + tokens += list(tokenizer.close()) + else: + tokens = list(tokenize(document, capture_attributes=False)) + + start, text, end = tokens + assert (start.tag, start.attrs, start.attr("href")) == ("a", [], None) + assert (text.data, end.tag) == ("text", "a") + + +@pytest.mark.parametrize( + "text", + [pytest.param("x", id="ascii"), pytest.param("ő", id="ucs2"), pytest.param("🎉", id="ucs4")], +) +def test_capture_attributes_disabled_handles_every_value_form(text: str) -> None: + document = f"{text}" + start, content, end = tokenize(document, capture_attributes=False) + assert (start.attrs, content.data, end.tag) == ([], text, "a") + + def test_non_latin1_tag_name() -> None: tokens = list(tokenize("x")) assert [token.tag for token in tokens if token.tag] == ["xmő", "xmő"] diff --git a/tools/bench/operations.py b/tools/bench/operations.py index 96bbd661..9066b490 100644 --- a/tools/bench/operations.py +++ b/tools/bench/operations.py @@ -847,6 +847,7 @@ def _normalize_cases() -> tuple[tuple[str, object], ...]: ("meta_refresh / get_meta_refresh", ("refresh", _URL_HINT_HTML)), ), "htmlparser": _readpath_cases, + "sax": _readpath_cases, "treebuild": _readpath_cases, "rewrite": _readpath_cases, "path": _readpath_cases, diff --git a/tools/generate_tags.py b/tools/generate_tags.py index 97baadd3..d73c02de 100644 --- a/tools/generate_tags.py +++ b/tools/generate_tags.py @@ -218,9 +218,11 @@ def generate(out_path: Path) -> None: enum_lines = "\n".join(f" {atom}," for atom in atoms) table_lines = [] + wide_names = [] for name, atom in zip(names, atoms, strict=True): flags = " | ".join(flag for flag, members in CATEGORY_FLAGS.items() if name in members) or "0" - table_lines.append(f' {{"{name}", {len(name)}u, {atom}, {flags}}},') + table_lines.append(f' {{"{name}", {len(name)}u, {atom}, {flags}, {len(wide_names)}u}},') + wide_names.extend(map(ord, name)) # Names are sorted, so entries sharing a first byte are contiguous. # first_index[b] holds the offset of the first entry whose name starts with a @@ -256,11 +258,18 @@ def generate(out_path: Path) -> None: " uint8_t name_len;\n" " uint16_t atom;\n" " uint8_t flags;\n" + " uint16_t wide_offset;\n" "} th_tag_entry;\n\n" f"static const int th_tag_count = {len(names)};\n" "static const th_tag_entry th_tag_table[] = {\n" f"{chr(10).join(table_lines)}\n" "};\n\n" + "static const uint32_t th_tag_wide_names[] = {\n" + f" {', '.join(f'{char}u' for char in wide_names)}\n" + "};\n\n" + "static inline const uint32_t *th_tag_wide_name(uint16_t atom) {\n" + " return th_tag_wide_names + th_tag_table[atom - 1].wide_offset;\n" + "}\n\n" "/* th_tag_first[c] is the first table index whose name starts with a byte\n" " >= c, so the entries beginning with byte c are\n" " [th_tag_first[c], th_tag_first[c + 1]). */\n" From b9657c535863fd68f2fcf64859ca9f3f6d4d057f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 11 Jul 2026 07:09:26 -0700 Subject: [PATCH 2/6] =?UTF-8?q?=F0=9F=93=9D=20docs(changelog):=20backfill?= =?UTF-8?q?=201.3.1=20fixes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/changelog/634.bugfix.rst | 1 + docs/changelog/635.bugfix.rst | 1 + docs/changelog/636.bugfix.rst | 2 ++ docs/changelog/637.bugfix.rst | 1 + docs/changelog/638.bugfix.rst | 2 ++ 5 files changed, 7 insertions(+) create mode 100644 docs/changelog/634.bugfix.rst create mode 100644 docs/changelog/635.bugfix.rst create mode 100644 docs/changelog/636.bugfix.rst create mode 100644 docs/changelog/637.bugfix.rst create mode 100644 docs/changelog/638.bugfix.rst diff --git a/docs/changelog/634.bugfix.rst b/docs/changelog/634.bugfix.rst new file mode 100644 index 00000000..7dd860b8 --- /dev/null +++ b/docs/changelog/634.bugfix.rst @@ -0,0 +1 @@ +Keep a :class:`turbohtml.Node` hash stable across cross-document adoption so sets and dictionaries can still find it. diff --git a/docs/changelog/635.bugfix.rst b/docs/changelog/635.bugfix.rst new file mode 100644 index 00000000..b438fa38 --- /dev/null +++ b/docs/changelog/635.bugfix.rst @@ -0,0 +1 @@ +Copy source subtrees from one state during DOM adoption, Range operations, and XSLT under free-threaded Python. diff --git a/docs/changelog/636.bugfix.rst b/docs/changelog/636.bugfix.rst new file mode 100644 index 00000000..ee40af90 --- /dev/null +++ b/docs/changelog/636.bugfix.rst @@ -0,0 +1,2 @@ +Recognize XSLT instructions by namespace URI and local name, including stylesheets with a default XSLT namespace or +rebound prefix. diff --git a/docs/changelog/637.bugfix.rst b/docs/changelog/637.bugfix.rst new file mode 100644 index 00000000..1cbede19 --- /dev/null +++ b/docs/changelog/637.bugfix.rst @@ -0,0 +1 @@ +Raise :exc:`ValueError` with the import chain for circular ``xsl:import`` references. diff --git a/docs/changelog/638.bugfix.rst b/docs/changelog/638.bugfix.rst new file mode 100644 index 00000000..e350b0bd --- /dev/null +++ b/docs/changelog/638.bugfix.rst @@ -0,0 +1,2 @@ +Return :meth:`turbohtml.query.Query.find` results in document order across connected roots; retain root input order +across disconnected trees. From 0964758e9ba3da9a55b131a0e8d537d9d24bff65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 11 Jul 2026 07:09:27 -0700 Subject: [PATCH 3/6] =?UTF-8?q?=F0=9F=A7=AA=20test(query):=20avoid=20liter?= =?UTF-8?q?al=20identity=20warning?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/dom/test_tree_index_cache.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/dom/test_tree_index_cache.py b/tests/dom/test_tree_index_cache.py index 278c82eb..b2921062 100644 --- a/tests/dom/test_tree_index_cache.py +++ b/tests/dom/test_tree_index_cache.py @@ -102,10 +102,11 @@ def test_cache_hit_on_equal_distinct_string() -> None: # the cache matches by identity first, then by string value, so a second call # with an equal but distinct selector object reuses the first compile doc = parse(_DOC) - assert [element.text for element in doc.select("div p")] == ["nested"] + selector = "div p" + assert [element.text for element in doc.select(selector)] == ["nested"] descendant = "p" rebuilt = f"div {descendant}" # an f-string is built at runtime: equal content, a distinct object - assert rebuilt is not "div p" # noqa: F632 # a content match, not an identity match, is the point + assert rebuilt is not selector assert [element.text for element in doc.select(rebuilt)] == ["nested"] From 6651100df2d80bde7d2041c5939d9f3b1455ba59 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 11 Jul 2026 14:11:08 +0000 Subject: [PATCH 4/6] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/turbohtml/_c/core/module.c | 20 +++---- src/turbohtml/_c/data/tag_atom.h | 54 +++++++++++++++++-- src/turbohtml/_c/dom/element.c | 6 +-- src/turbohtml/_c/dom/tree.c | 10 ++-- src/turbohtml/_c/dom/tree_internal.h | 3 +- src/turbohtml/_c/tokenizer/statemachine.c | 16 +++--- src/turbohtml/_c/tokenizer/statemachine_run.h | 15 +++--- src/turbohtml/_c/tokenizer/tokenizer.c | 29 +++++----- 8 files changed, 97 insertions(+), 56 deletions(-) diff --git a/src/turbohtml/_c/core/module.c b/src/turbohtml/_c/core/module.c index 1ef83c96..e1705863 100644 --- a/src/turbohtml/_c/core/module.c +++ b/src/turbohtml/_c/core/module.c @@ -65,16 +65,16 @@ PyDoc_STRVAR(markup_soft_str_doc, "soft_str(s, /)\n--\n\n" PyDoc_STRVAR(tokenize_doc, "tokenize(s, /, *, resolve_references=True, capture_source=False, capture_attributes=True)\n--\n\n" - "Tokenize a whole HTML string following the WHATWG tokenization algorithm.\n\n" - ":param s: the HTML to tokenize.\n" - ":param resolve_references: fold character references into the surrounding text\n" - " run; when False each one becomes its own CHARACTER_REFERENCE token.\n" - ":param capture_source: record each markup token's verbatim source on\n" - " Token.source.\n" - ":param capture_attributes: retain tag attributes; disable this when only tag names and\n" - " text are needed.\n" - ":returns: an iterator of Token objects in document order.\n" - ":raises TypeError: if s is not a str."); + "Tokenize a whole HTML string following the WHATWG tokenization algorithm.\n\n" + ":param s: the HTML to tokenize.\n" + ":param resolve_references: fold character references into the surrounding text\n" + " run; when False each one becomes its own CHARACTER_REFERENCE token.\n" + ":param capture_source: record each markup token's verbatim source on\n" + " Token.source.\n" + ":param capture_attributes: retain tag attributes; disable this when only tag names and\n" + " text are needed.\n" + ":returns: an iterator of Token objects in document order.\n" + ":raises TypeError: if s is not a str."); PyDoc_STRVAR(parse_doc, "parse(markup, *, encoding=None, strict=False, detect_encoding=False, positions=True, " "source_locations=False, scripting=False, allow_declarative_shadow_roots=True)\n" diff --git a/src/turbohtml/_c/data/tag_atom.h b/src/turbohtml/_c/data/tag_atom.h index 9b7ddc03..79096229 100644 --- a/src/turbohtml/_c/data/tag_atom.h +++ b/src/turbohtml/_c/data/tag_atom.h @@ -336,8 +336,44 @@ static const th_tag_entry th_tag_table[] = { }; static const uint32_t th_tag_wide_names[] = { - 97u, 97u, 98u, 98u, 114u, 97u, 100u, 100u, 114u, 101u, 115u, 115u, 97u, 110u, 110u, 111u, 116u, 97u, 116u, 105u, 111u, 110u, 45u, 120u, 109u, 108u, 97u, 112u, 112u, 108u, 101u, 116u, 97u, 114u, 101u, 97u, 97u, 114u, 116u, 105u, 99u, 108u, 101u, 97u, 115u, 105u, 100u, 101u, 97u, 117u, 100u, 105u, 111u, 98u, 98u, 97u, 115u, 101u, 98u, 97u, 115u, 101u, 102u, 111u, 110u, 116u, 98u, 100u, 105u, 98u, 100u, 111u, 98u, 103u, 115u, 111u, 117u, 110u, 100u, 98u, 105u, 103u, 98u, 108u, 111u, 99u, 107u, 113u, 117u, 111u, 116u, 101u, 98u, 111u, 100u, 121u, 98u, 114u, 98u, 117u, 116u, 116u, 111u, 110u, 99u, 97u, 110u, 118u, 97u, 115u, 99u, 97u, 112u, 116u, 105u, 111u, 110u, 99u, 101u, 110u, 116u, 101u, 114u, 99u, 105u, 116u, 101u, 99u, 111u, 100u, 101u, 99u, 111u, 108u, 99u, 111u, 108u, 103u, 114u, 111u, 117u, 112u, 99u, 111u, 109u, 109u, 97u, 110u, 100u, 100u, 97u, 116u, 97u, 100u, 97u, 116u, 97u, 108u, 105u, 115u, 116u, 100u, 100u, 100u, 101u, 108u, 100u, 101u, 115u, 99u, 100u, 101u, 116u, 97u, 105u, 108u, 115u, 100u, 102u, 110u, 100u, 105u, 97u, 108u, 111u, 103u, 100u, 105u, 114u, 100u, 105u, 118u, 100u, 108u, 100u, 116u, 101u, 109u, 101u, 109u, 98u, 101u, 100u, 102u, 105u, 101u, 108u, 100u, 115u, 101u, 116u, 102u, 105u, 103u, 99u, 97u, 112u, 116u, 105u, 111u, 110u, 102u, 105u, 103u, 117u, 114u, 101u, 102u, 111u, 110u, 116u, 102u, 111u, 111u, 116u, 101u, 114u, 102u, 111u, 114u, 101u, 105u, 103u, 110u, 111u, 98u, 106u, 101u, 99u, 116u, 102u, 111u, 114u, 109u, 102u, 114u, 97u, 109u, 101u, 102u, 114u, 97u, 109u, 101u, 115u, 101u, 116u, 103u, 104u, 49u, 104u, 50u, 104u, 51u, 104u, 52u, 104u, 53u, 104u, 54u, 104u, 101u, 97u, 100u, 104u, 101u, 97u, 100u, 101u, 114u, 104u, 103u, 114u, 111u, 117u, 112u, 104u, 114u, 104u, 116u, 109u, 108u, 105u, 105u, 102u, 114u, 97u, 109u, 101u, 105u, 109u, 97u, 103u, 101u, 105u, 109u, 103u, 105u, 110u, 112u, 117u, 116u, 105u, 110u, 115u, 105u, 115u, 105u, 110u, 100u, 101u, 120u, 107u, 98u, 100u, 107u, 101u, 121u, 103u, 101u, 110u, 108u, 97u, 98u, 101u, 108u, 108u, 101u, 103u, 101u, 110u, 100u, 108u, 105u, 108u, 105u, 110u, 107u, 108u, 105u, 115u, 116u, 105u, 110u, 103u, 109u, 97u, 105u, 110u, 109u, 97u, 108u, 105u, 103u, 110u, 109u, 97u, 114u, 107u, 109u, 97u, 112u, 109u, 97u, 114u, 107u, 109u, 97u, 114u, 113u, 117u, 101u, 101u, 109u, 97u, 116u, 104u, 109u, 101u, 110u, 117u, 109u, 101u, 116u, 97u, 109u, 101u, 116u, 101u, 114u, 109u, 103u, 108u, 121u, 112u, 104u, 109u, 105u, 109u, 110u, 109u, 111u, 109u, 115u, 109u, 116u, 101u, 120u, 116u, 110u, 97u, 118u, 110u, 111u, 98u, 114u, 110u, 111u, 101u, 109u, 98u, 101u, 100u, 110u, 111u, 102u, 114u, 97u, 109u, 101u, 115u, 110u, 111u, 115u, 99u, 114u, 105u, 112u, 116u, 111u, 98u, 106u, 101u, 99u, 116u, 111u, 108u, 111u, 112u, 116u, 103u, 114u, 111u, 117u, 112u, 111u, 112u, 116u, 105u, 111u, 110u, 111u, 117u, 116u, 112u, 117u, 116u, 112u, 112u, 97u, 114u, 97u, 109u, 112u, 97u, 116u, 104u, 112u, 105u, 99u, 116u, 117u, 114u, 101u, 112u, 108u, 97u, 105u, 110u, 116u, 101u, 120u, 116u, 112u, 114u, 101u, 112u, 114u, 111u, 103u, 114u, 101u, 115u, 115u, 113u, 114u, 98u, 114u, 112u, 114u, 116u, 114u, 116u, 99u, 114u, 117u, 98u, 121u, 115u, 115u, 97u, 109u, 112u, 115u, 99u, 114u, 105u, 112u, 116u, 115u, 101u, 97u, 114u, 99u, 104u, 115u, 101u, 99u, 116u, 105u, 111u, 110u, 115u, 101u, 108u, 101u, 99u, 116u, 115u, 101u, 108u, 101u, 99u, 116u, 101u, 100u, 99u, 111u, 110u, 116u, 101u, 110u, 116u, 115u, 108u, 111u, 116u, 115u, 109u, 97u, 108u, 108u, 115u, 111u, 117u, 114u, 99u, 101u, 115u, 112u, 97u, 110u, 115u, 116u, 114u, 105u, 107u, 101u, 115u, 116u, 114u, 111u, 110u, 103u, 115u, 116u, 121u, 108u, 101u, 115u, 117u, 98u, 115u, 117u, 109u, 109u, 97u, 114u, 121u, 115u, 117u, 112u, 115u, 118u, 103u, 116u, 97u, 98u, 108u, 101u, 116u, 98u, 111u, 100u, 121u, 116u, 100u, 116u, 101u, 109u, 112u, 108u, 97u, 116u, 101u, 116u, 101u, 120u, 116u, 97u, 114u, 101u, 97u, 116u, 102u, 111u, 111u, 116u, 116u, 104u, 116u, 104u, 101u, 97u, 100u, 116u, 105u, 109u, 101u, 116u, 105u, 116u, 108u, 101u, 116u, 114u, 116u, 114u, 97u, 99u, 107u, 116u, 116u, 117u, 117u, 108u, 118u, 97u, 114u, 118u, 105u, 100u, 101u, 111u, 119u, 98u, 114u, 120u, 109u, 112u -}; + 97u, 97u, 98u, 98u, 114u, 97u, 100u, 100u, 114u, 101u, 115u, 115u, 97u, 110u, 110u, 111u, 116u, 97u, 116u, + 105u, 111u, 110u, 45u, 120u, 109u, 108u, 97u, 112u, 112u, 108u, 101u, 116u, 97u, 114u, 101u, 97u, 97u, 114u, + 116u, 105u, 99u, 108u, 101u, 97u, 115u, 105u, 100u, 101u, 97u, 117u, 100u, 105u, 111u, 98u, 98u, 97u, 115u, + 101u, 98u, 97u, 115u, 101u, 102u, 111u, 110u, 116u, 98u, 100u, 105u, 98u, 100u, 111u, 98u, 103u, 115u, 111u, + 117u, 110u, 100u, 98u, 105u, 103u, 98u, 108u, 111u, 99u, 107u, 113u, 117u, 111u, 116u, 101u, 98u, 111u, 100u, + 121u, 98u, 114u, 98u, 117u, 116u, 116u, 111u, 110u, 99u, 97u, 110u, 118u, 97u, 115u, 99u, 97u, 112u, 116u, + 105u, 111u, 110u, 99u, 101u, 110u, 116u, 101u, 114u, 99u, 105u, 116u, 101u, 99u, 111u, 100u, 101u, 99u, 111u, + 108u, 99u, 111u, 108u, 103u, 114u, 111u, 117u, 112u, 99u, 111u, 109u, 109u, 97u, 110u, 100u, 100u, 97u, 116u, + 97u, 100u, 97u, 116u, 97u, 108u, 105u, 115u, 116u, 100u, 100u, 100u, 101u, 108u, 100u, 101u, 115u, 99u, 100u, + 101u, 116u, 97u, 105u, 108u, 115u, 100u, 102u, 110u, 100u, 105u, 97u, 108u, 111u, 103u, 100u, 105u, 114u, 100u, + 105u, 118u, 100u, 108u, 100u, 116u, 101u, 109u, 101u, 109u, 98u, 101u, 100u, 102u, 105u, 101u, 108u, 100u, 115u, + 101u, 116u, 102u, 105u, 103u, 99u, 97u, 112u, 116u, 105u, 111u, 110u, 102u, 105u, 103u, 117u, 114u, 101u, 102u, + 111u, 110u, 116u, 102u, 111u, 111u, 116u, 101u, 114u, 102u, 111u, 114u, 101u, 105u, 103u, 110u, 111u, 98u, 106u, + 101u, 99u, 116u, 102u, 111u, 114u, 109u, 102u, 114u, 97u, 109u, 101u, 102u, 114u, 97u, 109u, 101u, 115u, 101u, + 116u, 103u, 104u, 49u, 104u, 50u, 104u, 51u, 104u, 52u, 104u, 53u, 104u, 54u, 104u, 101u, 97u, 100u, 104u, + 101u, 97u, 100u, 101u, 114u, 104u, 103u, 114u, 111u, 117u, 112u, 104u, 114u, 104u, 116u, 109u, 108u, 105u, 105u, + 102u, 114u, 97u, 109u, 101u, 105u, 109u, 97u, 103u, 101u, 105u, 109u, 103u, 105u, 110u, 112u, 117u, 116u, 105u, + 110u, 115u, 105u, 115u, 105u, 110u, 100u, 101u, 120u, 107u, 98u, 100u, 107u, 101u, 121u, 103u, 101u, 110u, 108u, + 97u, 98u, 101u, 108u, 108u, 101u, 103u, 101u, 110u, 100u, 108u, 105u, 108u, 105u, 110u, 107u, 108u, 105u, 115u, + 116u, 105u, 110u, 103u, 109u, 97u, 105u, 110u, 109u, 97u, 108u, 105u, 103u, 110u, 109u, 97u, 114u, 107u, 109u, + 97u, 112u, 109u, 97u, 114u, 107u, 109u, 97u, 114u, 113u, 117u, 101u, 101u, 109u, 97u, 116u, 104u, 109u, 101u, + 110u, 117u, 109u, 101u, 116u, 97u, 109u, 101u, 116u, 101u, 114u, 109u, 103u, 108u, 121u, 112u, 104u, 109u, 105u, + 109u, 110u, 109u, 111u, 109u, 115u, 109u, 116u, 101u, 120u, 116u, 110u, 97u, 118u, 110u, 111u, 98u, 114u, 110u, + 111u, 101u, 109u, 98u, 101u, 100u, 110u, 111u, 102u, 114u, 97u, 109u, 101u, 115u, 110u, 111u, 115u, 99u, 114u, + 105u, 112u, 116u, 111u, 98u, 106u, 101u, 99u, 116u, 111u, 108u, 111u, 112u, 116u, 103u, 114u, 111u, 117u, 112u, + 111u, 112u, 116u, 105u, 111u, 110u, 111u, 117u, 116u, 112u, 117u, 116u, 112u, 112u, 97u, 114u, 97u, 109u, 112u, + 97u, 116u, 104u, 112u, 105u, 99u, 116u, 117u, 114u, 101u, 112u, 108u, 97u, 105u, 110u, 116u, 101u, 120u, 116u, + 112u, 114u, 101u, 112u, 114u, 111u, 103u, 114u, 101u, 115u, 115u, 113u, 114u, 98u, 114u, 112u, 114u, 116u, 114u, + 116u, 99u, 114u, 117u, 98u, 121u, 115u, 115u, 97u, 109u, 112u, 115u, 99u, 114u, 105u, 112u, 116u, 115u, 101u, + 97u, 114u, 99u, 104u, 115u, 101u, 99u, 116u, 105u, 111u, 110u, 115u, 101u, 108u, 101u, 99u, 116u, 115u, 101u, + 108u, 101u, 99u, 116u, 101u, 100u, 99u, 111u, 110u, 116u, 101u, 110u, 116u, 115u, 108u, 111u, 116u, 115u, 109u, + 97u, 108u, 108u, 115u, 111u, 117u, 114u, 99u, 101u, 115u, 112u, 97u, 110u, 115u, 116u, 114u, 105u, 107u, 101u, + 115u, 116u, 114u, 111u, 110u, 103u, 115u, 116u, 121u, 108u, 101u, 115u, 117u, 98u, 115u, 117u, 109u, 109u, 97u, + 114u, 121u, 115u, 117u, 112u, 115u, 118u, 103u, 116u, 97u, 98u, 108u, 101u, 116u, 98u, 111u, 100u, 121u, 116u, + 100u, 116u, 101u, 109u, 112u, 108u, 97u, 116u, 101u, 116u, 101u, 120u, 116u, 97u, 114u, 101u, 97u, 116u, 102u, + 111u, 111u, 116u, 116u, 104u, 116u, 104u, 101u, 97u, 100u, 116u, 105u, 109u, 101u, 116u, 105u, 116u, 108u, 101u, + 116u, 114u, 116u, 114u, 97u, 99u, 107u, 116u, 116u, 117u, 117u, 108u, 118u, 97u, 114u, 118u, 105u, 100u, 101u, + 111u, 119u, 98u, 114u, 120u, 109u, 112u}; static inline const uint32_t *th_tag_wide_name(uint16_t atom) { return th_tag_wide_names + th_tag_table[atom - 1].wide_offset; @@ -347,7 +383,17 @@ static inline const uint32_t *th_tag_wide_name(uint16_t atom) { >= c, so the entries beginning with byte c are [th_tag_first[c], th_tag_first[c + 1]). */ static const uint16_t th_tag_first[257] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 20, 28, 40, 42, 51, 52, 63, 70, 70, 72, 77, 92, 97, 102, 109, 110, 115, 133, 146, 148, 150, 151, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152 -}; + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 20, 28, 40, 42, 51, 52, 63, 70, 70, 72, 77, + 92, 97, 102, 109, 110, 115, 133, 146, 148, 150, 151, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, + 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, + 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, + 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, + 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, + 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, + 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152}; #endif /* TURBOHTML_TAG_ATOM_H */ diff --git a/src/turbohtml/_c/dom/element.c b/src/turbohtml/_c/dom/element.c index 65f94638..da126627 100644 --- a/src/turbohtml/_c/dom/element.c +++ b/src/turbohtml/_c/dom/element.c @@ -1877,9 +1877,9 @@ PyObject *make_element(PyTypeObject *type, PyObject *tag, PyObject *attrs) { } Py_UCS4 *tag_points = atom == TH_TAG_UNKNOWN ? PyUnicode_AsUCS4Copy(tag) : NULL; if (atom == TH_TAG_UNKNOWN && tag_points == NULL) { /* GCOVR_EXCL_BR_LINE: allocation failure */ - th_tree_free(tree); /* GCOVR_EXCL_LINE: allocation-failure path */ - Py_XDECREF(keys); /* GCOVR_EXCL_LINE: allocation-failure path */ - return NULL; /* GCOVR_EXCL_LINE: allocation-failure path */ + th_tree_free(tree); /* GCOVR_EXCL_LINE: allocation-failure path */ + Py_XDECREF(keys); /* GCOVR_EXCL_LINE: allocation-failure path */ + return NULL; /* GCOVR_EXCL_LINE: allocation-failure path */ } /* Unknown tag names are ASCII-lowercased to match what the parser stores. Known names already point at their lowercase generated entry. */ diff --git a/src/turbohtml/_c/dom/tree.c b/src/turbohtml/_c/dom/tree.c index 810dd319..cbe17d70 100644 --- a/src/turbohtml/_c/dom/tree.c +++ b/src/turbohtml/_c/dom/tree.c @@ -652,9 +652,8 @@ static void insertion_location(th_tree *tree, th_node **parent, th_node **before left absent until the source closes the element (record_end_tag_location). NULL on allocation failure. */ static th_src_loc *build_source_location(th_tree *tree, th_node *node, const th_token *token) { - th_src_loc *loc = arena_alloc( - tree, (Py_ssize_t)sizeof(th_src_loc) + token->attr_count * (Py_ssize_t)sizeof(th_src_attr) - ); + th_src_loc *loc = + arena_alloc(tree, (Py_ssize_t)sizeof(th_src_loc) + token->attr_count * (Py_ssize_t)sizeof(th_src_attr)); if (loc == NULL) { /* GCOVR_EXCL_BR_LINE: allocation failure cannot be forced from a test */ return NULL; /* GCOVR_EXCL_LINE: allocation-failure path, unreachable from a test */ } @@ -670,9 +669,8 @@ static th_src_loc *build_source_location(th_tree *tree, th_node *node, const th_ const th_attr *src = &token->attrs[index]; const th_attr_loc *src_loc = &token->attr_locs[index]; loc->attrs[index].name_atom = intern_attr(tree, &src->name); /* idempotent with insert_element's */ - loc->attrs[index].span = - (th_src_span){src_loc->name_line, src_loc->name_col, src_loc->name_off, src_loc->end_line, - src_loc->end_col, src_loc->end_off}; + loc->attrs[index].span = (th_src_span){src_loc->name_line, src_loc->name_col, src_loc->name_off, + src_loc->end_line, src_loc->end_col, src_loc->end_off}; } } *node_loc(node) = loc; diff --git a/src/turbohtml/_c/dom/tree_internal.h b/src/turbohtml/_c/dom/tree_internal.h index 902ae61a..3c0f8cd5 100644 --- a/src/turbohtml/_c/dom/tree_internal.h +++ b/src/turbohtml/_c/dom/tree_internal.h @@ -226,8 +226,7 @@ static inline th_node *node_new(th_tree *tree, enum th_node_type type) { int positioned = tree->track_positions && type == TH_NODE_ELEMENT; int located = tree->track_locations && type == TH_NODE_ELEMENT; Py_ssize_t size = type == TH_NODE_TEXT ? (Py_ssize_t)offsetof(th_node, attrs) : (Py_ssize_t)sizeof(th_node); - size += (positioned ? 2 * (Py_ssize_t)sizeof(uint32_t) : 0) + - (located ? (Py_ssize_t)sizeof(th_src_loc *) : 0); + size += (positioned ? 2 * (Py_ssize_t)sizeof(uint32_t) : 0) + (located ? (Py_ssize_t)sizeof(th_src_loc *) : 0); th_node *node = arena_alloc(tree, size); if (node == NULL) { /* GCOVR_EXCL_BR_LINE: allocation failure cannot be forced from a test */ return NULL; /* GCOVR_EXCL_LINE: allocation-failure path, unreachable from a test */ diff --git a/src/turbohtml/_c/tokenizer/statemachine.c b/src/turbohtml/_c/tokenizer/statemachine.c index bd5b82b2..f29195a5 100644 --- a/src/turbohtml/_c/tokenizer/statemachine.c +++ b/src/turbohtml/_c/tokenizer/statemachine.c @@ -232,8 +232,8 @@ struct th_tokenizer { Py_ssize_t slice_len; int input_borrowed; /* input.data is caller-owned storage, never freed here */ - th_token tok; /* tag/comment/doctype under construction */ - th_attr *attr; /* attribute under construction (points into tok.attrs) */ + th_token tok; /* tag/comment/doctype under construction */ + th_attr *attr; /* attribute under construction (points into tok.attrs) */ th_attr_loc *attr_loc; th_attr oom_attr; /* writable sink for attribute data after an allocation failure */ th_attr_loc oom_attr_loc; @@ -721,16 +721,16 @@ static void new_attr(th_tokenizer *self) { int grew = th_grow_cap((size_t)(tok->attr_cap + 1), (size_t)tok->attr_cap, 4, sizeof(th_attr), &cap, &bytes); if (!grew) { /* GCOVR_EXCL_BR_LINE: size overflow needs a length no allocation could hold */ self->oom = 1; /* GCOVR_EXCL_LINE: size-overflow path, unreachable from a test */ - self->attr = &self->oom_attr; /* GCOVR_EXCL_LINE: size-overflow path, unreachable from a test */ + self->attr = &self->oom_attr; /* GCOVR_EXCL_LINE: size-overflow path, unreachable from a test */ self->attr_loc = &self->oom_attr_loc; /* GCOVR_EXCL_LINE: size-overflow path */ - return; /* GCOVR_EXCL_LINE: size-overflow path, unreachable from a test */ + return; /* GCOVR_EXCL_LINE: size-overflow path, unreachable from a test */ } th_attr *grown = PyMem_Realloc(tok->attrs, bytes); if (grown == NULL) { /* GCOVR_EXCL_BR_LINE: allocation failure cannot be forced from a test */ self->oom = 1; /* GCOVR_EXCL_LINE: out-of-memory path, unreachable from a test */ self->attr = &self->oom_attr; /* GCOVR_EXCL_LINE: out-of-memory path, unreachable from a test */ self->attr_loc = &self->oom_attr_loc; /* GCOVR_EXCL_LINE: out-of-memory path */ - return; /* GCOVR_EXCL_LINE: allocation-failure path, unreachable from a test */ + return; /* GCOVR_EXCL_LINE: allocation-failure path, unreachable from a test */ } for (Py_ssize_t index = tok->attr_cap; index < (Py_ssize_t)cap; index++) { buf_init(&grown[index].name); @@ -740,11 +740,11 @@ static void new_attr(th_tokenizer *self) { tok->attrs = grown; if (self->capture_locations) { th_attr_loc *grown_locs = PyMem_Realloc(tok->attr_locs, cap * sizeof(th_attr_loc)); - if (grown_locs == NULL) { /* GCOVR_EXCL_BR_LINE: allocation failure cannot be forced from a test */ - self->oom = 1; /* GCOVR_EXCL_LINE: out-of-memory path, unreachable from a test */ + if (grown_locs == NULL) { /* GCOVR_EXCL_BR_LINE: allocation failure cannot be forced from a test */ + self->oom = 1; /* GCOVR_EXCL_LINE: out-of-memory path, unreachable from a test */ self->attr = &self->oom_attr; /* GCOVR_EXCL_LINE: out-of-memory path */ self->attr_loc = &self->oom_attr_loc; /* GCOVR_EXCL_LINE: out-of-memory path */ - return; /* GCOVR_EXCL_LINE: allocation-failure path */ + return; /* GCOVR_EXCL_LINE: allocation-failure path */ } tok->attr_locs = grown_locs; } diff --git a/src/turbohtml/_c/tokenizer/statemachine_run.h b/src/turbohtml/_c/tokenizer/statemachine_run.h index ce5a4f3b..520e390c 100644 --- a/src/turbohtml/_c/tokenizer/statemachine_run.h +++ b/src/turbohtml/_c/tokenizer/statemachine_run.h @@ -1146,9 +1146,8 @@ static enum run_result TH_NAME(run)(th_tokenizer *self) { continue; } if (ch == '&') { - Py_ssize_t consumed = TH_NAME(consume_charref)( - self, self->capture_attributes ? &self->attr->value : NULL, 1, NULL - ); + Py_ssize_t consumed = + TH_NAME(consume_charref)(self, self->capture_attributes ? &self->attr->value : NULL, 1, NULL); if (consumed == -1) { return RUN_NEED_MORE; } @@ -1185,9 +1184,8 @@ static enum run_result TH_NAME(run)(th_tokenizer *self) { continue; } if (ch == '&') { - Py_ssize_t consumed = TH_NAME(consume_charref)( - self, self->capture_attributes ? &self->attr->value : NULL, 1, NULL - ); + Py_ssize_t consumed = + TH_NAME(consume_charref)(self, self->capture_attributes ? &self->attr->value : NULL, 1, NULL); if (consumed == -1) { return RUN_NEED_MORE; } @@ -1215,9 +1213,8 @@ static enum run_result TH_NAME(run)(th_tokenizer *self) { continue; } if (ch == '&') { - Py_ssize_t consumed = TH_NAME(consume_charref)( - self, self->capture_attributes ? &self->attr->value : NULL, 1, NULL - ); + Py_ssize_t consumed = + TH_NAME(consume_charref)(self, self->capture_attributes ? &self->attr->value : NULL, 1, NULL); if (consumed == -1) { return RUN_NEED_MORE; } diff --git a/src/turbohtml/_c/tokenizer/tokenizer.c b/src/turbohtml/_c/tokenizer/tokenizer.c index 3b7664f6..f0c937ab 100644 --- a/src/turbohtml/_c/tokenizer/tokenizer.c +++ b/src/turbohtml/_c/tokenizer/tokenizer.c @@ -272,20 +272,21 @@ static PyMethodDef tokenizer_methods[] = { {NULL, NULL, 0, NULL}, }; -PyDoc_STRVAR(tokenizer_doc, "Tokenizer(*, resolve_references=True, capture_source=False, capture_attributes=True)\n--\n\n" - "Streaming HTML tokenizer. Feed markup with feed() and iterate the\n" - "returned iterators; call close() at the end, or use the tokenizer as a\n" - "context manager so leaving the with block signals end of input, then\n" - "iterate the tokenizer itself for the remaining tokens. For a whole\n" - "string at once use tokenize().\n\n" - ":param resolve_references: fold character references into the surrounding\n" - " text run; when False each one is emitted as its own CHARACTER_REFERENCE\n" - " token (its data the resolved value, its source the verbatim reference).\n" - " Attribute-value references are always resolved.\n" - ":param capture_source: record each markup token's verbatim source slice,\n" - " available as Token.source.\n" - ":param capture_attributes: retain tag attributes; disable this when only tag names and\n" - " text are needed to skip attribute storage and copying."); +PyDoc_STRVAR(tokenizer_doc, + "Tokenizer(*, resolve_references=True, capture_source=False, capture_attributes=True)\n--\n\n" + "Streaming HTML tokenizer. Feed markup with feed() and iterate the\n" + "returned iterators; call close() at the end, or use the tokenizer as a\n" + "context manager so leaving the with block signals end of input, then\n" + "iterate the tokenizer itself for the remaining tokens. For a whole\n" + "string at once use tokenize().\n\n" + ":param resolve_references: fold character references into the surrounding\n" + " text run; when False each one is emitted as its own CHARACTER_REFERENCE\n" + " token (its data the resolved value, its source the verbatim reference).\n" + " Attribute-value references are always resolved.\n" + ":param capture_source: record each markup token's verbatim source slice,\n" + " available as Token.source.\n" + ":param capture_attributes: retain tag attributes; disable this when only tag names and\n" + " text are needed to skip attribute storage and copying."); static PyType_Slot tokenizer_slots[] = { {Py_tp_doc, (void *)tokenizer_doc}, From ab76eef40dc4b42891c54fb70606561d0879f380 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 11 Jul 2026 08:11:44 -0700 Subject: [PATCH 5/6] =?UTF-8?q?=F0=9F=90=9B=20fix(core):=20restore=20Linux?= =?UTF-8?q?=20builds?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Linux C11 builds did not expose SSIZE_MAX through PyMem_New, and clang-tidy rejected the tagged text span's integer-to-pointer cast. Allocate the fixed-size caches by byte count and copy the tagged value into the pointer field. --- src/turbohtml/_c/dom/tree_internal.h | 4 +++- src/turbohtml/_c/query/methods.c | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/turbohtml/_c/dom/tree_internal.h b/src/turbohtml/_c/dom/tree_internal.h index 3c0f8cd5..86bc488c 100644 --- a/src/turbohtml/_c/dom/tree_internal.h +++ b/src/turbohtml/_c/dom/tree_internal.h @@ -167,7 +167,9 @@ static inline Py_ssize_t text_span_offset(const th_node *node) { } static inline void text_set_span(th_node *node, Py_ssize_t offset) { - node->text = (Py_UCS4 *)(((uintptr_t)offset << 1) | 1u); + uintptr_t encoded = ((uintptr_t)offset << 1) | 1u; + _Static_assert(sizeof(node->text) == sizeof(encoded), "tagged text spans require uintptr_t-sized pointers"); + memcpy(&node->text, &encoded, sizeof(encoded)); } static inline Py_UCS4 *need_text(th_tree *tree, th_node *node) { diff --git a/src/turbohtml/_c/query/methods.c b/src/turbohtml/_c/query/methods.c index aa6bc5b4..46d0bf81 100644 --- a/src/turbohtml/_c/query/methods.c +++ b/src/turbohtml/_c/query/methods.c @@ -85,7 +85,7 @@ static sel_compiled *cached_compile(PyObject *selector_error, HandleObject *hand return NULL; } if (handle->sel_cache == NULL) { - handle->sel_cache = PyMem_New(sel_cache_entry, SEL_CACHE_CAP); + handle->sel_cache = PyMem_Malloc(sizeof(sel_cache_entry) * SEL_CACHE_CAP); if (handle->sel_cache == NULL) { /* GCOVR_EXCL_BR_LINE: allocation failure cannot be forced from a test */ selector_free(compiled); /* GCOVR_EXCL_LINE: allocation-failure path */ PyErr_NoMemory(); /* GCOVR_EXCL_LINE: allocation-failure path */ @@ -589,7 +589,7 @@ static xp_program *cached_xpath_compile(HandleObject *handle, PyObject *arg) { return NULL; } if (handle->xpath_cache == NULL) { - handle->xpath_cache = PyMem_New(xpath_cache_entry, XPATH_CACHE_CAP); + handle->xpath_cache = PyMem_Malloc(sizeof(xpath_cache_entry) * XPATH_CACHE_CAP); if (handle->xpath_cache == NULL) { /* GCOVR_EXCL_BR_LINE: allocation failure cannot be forced from a test */ xp_free(prog); /* GCOVR_EXCL_LINE: allocation-failure path */ PyErr_NoMemory(); /* GCOVR_EXCL_LINE: allocation-failure path */ From 329eb409efc74092c40475c282d347f363204a06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 11 Jul 2026 08:11:45 -0700 Subject: [PATCH 6/6] =?UTF-8?q?=F0=9F=93=9D=20docs(changelog):=20use=20PR?= =?UTF-8?q?=20fragment=20number?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/changelog/{simple-parser.feature.rst => 640.feature.rst} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename docs/changelog/{simple-parser.feature.rst => 640.feature.rst} (100%) diff --git a/docs/changelog/simple-parser.feature.rst b/docs/changelog/640.feature.rst similarity index 100% rename from docs/changelog/simple-parser.feature.rst rename to docs/changelog/640.feature.rst