Skip to content

Commit 75dfa0d

Browse files
mikebarkminmikebarkmin
committed
fix typst editor
1 parent 0331bd1 commit 75dfa0d

3 files changed

Lines changed: 205 additions & 48 deletions

File tree

‎.changeset/fix-typst-assets.md‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
---
2+
"@hyperbook/markdown": patch
3+
"hyperbook": patch
4+
---
5+
6+
Fix Typst directive issues:
7+
- Fix preview not updating when editor content changes
8+
- Fix UTF-8 encoding for umlauts and special characters in base64 decoding
9+
- Fix CSV/JSON/YAML/XML file loading by inlining assets as bytes with proper UTF-8 handling
10+
- Add assets preamble to all source files to support `#include` with assets

‎packages/markdown/assets/directive-typst/client.js‎

Lines changed: 182 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ hyperbook.typst = (function () {
1515
};
1616

1717
const REGEX_PATTERNS = {
18-
READ: /#read\s*\(\s*(['"])([^'"]+)\1[^)]*\)/gi,
18+
READ: /read\s*\(\s*(['"])([^'"]+)\1[^)]*\)/gi,
1919
CSV: /csv\s*\(\s*(['"])([^'"]+)\1[^)]*\)/gi,
2020
JSON: /json\s*\(\s*(['"])([^'"]+)\1[^)]*\)/gi,
2121
YAML: /yaml\s*\(\s*(['"])([^'"]+)\1[^)]*\)/gi,
@@ -25,6 +25,9 @@ hyperbook.typst = (function () {
2525
ERROR_MESSAGE: /message:\s*"([^"]+)"/,
2626
};
2727

28+
// Text file patterns that need UTF-8 encoding
29+
const TEXT_PATTERNS = ['READ', 'CSV', 'JSON', 'YAML', 'XML'];
30+
2831
// ============================================================================
2932
// UTILITY FUNCTIONS
3033
// ============================================================================
@@ -229,15 +232,15 @@ hyperbook.typst = (function () {
229232
/**
230233
* Extract relative file paths from Typst source code
231234
* @param {string} src - Typst source code
232-
* @returns {Array<string>} Array of file paths
235+
* @returns {Array<{path: string, isText: boolean}>} Array of file paths with type info
233236
*/
234237
extractFilePaths(src) {
235-
const paths = new Set();
236-
const patterns = Object.values(REGEX_PATTERNS).filter(
237-
p => p !== REGEX_PATTERNS.ABSOLUTE_URL && p !== REGEX_PATTERNS.ERROR_MESSAGE
238-
);
238+
const paths = new Map(); // path -> isText
239239

240-
for (const pattern of patterns) {
240+
for (const [name, pattern] of Object.entries(REGEX_PATTERNS)) {
241+
if (name === 'ABSOLUTE_URL' || name === 'ERROR_MESSAGE') continue;
242+
243+
const isText = TEXT_PATTERNS.includes(name);
241244
let match;
242245
// Reset regex lastIndex
243246
pattern.lastIndex = 0;
@@ -246,21 +249,22 @@ hyperbook.typst = (function () {
246249
const path = match[2];
247250
// Skip absolute URLs, data URLs, blob URLs
248251
if (REGEX_PATTERNS.ABSOLUTE_URL.test(path)) continue;
249-
paths.add(path);
252+
paths.set(path, isText);
250253
}
251254
}
252255

253-
return Array.from(paths);
256+
return Array.from(paths.entries()).map(([path, isText]) => ({ path, isText }));
254257
}
255258

256259
/**
257260
* Fetch single asset from server
258261
* @param {string} path - Asset path
259262
* @param {string} basePath - Base path
260263
* @param {string} pagePath - Page path
264+
* @param {boolean} isText - Whether this is a text file
261265
* @returns {Promise<Uint8Array|null>}
262266
*/
263-
async fetchAsset(path, basePath, pagePath) {
267+
async fetchAsset(path, basePath, pagePath, isText = false) {
264268
try {
265269
const url = constructUrl(path, basePath, pagePath);
266270
const response = await fetch(url);
@@ -270,8 +274,15 @@ hyperbook.typst = (function () {
270274
return null;
271275
}
272276

273-
const arrayBuffer = await response.arrayBuffer();
274-
return new Uint8Array(arrayBuffer);
277+
if (isText) {
278+
// For text files, decode as text and re-encode as UTF-8
279+
const text = await response.text();
280+
return new TextEncoder().encode(text);
281+
} else {
282+
// For binary files, use arrayBuffer directly
283+
const arrayBuffer = await response.arrayBuffer();
284+
return new Uint8Array(arrayBuffer);
285+
}
275286
} catch (error) {
276287
console.warn(`Error loading asset ${path}:`, error);
277288
return null;
@@ -280,58 +291,149 @@ hyperbook.typst = (function () {
280291

281292
/**
282293
* Fetch multiple assets and cache them
283-
* @param {Array<string>} paths - Array of asset paths
294+
* @param {Array<{path: string, isText: boolean}>} pathInfos - Array of path info objects
284295
* @param {string} basePath - Base path
285296
* @param {string} pagePath - Page path
286297
* @returns {Promise<void>}
287298
*/
288-
async fetchAssets(paths, basePath, pagePath) {
289-
const missingPaths = paths.filter((p) => !this.cache.has(p));
299+
async fetchAssets(pathInfos, basePath, pagePath) {
300+
const missingPaths = pathInfos.filter(({ path }) => !this.cache.has(path));
290301

291302
await Promise.all(
292-
missingPaths.map(async (path) => {
293-
const data = await this.fetchAsset(path, basePath, pagePath);
303+
missingPaths.map(async ({ path, isText }) => {
304+
const data = await this.fetchAsset(path, basePath, pagePath, isText);
294305
this.cache.set(path, data);
295306
})
296307
);
297308
}
298309

299310
/**
300-
* Map cached assets to Typst virtual filesystem
311+
* Build Typst preamble with inlined assets as bytes
312+
* @returns {string} Typst preamble code
301313
*/
302-
mapToShadow() {
303-
for (const [path, data] of this.cache.entries()) {
304-
if (data !== null) {
305-
const normalizedPath = normalizePath(path);
306-
window.$typst.mapShadow(normalizedPath, data);
314+
buildAssetsPreamble() {
315+
if (this.cache.size === 0) return "";
316+
const entries = [...this.cache.entries()]
317+
.filter(([name, u8]) => u8 !== null)
318+
.map(([name, u8]) => {
319+
const nums = Array.from(u8).join(",");
320+
return ` "${name}": bytes((${nums}))`;
321+
})
322+
.join(",\n");
323+
if (!entries) return "";
324+
return `#let __assets = (\n${entries}\n)\n\n`;
325+
}
326+
327+
/**
328+
* Rewrite file calls (image, read, csv, json, yaml, xml) to use inlined assets
329+
* @param {string} src - Typst source code
330+
* @returns {string} Rewritten source code
331+
*/
332+
rewriteAssetCalls(src) {
333+
if (this.cache.size === 0) return src;
334+
335+
// Rewrite image() calls
336+
src = src.replace(/image\s*\(\s*(['"])([^'"]+)\1/g, (m, q, fname) => {
337+
if (this.cache.has(fname)) {
338+
const asset = this.cache.get(fname);
339+
if (asset === null) {
340+
return `[File not found: _${fname}_]`;
341+
}
342+
return `image(__assets.at("${fname}")`;
307343
}
308-
}
344+
return m;
345+
});
346+
347+
// Rewrite read() calls
348+
src = src.replace(/read\s*\(\s*(['"])([^'"]+)\1/g, (m, q, fname) => {
349+
if (this.cache.has(fname)) {
350+
const asset = this.cache.get(fname);
351+
if (asset === null) {
352+
return `[File not found: _${fname}_]`;
353+
}
354+
return `read(__assets.at("${fname}")`;
355+
}
356+
return m;
357+
});
358+
359+
// Rewrite csv() calls
360+
src = src.replace(/csv\s*\(\s*(['"])([^'"]+)\1/g, (m, q, fname) => {
361+
if (this.cache.has(fname)) {
362+
const asset = this.cache.get(fname);
363+
if (asset === null) {
364+
return `[File not found: _${fname}_]`;
365+
}
366+
return `csv(__assets.at("${fname}")`;
367+
}
368+
return m;
369+
});
370+
371+
// Rewrite json() calls
372+
src = src.replace(/json\s*\(\s*(['"])([^'"]+)\1/g, (m, q, fname) => {
373+
if (this.cache.has(fname)) {
374+
const asset = this.cache.get(fname);
375+
if (asset === null) {
376+
return `[File not found: _${fname}_]`;
377+
}
378+
return `json(__assets.at("${fname}")`;
379+
}
380+
return m;
381+
});
382+
383+
// Rewrite yaml() calls
384+
src = src.replace(/yaml\s*\(\s*(['"])([^'"]+)\1/g, (m, q, fname) => {
385+
if (this.cache.has(fname)) {
386+
const asset = this.cache.get(fname);
387+
if (asset === null) {
388+
return `[File not found: _${fname}_]`;
389+
}
390+
return `yaml(__assets.at("${fname}")`;
391+
}
392+
return m;
393+
});
394+
395+
// Rewrite xml() calls
396+
src = src.replace(/xml\s*\(\s*(['"])([^'"]+)\1/g, (m, q, fname) => {
397+
if (this.cache.has(fname)) {
398+
const asset = this.cache.get(fname);
399+
if (asset === null) {
400+
return `[File not found: _${fname}_]`;
401+
}
402+
return `xml(__assets.at("${fname}")`;
403+
}
404+
return m;
405+
});
406+
407+
return src;
309408
}
310409

311410
/**
312-
* Prepare assets for rendering (extract, fetch, and map)
411+
* Prepare assets for rendering (extract and fetch)
313412
* @param {string} mainSrc - Main Typst source
314413
* @param {Array} sourceFiles - Source file objects
315414
* @param {string} basePath - Base path
316415
* @param {string} pagePath - Page path
317416
* @returns {Promise<void>}
318417
*/
319418
async prepare(mainSrc, sourceFiles, basePath, pagePath) {
320-
const allPaths = new Set();
419+
const allPaths = new Map(); // path -> isText
321420

322421
// Extract from main source
323-
this.extractFilePaths(mainSrc).forEach((p) => allPaths.add(p));
422+
for (const { path, isText } of this.extractFilePaths(mainSrc)) {
423+
allPaths.set(path, isText);
424+
}
324425

325426
// Extract from all source files
326427
for (const { content } of sourceFiles) {
327-
this.extractFilePaths(content).forEach((p) => allPaths.add(p));
428+
for (const { path, isText } of this.extractFilePaths(content)) {
429+
allPaths.set(path, isText);
430+
}
328431
}
329432

330-
const paths = Array.from(allPaths);
433+
const pathInfos = Array.from(allPaths.entries()).map(([path, isText]) => ({ path, isText }));
331434

332-
if (paths.length > 0) {
333-
await this.fetchAssets(paths, basePath, pagePath);
334-
this.mapToShadow();
435+
if (pathInfos.length > 0) {
436+
await this.fetchAssets(pathInfos, basePath, pagePath);
335437
}
336438
}
337439
}
@@ -516,14 +618,23 @@ hyperbook.typst = (function () {
516618
// Prepare assets
517619
await this.assetManager.prepare(code, sourceFiles, basePath, pagePath);
518620

519-
// Add source files
520-
await this.addSourceFiles(sourceFiles);
621+
// Build assets preamble and rewrite source files
622+
const assetsPreamble = this.assetManager.buildAssetsPreamble();
623+
const rewrittenCode = this.assetManager.rewriteAssetCalls(code);
624+
const rewrittenSourceFiles = sourceFiles.map(({ filename, content }) => ({
625+
filename,
626+
content: assetsPreamble + this.assetManager.rewriteAssetCalls(content),
627+
}));
628+
629+
// Add source files with rewritten content (includes preamble)
630+
await this.addSourceFiles(rewrittenSourceFiles);
521631

522632
// Add binary files
523633
await BinaryFileHandler.addToShadow(binaryFiles);
524634

525-
// Render to SVG
526-
const svg = await window.$typst.svg({ mainContent: code });
635+
// Render to SVG with preamble prepended
636+
const mainContent = assetsPreamble + rewrittenCode;
637+
const svg = await window.$typst.svg({ mainContent });
527638

528639
// Clear any existing errors
529640
if (previewContainer) {
@@ -584,14 +695,23 @@ hyperbook.typst = (function () {
584695
// Prepare assets
585696
await this.assetManager.prepare(code, sourceFiles, basePath, pagePath);
586697

587-
// Add source files
588-
await this.addSourceFiles(sourceFiles);
698+
// Build assets preamble and rewrite source files
699+
const assetsPreamble = this.assetManager.buildAssetsPreamble();
700+
const rewrittenCode = this.assetManager.rewriteAssetCalls(code);
701+
const rewrittenSourceFiles = sourceFiles.map(({ filename, content }) => ({
702+
filename,
703+
content: assetsPreamble + this.assetManager.rewriteAssetCalls(content),
704+
}));
705+
706+
// Add source files with rewritten content (includes preamble)
707+
await this.addSourceFiles(rewrittenSourceFiles);
589708

590709
// Add binary files
591710
await BinaryFileHandler.addToShadow(binaryFiles);
592711

593-
// Generate PDF
594-
const pdfData = await window.$typst.pdf({ mainContent: code });
712+
// Generate PDF with preamble prepended
713+
const mainContent = assetsPreamble + rewrittenCode;
714+
const pdfData = await window.$typst.pdf({ mainContent });
595715
const pdfBlob = new Blob([pdfData], { type: 'application/pdf' });
596716

597717
// Download PDF
@@ -803,9 +923,9 @@ hyperbook.typst = (function () {
803923
* @returns {Promise<void>}
804924
*/
805925
async addAssets(zipFiles, code, basePath, pagePath) {
806-
const relPaths = this.assetManager.extractFilePaths(code);
926+
const pathInfos = this.assetManager.extractFilePaths(code);
807927

808-
for (const relPath of relPaths) {
928+
for (const { path: relPath, isText } of pathInfos) {
809929
const normalizedPath = relPath.startsWith('/')
810930
? relPath.substring(1)
811931
: relPath;
@@ -821,8 +941,13 @@ hyperbook.typst = (function () {
821941
const response = await fetch(url);
822942

823943
if (response.ok) {
824-
const arrayBuffer = await response.arrayBuffer();
825-
zipFiles[normalizedPath] = new Uint8Array(arrayBuffer);
944+
if (isText) {
945+
const text = await response.text();
946+
zipFiles[normalizedPath] = new TextEncoder().encode(text);
947+
} else {
948+
const arrayBuffer = await response.arrayBuffer();
949+
zipFiles[normalizedPath] = new Uint8Array(arrayBuffer);
950+
}
826951
} else {
827952
console.warn(`Failed to load asset: ${relPath} at ${url}`);
828953
}
@@ -1189,7 +1314,9 @@ hyperbook.typst = (function () {
11891314
this.fileManager.updateCurrentContent(this.editor.value);
11901315

11911316
const mainFile = this.fileManager.findMainFile();
1192-
const mainCode = mainFile ? mainFile.content : '';
1317+
const mainCode = mainFile
1318+
? this.fileManager.contents.get(mainFile.filename) || mainFile.content
1319+
: '';
11931320

11941321
this.renderer.render({
11951322
code: mainCode,
@@ -1388,9 +1515,16 @@ hyperbook.typst = (function () {
13881515
const basePath = elem.getAttribute('data-base-path') || '';
13891516
const pagePath = elem.getAttribute('data-page-path') || '';
13901517

1391-
const sourceFiles = sourceFilesData ? JSON.parse(atob(sourceFilesData)) : [];
1392-
const binaryFiles = binaryFilesData ? JSON.parse(atob(binaryFilesData)) : [];
1393-
const fontFiles = fontFilesData ? JSON.parse(atob(fontFilesData)) : [];
1518+
// Decode base64 with proper UTF-8 handling
1519+
const decodeBase64 = (str) => {
1520+
const binaryStr = atob(str);
1521+
const bytes = Uint8Array.from(binaryStr, (c) => c.charCodeAt(0));
1522+
return new TextDecoder('utf-8').decode(bytes);
1523+
};
1524+
1525+
const sourceFiles = sourceFilesData ? JSON.parse(decodeBase64(sourceFilesData)) : [];
1526+
const binaryFiles = binaryFilesData ? JSON.parse(decodeBase64(binaryFilesData)) : [];
1527+
const fontFiles = fontFilesData ? JSON.parse(decodeBase64(fontFilesData)) : [];
13941528

13951529
new TypstEditor({
13961530
elem,

0 commit comments

Comments
 (0)