From b6a0db27421f163418ab4653db9a5d66eb864278 Mon Sep 17 00:00:00 2001 From: Ilyes512 Date: Mon, 1 Jun 2026 22:07:41 +0100 Subject: [PATCH] Implemented improved binary detection --- .../docs/architecture/template-engine.md | 32 ++++- internal/template/template.go | 14 ++- internal/template/template_test.go | 111 ++++++++++++++++++ 3 files changed, 153 insertions(+), 4 deletions(-) diff --git a/docs/content/docs/architecture/template-engine.md b/docs/content/docs/architecture/template-engine.md index 3e1d6e7..e1cbb19 100644 --- a/docs/content/docs/architecture/template-engine.md +++ b/docs/content/docs/architecture/template-engine.md @@ -170,8 +170,36 @@ If any `RenderWarning` entries are present after `Execute` returns (only possibl ### Binary Detection -A file is treated as binary if the first 512 bytes contain a null byte or are not valid UTF-8. -Binary files are copied byte-for-byte; no template rendering is attempted. +The engine inspects the first 512 bytes of every file using a two-stage check. Binary files +are copied byte-for-byte; no template rendering is attempted. + +**Detection order:** + +1. **`http.DetectContentType`** — identifies known binary formats by magic bytes (JPEG, PNG, + PDF, ZIP, gzip, and others). If the detected content type is not a `text/*` type, the + file is treated as binary. +2. **Null-byte / invalid-UTF-8 fallback** — if `DetectContentType` returns `text/plain`, + the file is still treated as binary when the first 512 bytes contain a null byte or are + not valid UTF-8. This catches edge cases such as UTF-16 LE files (which have many null + bytes) or binary payloads whose prefix happens to look like text. + +**Limitations and `.specsverbatim`:** +Binary detection is best-effort. Any file that must be copied verbatim should be listed +explicitly in `.specsverbatim` rather than relying on auto-detection: + +``` +# .specsverbatim — recommended patterns for common binary assets +*.png +*.jpg +*.ico +*.woff +*.woff2 +*.ttf +*.pdf +*.gz +*.tar +*.zip +``` ### File Permissions diff --git a/internal/template/template.go b/internal/template/template.go index beba330..5ee35f1 100644 --- a/internal/template/template.go +++ b/internal/template/template.go @@ -5,6 +5,7 @@ import ( "io" "io/fs" "log/slog" + "net/http" "os" "path/filepath" "slices" @@ -303,8 +304,13 @@ func contentPreview(data []byte) string { return s } -// isBinary returns true if the file contains a null byte or invalid UTF-8. -// Only the first 512 bytes are examined for performance. +// isBinary reports whether the file should be copied verbatim rather than rendered. +// Only the first 512 bytes are examined. Two checks run in order: +// +// 1. http.DetectContentType: if the content is not a text/* type (e.g. image/jpeg, +// application/pdf, application/octet-stream), the file is binary. +// 2. Null-byte / invalid-UTF-8 fallback: catches edge cases that DetectContentType +// classifies as text/plain (e.g. a binary payload whose prefix looks like text). func isBinary(path string) bool { f, err := os.Open(path) if err != nil { @@ -316,6 +322,10 @@ func isBinary(path string) bool { n, _ := f.Read(buf) buf = buf[:n] + if ct := http.DetectContentType(buf); !strings.HasPrefix(ct, "text/") { + return true + } + if slices.Contains(buf, 0) { return true } diff --git a/internal/template/template_test.go b/internal/template/template_test.go index 0692e08..98fc980 100644 --- a/internal/template/template_test.go +++ b/internal/template/template_test.go @@ -526,6 +526,117 @@ func TestExecute_PreservesPermissions_BinaryFile(t *testing.T) { } } +// Binary detection tests — each case documents the expected behaviour of isBinary +// under the two-stage check: http.DetectContentType first, null/UTF-8 fallback second. + +func TestExecute_BinaryDetection_JPEG(t *testing.T) { + // JPEG magic bytes (FF D8 FF) — detected as image/jpeg by http.DetectContentType. + // Content includes a template marker to prove the file is NOT rendered as a template. + content := append([]byte{0xFF, 0xD8, 0xFF, 0xE0}, []byte("{{ .Name }}")...) + root := buildTemplate(t, "Name: test\n", map[string][]byte{ + "photo.jpg": content, + }) + + tmpl, err := pkgtemplate.Get(root, pkgtemplate.Config{}) + if err != nil { + t.Fatalf("Get: %v", err) + } + target := t.TempDir() + if err := tmpl.Execute(target); err != nil { + t.Fatalf("Execute: %v", err) + } + + got, err := os.ReadFile(filepath.Join(target, "photo.jpg")) + if err != nil { + t.Fatalf("reading photo.jpg: %v", err) + } + if string(got) != string(content) { + t.Error("photo.jpg not copied verbatim (JPEG magic bytes should be detected as binary)") + } +} + +func TestExecute_BinaryDetection_Gzip(t *testing.T) { + // Gzip magic bytes (1F 8B) — 0x1F is a binary data byte per the WHATWG sniff + // algorithm, so http.DetectContentType returns application/octet-stream. + content := append([]byte{0x1F, 0x8B, 0x08, 0x00}, []byte("{{ .Name }}")...) + root := buildTemplate(t, "Name: test\n", map[string][]byte{ + "archive.tar.gz": content, + }) + + tmpl, err := pkgtemplate.Get(root, pkgtemplate.Config{}) + if err != nil { + t.Fatalf("Get: %v", err) + } + target := t.TempDir() + if err := tmpl.Execute(target); err != nil { + t.Fatalf("Execute: %v", err) + } + + got, err := os.ReadFile(filepath.Join(target, "archive.tar.gz")) + if err != nil { + t.Fatalf("reading archive.tar.gz: %v", err) + } + if string(got) != string(content) { + t.Error("archive.tar.gz not copied verbatim (gzip magic bytes should be detected as binary)") + } +} + +func TestExecute_BinaryDetection_UTF16BOM(t *testing.T) { + // UTF-16 LE BOM (FF FE) — 0xFF is in the high-byte range, so + // http.DetectContentType returns application/octet-stream. + // Preserved as binary; template expansion is never attempted. + content := []byte{0xFF, 0xFE, 0x41, 0x00, 0x42, 0x00, 0x43, 0x00} // BOM + "ABC" in UTF-16 LE + root := buildTemplate(t, "Name: test\n", map[string][]byte{ + "utf16.txt": content, + }) + + tmpl, err := pkgtemplate.Get(root, pkgtemplate.Config{}) + if err != nil { + t.Fatalf("Get: %v", err) + } + target := t.TempDir() + if err := tmpl.Execute(target); err != nil { + t.Fatalf("Execute: %v", err) + } + + got, err := os.ReadFile(filepath.Join(target, "utf16.txt")) + if err != nil { + t.Fatalf("reading utf16.txt: %v", err) + } + if string(got) != string(content) { + t.Error("utf16.txt not copied verbatim (UTF-16 BOM should be detected as binary)") + } +} + +func TestExecute_BinaryDetection_PDFHeader(t *testing.T) { + // PDF header (%PDF-) — all bytes are valid ASCII/UTF-8 with no null bytes, so + // the old null/UTF-8 heuristic alone would treat this as text. http.DetectContentType + // recognises the magic bytes and returns application/pdf, which the primary check + // catches as non-text → the file is copied verbatim. + // The embedded {{ .Name }} must be preserved literally, not expanded. + content := []byte("%PDF-1.4\n{{ .Name }}") + root := buildTemplate(t, "Name: test\n", map[string][]byte{ + "document.pdf": content, + }) + + tmpl, err := pkgtemplate.Get(root, pkgtemplate.Config{}) + if err != nil { + t.Fatalf("Get: %v", err) + } + target := t.TempDir() + if err := tmpl.Execute(target); err != nil { + t.Fatalf("Execute: %v", err) + } + + got, err := os.ReadFile(filepath.Join(target, "document.pdf")) + if err != nil { + t.Fatalf("reading document.pdf: %v", err) + } + if string(got) != string(content) { + t.Errorf("document.pdf = %q, want verbatim %q (PDF header should be detected as binary even though bytes are valid UTF-8)", got, string(content)) + } +} + func TestGet_CustomDelimiters_ConditionalFilename(t *testing.T) { // Custom delimiters affect filename templates too. yaml := "UseX: true\n__delimiters:\n left: \"[[\"\n right: \"]]\"\n"