From ca6f387d8a0cb981002264d946ffd631af0dff46 Mon Sep 17 00:00:00 2001 From: Jack Firth Date: Sat, 11 Jul 2026 00:08:54 -0700 Subject: [PATCH 01/13] Move linemap module into resyntax/grimoire Co-Authored-By: Claude Fable 5 --- {private => grimoire}/linemap.rkt | 0 private/analysis.rkt | 2 +- private/line-replacement.rkt | 2 +- private/refactoring-result.rkt | 2 +- private/syntax-replacement.rkt | 2 +- 5 files changed, 4 insertions(+), 4 deletions(-) rename {private => grimoire}/linemap.rkt (100%) diff --git a/private/linemap.rkt b/grimoire/linemap.rkt similarity index 100% rename from private/linemap.rkt rename to grimoire/linemap.rkt diff --git a/private/analysis.rkt b/private/analysis.rkt index e723a5b..9e21409 100644 --- a/private/analysis.rkt +++ b/private/analysis.rkt @@ -42,7 +42,7 @@ resyntax/default-recommendations/analyzers/ignored-result-values resyntax/default-recommendations/analyzers/variable-mutability resyntax/private/analyzer - resyntax/private/linemap + resyntax/grimoire/linemap resyntax/private/logger resyntax/grimoire/source resyntax/private/string-indent diff --git a/private/line-replacement.rkt b/private/line-replacement.rkt index 639f74f..5e52a29 100644 --- a/private/line-replacement.rkt +++ b/private/line-replacement.rkt @@ -27,7 +27,7 @@ rebellion/streaming/reducer rebellion/streaming/transducer rebellion/type/record - resyntax/private/linemap + resyntax/grimoire/linemap resyntax/grimoire/string-replacement) diff --git a/private/refactoring-result.rkt b/private/refactoring-result.rkt index 7b8aa47..734591f 100644 --- a/private/refactoring-result.rkt +++ b/private/refactoring-result.rkt @@ -50,7 +50,7 @@ resyntax/private/code-snippet resyntax/private/commit resyntax/private/line-replacement - resyntax/private/linemap + resyntax/grimoire/linemap resyntax/private/logger resyntax/grimoire/source resyntax/grimoire/string-replacement diff --git a/private/syntax-replacement.rkt b/private/syntax-replacement.rkt index 0789a78..8dd87c1 100644 --- a/private/syntax-replacement.rkt +++ b/private/syntax-replacement.rkt @@ -42,7 +42,7 @@ rebellion/collection/range-set rebellion/private/static-name rebellion/type/record - resyntax/private/linemap + resyntax/grimoire/linemap resyntax/private/logger resyntax/grimoire/source resyntax/private/string-indent From 172371834940454cb51fa0d871e6c66f55b8633f Mon Sep 17 00:00:00 2001 From: Jack Firth Date: Sat, 11 Jul 2026 00:08:54 -0700 Subject: [PATCH 02/13] Make string-linemap produce immutable line strings The linemap-lines contract has always promised immutable strings, but the implementation built lines with substring, which returns mutable ones. Any use of linemap-lines through the contract boundary raised a broke-its-own- contract error. It has no callers outside the module, so nothing noticed. Co-Authored-By: Claude Fable 5 --- grimoire/linemap.rkt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/grimoire/linemap.rkt b/grimoire/linemap.rkt index f499ce6..536c43e 100644 --- a/grimoire/linemap.rkt +++ b/grimoire/linemap.rkt @@ -45,12 +45,12 @@ (let loop ([line-number 1] [line-start-index 0] [index 0]) (cond [(= index char-count) - (define last-line (substring str line-start-index index)) + (define last-line (string->immutable-string (substring str line-start-index index))) (vector-builder-add lines last-line) (sorted-map-builder-put line-numbers-by-start-position (add1 line-start-index) line-number) (vector-builder-add start-positions-by-line-number (add1 line-start-index))] [(equal? (string-ref str index) #\newline) - (define next-line (substring str line-start-index index)) + (define next-line (string->immutable-string (substring str line-start-index index))) (vector-builder-add lines next-line) (sorted-map-builder-put line-numbers-by-start-position (add1 line-start-index) line-number) (vector-builder-add start-positions-by-line-number (add1 line-start-index)) From 2a3855d97a2e75471e944c0b955f10a81651f8dd Mon Sep 17 00:00:00 2001 From: Jack Firth Date: Sat, 11 Jul 2026 00:08:54 -0700 Subject: [PATCH 03/13] Draft API reference for linemaps Co-Authored-By: Claude Fable 5 --- grimoire.scrbl | 1 + grimoire/linemap.scrbl | 96 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 97 insertions(+) create mode 100644 grimoire/linemap.scrbl diff --git a/grimoire.scrbl b/grimoire.scrbl index 02bf46d..5f058ee 100644 --- a/grimoire.scrbl +++ b/grimoire.scrbl @@ -18,3 +18,4 @@ programmatically on anything found here. @include-section[(lib "resyntax/grimoire/syntax-property-bundle.scrbl")] @include-section[(lib "resyntax/grimoire/expansion-analyzers.scrbl")] @include-section[(lib "resyntax/grimoire/string-replacement.scrbl")] +@include-section[(lib "resyntax/grimoire/linemap.scrbl")] diff --git a/grimoire/linemap.scrbl b/grimoire/linemap.scrbl new file mode 100644 index 0000000..6c64fad --- /dev/null +++ b/grimoire/linemap.scrbl @@ -0,0 +1,96 @@ +#lang scribble/manual + + +@(require (for-label racket/base + racket/contract/base + rebellion/base/comparator + rebellion/base/range + resyntax/grimoire/linemap)) + + +@title[#:tag "linemap"]{Linemaps} +@defmodule[resyntax/grimoire/linemap] + +A @deftech{linemap} is a precomputed index of a string's line structure that supports converting +between character positions and line numbers. Resyntax straddles two views of source code: the +line-oriented view, used by @tech{source groups} and the command-line interface (see +@secref["cli"]) to select which code to analyze, and the character-oriented view, used by +@tech{string replacements} and syntax object source locations. Linemaps are the bridge between them --- Resyntax uses linemaps to compute which +lines a refactoring suggestion modifies, to render string replacements as line-based diffs, and to +restrict analysis to the requested lines. + +@bold{All positions and line numbers in a linemap are one-based}, matching the conventions of +@racket[syntax-position] and @racket[syntax-line]: the first character of a string is at position +@racket[1], on line @racket[1]. Beware that this is the @emph{opposite} of the convention used by +@tech{string replacements}, which address characters with @emph{zero-based} indices. Converting +between the two worlds requires adding or subtracting one, as discussed in +@secref["string-replacement"]. + +The lines of a string are the segments separated by newline characters. The terminating newline is +not part of a line's contents, but positions of newline characters belong to the lines they +terminate. A string that ends with a newline has a final empty line after it, and the empty string +consists of a single empty line. + + +@defproc[(linemap? [v any/c]) boolean?]{ + A predicate that recognizes @tech{linemaps}.} + + +@defproc[(string-linemap [str string?]) linemap?]{ + Constructs a @tech{linemap} of the lines in @racket[str]. Only @racket[#\newline] characters are + treated as line separators.} + + +@defproc[(linemap-lines [map linemap?]) + (vectorof (and/c string? immutable?) #:immutable #true)]{ + Returns a vector of the lines of the string that @racket[map] was built from, without their + terminating newlines. Note that because line numbers are one-based and vector indices are + zero-based, line @racket[_n] is at index @racket[_n] minus one in the returned vector.} + + +@defproc[(linemap-position-to-line [map linemap?] [position exact-positive-integer?]) + exact-positive-integer?]{ + Returns the line number of the line containing @racket[position]. The position of a newline + character is considered contained by the line that the newline terminates. Positions beyond the + end of the string do not raise an error; they are all treated as belonging to the last line.} + + +@defproc[(linemap-line-start-position [map linemap?] [line exact-positive-integer?]) + exact-positive-integer?]{ + Returns the position of the first character of @racket[line]. If the string ends with a newline, + the start position of its final, empty line is one past the end of the string. Raises an error if + @racket[line] is greater than the number of lines in the string.} + + +@defproc[(linemap-position-to-start-of-line [map linemap?] [position exact-positive-integer?]) + exact-positive-integer?]{ + Returns the position of the first character of the line containing @racket[position]. Equivalent + to composing @racket[linemap-position-to-line] with @racket[linemap-line-start-position].} + + +@defproc[(linemap-position-to-end-of-line [map linemap?] [position exact-positive-integer?]) + exact-positive-integer?]{ + Returns the position just past the last character of the contents of the line containing + @racket[position] --- that is, the position of the line's terminating newline, or one past the + end of the string if the line is the last one.} + + +@defproc[(syntax-start-line-position [stx syntax?] [#:linemap map linemap?]) + exact-positive-integer?]{ + Returns the position of the start of the line on which @racket[stx] begins. The + @tech[#:doc '(lib "scribblings/reference/reference.scrbl")]{source location} of @racket[stx] must + refer to positions within the string that @racket[map] was built from.} + + +@defproc[(syntax-end-line-position [stx syntax?] [#:linemap map linemap?]) + exact-positive-integer?]{ + Returns the position of the end of the line on which @racket[stx] ends, in the same sense as + @racket[linemap-position-to-end-of-line]. The source location of @racket[stx] must refer to + positions within the string that @racket[map] was built from.} + + +@defproc[(syntax-line-range [stx syntax?] [#:linemap map linemap?]) range?]{ + Returns a closed range (with @racket[natural<=>] as its comparator) containing the line numbers + of every line that @racket[stx] spans, from the line on which it begins to the line on which it + ends. The source location of @racket[stx] must refer to positions within the string that + @racket[map] was built from.} From 39d126d2afb5e9ca188635aa5cbda70bc62ae009 Mon Sep 17 00:00:00 2001 From: Jack Firth Date: Sat, 11 Jul 2026 00:18:19 -0700 Subject: [PATCH 04/13] Delete unused linemap bindings Co-Authored-By: Claude Fable 5 --- grimoire/linemap.rkt | 12 ------------ grimoire/linemap.scrbl | 37 +++++-------------------------------- 2 files changed, 5 insertions(+), 44 deletions(-) diff --git a/grimoire/linemap.rkt b/grimoire/linemap.rkt index 536c43e..4b05243 100644 --- a/grimoire/linemap.rkt +++ b/grimoire/linemap.rkt @@ -8,13 +8,9 @@ (contract-out [string-linemap (-> string? linemap?)] [linemap? (-> any/c boolean?)] - [linemap-lines (-> linemap? (vectorof (and/c string? immutable?) #:immutable #true #:flat? #true))] [linemap-position-to-line (-> linemap? exact-positive-integer? exact-positive-integer?)] - [linemap-line-start-position (-> linemap? exact-positive-integer? exact-positive-integer?)] [linemap-position-to-start-of-line (-> linemap? exact-positive-integer? exact-positive-integer?)] [linemap-position-to-end-of-line (-> linemap? exact-positive-integer? exact-positive-integer?)] - [syntax-start-line-position (-> syntax? #:linemap linemap? exact-positive-integer?)] - [syntax-end-line-position (-> syntax? #:linemap linemap? exact-positive-integer?)] [syntax-line-range (-> syntax? #:linemap linemap? range?)])) @@ -88,14 +84,6 @@ (linemap-line-end-position map (linemap-position-to-line map position))) -(define (syntax-start-line-position stx #:linemap map) - (linemap-position-to-start-of-line map (syntax-position stx))) - - -(define (syntax-end-line-position stx #:linemap map) - (linemap-position-to-end-of-line map (+ (syntax-position stx) (syntax-span stx)))) - - (define (syntax-line-range stx #:linemap map) (define first-line (syntax-line stx)) (define last-line (linemap-position-to-line map (+ (syntax-position stx) (syntax-span stx)))) diff --git a/grimoire/linemap.scrbl b/grimoire/linemap.scrbl index 6c64fad..de76b97 100644 --- a/grimoire/linemap.scrbl +++ b/grimoire/linemap.scrbl @@ -41,13 +41,6 @@ consists of a single empty line. treated as line separators.} -@defproc[(linemap-lines [map linemap?]) - (vectorof (and/c string? immutable?) #:immutable #true)]{ - Returns a vector of the lines of the string that @racket[map] was built from, without their - terminating newlines. Note that because line numbers are one-based and vector indices are - zero-based, line @racket[_n] is at index @racket[_n] minus one in the returned vector.} - - @defproc[(linemap-position-to-line [map linemap?] [position exact-positive-integer?]) exact-positive-integer?]{ Returns the line number of the line containing @racket[position]. The position of a newline @@ -55,17 +48,11 @@ consists of a single empty line. end of the string do not raise an error; they are all treated as belonging to the last line.} -@defproc[(linemap-line-start-position [map linemap?] [line exact-positive-integer?]) - exact-positive-integer?]{ - Returns the position of the first character of @racket[line]. If the string ends with a newline, - the start position of its final, empty line is one past the end of the string. Raises an error if - @racket[line] is greater than the number of lines in the string.} - - @defproc[(linemap-position-to-start-of-line [map linemap?] [position exact-positive-integer?]) exact-positive-integer?]{ - Returns the position of the first character of the line containing @racket[position]. Equivalent - to composing @racket[linemap-position-to-line] with @racket[linemap-line-start-position].} + Returns the position of the first character of the line containing @racket[position]. If the + string ends with a newline and @racket[position] is on the final, empty line after it, that + line's start position is one past the end of the string.} @defproc[(linemap-position-to-end-of-line [map linemap?] [position exact-positive-integer?]) @@ -75,22 +62,8 @@ consists of a single empty line. end of the string if the line is the last one.} -@defproc[(syntax-start-line-position [stx syntax?] [#:linemap map linemap?]) - exact-positive-integer?]{ - Returns the position of the start of the line on which @racket[stx] begins. The - @tech[#:doc '(lib "scribblings/reference/reference.scrbl")]{source location} of @racket[stx] must - refer to positions within the string that @racket[map] was built from.} - - -@defproc[(syntax-end-line-position [stx syntax?] [#:linemap map linemap?]) - exact-positive-integer?]{ - Returns the position of the end of the line on which @racket[stx] ends, in the same sense as - @racket[linemap-position-to-end-of-line]. The source location of @racket[stx] must refer to - positions within the string that @racket[map] was built from.} - - @defproc[(syntax-line-range [stx syntax?] [#:linemap map linemap?]) range?]{ Returns a closed range (with @racket[natural<=>] as its comparator) containing the line numbers of every line that @racket[stx] spans, from the line on which it begins to the line on which it - ends. The source location of @racket[stx] must refer to positions within the string that - @racket[map] was built from.} + ends. The @tech[#:doc '(lib "scribblings/reference/reference.scrbl")]{source location} of + @racket[stx] must refer to positions within the string that @racket[map] was built from.} From 8ef47bd3de6a69fcfcfb57246c57baf4ca7bc450 Mon Sep 17 00:00:00 2001 From: Jack Firth Date: Sat, 11 Jul 2026 00:35:35 -0700 Subject: [PATCH 05/13] Make linemap positions zero-indexed Linemaps are built from strings, and Racket string APIs (and Resyntax's string replacements) index characters from zero. One-indexed positions are a file port and IDE convention that doesn't belong here. Line numbers stay one-indexed, since they're primarily useful in UI contexts. Every caller was converting with add1/sub1 at the boundary; those shims are now gone, and syntax-line-range converts from one-indexed syntax positions itself. Co-Authored-By: Claude Fable 5 --- grimoire/linemap.rkt | 139 +++++++++++++++++---------------- grimoire/linemap.scrbl | 31 ++++---- private/line-replacement.rkt | 11 +-- private/refactoring-result.rkt | 8 +- private/syntax-replacement.rkt | 7 +- 5 files changed, 100 insertions(+), 96 deletions(-) diff --git a/grimoire/linemap.rkt b/grimoire/linemap.rkt index 4b05243..c88634b 100644 --- a/grimoire/linemap.rkt +++ b/grimoire/linemap.rkt @@ -8,9 +8,11 @@ (contract-out [string-linemap (-> string? linemap?)] [linemap? (-> any/c boolean?)] - [linemap-position-to-line (-> linemap? exact-positive-integer? exact-positive-integer?)] - [linemap-position-to-start-of-line (-> linemap? exact-positive-integer? exact-positive-integer?)] - [linemap-position-to-end-of-line (-> linemap? exact-positive-integer? exact-positive-integer?)] + [linemap-position-to-line (-> linemap? exact-nonnegative-integer? exact-positive-integer?)] + [linemap-position-to-start-of-line + (-> linemap? exact-nonnegative-integer? exact-nonnegative-integer?)] + [linemap-position-to-end-of-line + (-> linemap? exact-nonnegative-integer? exact-nonnegative-integer?)] [syntax-line-range (-> syntax? #:linemap linemap? range?)])) @@ -43,13 +45,13 @@ [(= index char-count) (define last-line (string->immutable-string (substring str line-start-index index))) (vector-builder-add lines last-line) - (sorted-map-builder-put line-numbers-by-start-position (add1 line-start-index) line-number) - (vector-builder-add start-positions-by-line-number (add1 line-start-index))] + (sorted-map-builder-put line-numbers-by-start-position line-start-index line-number) + (vector-builder-add start-positions-by-line-number line-start-index)] [(equal? (string-ref str index) #\newline) (define next-line (string->immutable-string (substring str line-start-index index))) (vector-builder-add lines next-line) - (sorted-map-builder-put line-numbers-by-start-position (add1 line-start-index) line-number) - (vector-builder-add start-positions-by-line-number (add1 line-start-index)) + (sorted-map-builder-put line-numbers-by-start-position line-start-index line-number) + (vector-builder-add start-positions-by-line-number line-start-index) (define next-index (add1 index)) (loop (add1 line-number) next-index next-index)] [else (loop line-number line-start-index (add1 index))])) @@ -86,7 +88,9 @@ (define (syntax-line-range stx #:linemap map) (define first-line (syntax-line stx)) - (define last-line (linemap-position-to-line map (+ (syntax-position stx) (syntax-span stx)))) + ;; Syntax object positions are one-indexed, unlike linemap positions. + (define last-line + (linemap-position-to-line map (+ (sub1 (syntax-position stx)) (syntax-span stx)))) (unless (<= first-line last-line) (raise-arguments-error 'syntax-line-range "syntax object's last line number is before its first line number" @@ -105,118 +109,119 @@ (define (nat-map . args) (apply sorted-map #:key-comparator natural<=> args)) - (check-equal? (string-linemap "") (linemap #("") (nat-map 1 1) #(1))) - (check-equal? (string-linemap "a") (linemap #("a") (nat-map 1 1) #(1))) - (check-equal? (string-linemap "λ") (linemap #("λ") (nat-map 1 1) #(1))) - (check-equal? (string-linemap "a\n") (linemap #("a" "") (nat-map 1 1 3 2) #(1 3))) - (check-equal? (string-linemap "λ\n") (linemap #("λ" "") (nat-map 1 1 3 2) #(1 3))) - (check-equal? (string-linemap "aaa\n") (linemap #("aaa" "") (nat-map 1 1 5 2) #(1 5))) - (check-equal? (string-linemap "aaa\nbbb") (linemap #("aaa" "bbb") (nat-map 1 1 5 2) #(1 5))) + (check-equal? (string-linemap "") (linemap #("") (nat-map 0 1) #(0))) + (check-equal? (string-linemap "a") (linemap #("a") (nat-map 0 1) #(0))) + (check-equal? (string-linemap "λ") (linemap #("λ") (nat-map 0 1) #(0))) + (check-equal? (string-linemap "a\n") (linemap #("a" "") (nat-map 0 1 2 2) #(0 2))) + (check-equal? (string-linemap "λ\n") (linemap #("λ" "") (nat-map 0 1 2 2) #(0 2))) + (check-equal? (string-linemap "aaa\n") (linemap #("aaa" "") (nat-map 0 1 4 2) #(0 4))) + (check-equal? (string-linemap "aaa\nbbb") (linemap #("aaa" "bbb") (nat-map 0 1 4 2) #(0 4))) (check-equal? (string-linemap "aaa\nbbb\n") - (linemap #("aaa" "bbb" "") (nat-map 1 1 5 2 9 3) #(1 5 9))) + (linemap #("aaa" "bbb" "") (nat-map 0 1 4 2 8 3) #(0 4 8))) (check-equal? (string-linemap "a\n\n\nb") - (linemap #("a" "" "" "b") (nat-map 1 1 3 2 4 3 5 4) #(1 3 4 5)))) + (linemap #("a" "" "" "b") (nat-map 0 1 2 2 3 3 4 4) #(0 2 3 4)))) (test-case (name-string linemap-position-to-line) (test-case "two-line string with ending newline" (define map (string-linemap "hello\nworld\n")) + (check-equal? (linemap-position-to-line map 0) 1) (check-equal? (linemap-position-to-line map 1) 1) (check-equal? (linemap-position-to-line map 2) 1) (check-equal? (linemap-position-to-line map 3) 1) (check-equal? (linemap-position-to-line map 4) 1) (check-equal? (linemap-position-to-line map 5) 1) - (check-equal? (linemap-position-to-line map 6) 1) + (check-equal? (linemap-position-to-line map 6) 2) (check-equal? (linemap-position-to-line map 7) 2) (check-equal? (linemap-position-to-line map 8) 2) (check-equal? (linemap-position-to-line map 9) 2) (check-equal? (linemap-position-to-line map 10) 2) (check-equal? (linemap-position-to-line map 11) 2) - (check-equal? (linemap-position-to-line map 12) 2) - (check-equal? (linemap-position-to-line map 13) 3)) + (check-equal? (linemap-position-to-line map 12) 3)) (test-case "multiple blank lines" (define map (string-linemap "a\n\nb")) + (check-equal? (linemap-position-to-line map 0) 1) (check-equal? (linemap-position-to-line map 1) 1) - (check-equal? (linemap-position-to-line map 2) 1) - (check-equal? (linemap-position-to-line map 3) 2) - (check-equal? (linemap-position-to-line map 4) 3) - (check-equal? (linemap-position-to-line map 5) 3))) + (check-equal? (linemap-position-to-line map 2) 2) + (check-equal? (linemap-position-to-line map 3) 3) + (check-equal? (linemap-position-to-line map 4) 3))) (test-case (name-string linemap-line-start-position) (test-case "two-line string with ending newline" (define map (string-linemap "hello\nworld\n")) - (check-equal? (linemap-line-start-position map 1) 1) - (check-equal? (linemap-line-start-position map 2) 7) - (check-equal? (linemap-line-start-position map 3) 13)) + (check-equal? (linemap-line-start-position map 1) 0) + (check-equal? (linemap-line-start-position map 2) 6) + (check-equal? (linemap-line-start-position map 3) 12)) (test-case "multiple blank lines" (define map (string-linemap "a\n\nb")) - (check-equal? (linemap-line-start-position map 1) 1) - (check-equal? (linemap-line-start-position map 2) 3) - (check-equal? (linemap-line-start-position map 3) 4))) + (check-equal? (linemap-line-start-position map 1) 0) + (check-equal? (linemap-line-start-position map 2) 2) + (check-equal? (linemap-line-start-position map 3) 3))) (test-case (name-string linemap-line-end-position) (test-case "two-line string with ending newline" (define map (string-linemap "hello\nworld\n")) - (check-equal? (linemap-line-end-position map 1) 6) - (check-equal? (linemap-line-end-position map 2) 12)) + (check-equal? (linemap-line-end-position map 1) 5) + (check-equal? (linemap-line-end-position map 2) 11)) (test-case "multiple blank lines" (define map (string-linemap "a\n\nb")) - (check-equal? (linemap-line-end-position map 1) 2) - (check-equal? (linemap-line-end-position map 2) 3) - (check-equal? (linemap-line-end-position map 3) 5))) + (check-equal? (linemap-line-end-position map 1) 1) + (check-equal? (linemap-line-end-position map 2) 2) + (check-equal? (linemap-line-end-position map 3) 4))) (test-case (name-string linemap-position-to-start-of-line) (test-case "two-line string with ending newline" (define map (string-linemap "hello\nworld\n")) - (check-equal? (linemap-position-to-start-of-line map 1) 1) - (check-equal? (linemap-position-to-start-of-line map 2) 1) - (check-equal? (linemap-position-to-start-of-line map 3) 1) - (check-equal? (linemap-position-to-start-of-line map 4) 1) - (check-equal? (linemap-position-to-start-of-line map 5) 1) - (check-equal? (linemap-position-to-start-of-line map 6) 1) - (check-equal? (linemap-position-to-start-of-line map 7) 7) - (check-equal? (linemap-position-to-start-of-line map 8) 7) - (check-equal? (linemap-position-to-start-of-line map 9) 7) - (check-equal? (linemap-position-to-start-of-line map 10) 7) - (check-equal? (linemap-position-to-start-of-line map 11) 7) - (check-equal? (linemap-position-to-start-of-line map 12) 7) - (check-equal? (linemap-position-to-start-of-line map 13) 13)) + (check-equal? (linemap-position-to-start-of-line map 0) 0) + (check-equal? (linemap-position-to-start-of-line map 1) 0) + (check-equal? (linemap-position-to-start-of-line map 2) 0) + (check-equal? (linemap-position-to-start-of-line map 3) 0) + (check-equal? (linemap-position-to-start-of-line map 4) 0) + (check-equal? (linemap-position-to-start-of-line map 5) 0) + (check-equal? (linemap-position-to-start-of-line map 6) 6) + (check-equal? (linemap-position-to-start-of-line map 7) 6) + (check-equal? (linemap-position-to-start-of-line map 8) 6) + (check-equal? (linemap-position-to-start-of-line map 9) 6) + (check-equal? (linemap-position-to-start-of-line map 10) 6) + (check-equal? (linemap-position-to-start-of-line map 11) 6) + (check-equal? (linemap-position-to-start-of-line map 12) 12)) (test-case "multiple blank lines" (define map (string-linemap "a\n\nb")) - (check-equal? (linemap-position-to-start-of-line map 1) 1) - (check-equal? (linemap-position-to-start-of-line map 2) 1) + (check-equal? (linemap-position-to-start-of-line map 0) 0) + (check-equal? (linemap-position-to-start-of-line map 1) 0) + (check-equal? (linemap-position-to-start-of-line map 2) 2) (check-equal? (linemap-position-to-start-of-line map 3) 3) - (check-equal? (linemap-position-to-start-of-line map 4) 4) - (check-equal? (linemap-position-to-start-of-line map 5) 4))) + (check-equal? (linemap-position-to-start-of-line map 4) 3))) (test-case (name-string linemap-position-to-end-of-line) (test-case "two-line string with ending newline" (define map (string-linemap "hello\nworld\n")) - (check-equal? (linemap-position-to-end-of-line map 1) 6) - (check-equal? (linemap-position-to-end-of-line map 2) 6) - (check-equal? (linemap-position-to-end-of-line map 3) 6) - (check-equal? (linemap-position-to-end-of-line map 4) 6) - (check-equal? (linemap-position-to-end-of-line map 5) 6) - (check-equal? (linemap-position-to-end-of-line map 6) 6) - (check-equal? (linemap-position-to-end-of-line map 7) 12) - (check-equal? (linemap-position-to-end-of-line map 8) 12) - (check-equal? (linemap-position-to-end-of-line map 9) 12) - (check-equal? (linemap-position-to-end-of-line map 10) 12) - (check-equal? (linemap-position-to-end-of-line map 11) 12) + (check-equal? (linemap-position-to-end-of-line map 0) 5) + (check-equal? (linemap-position-to-end-of-line map 1) 5) + (check-equal? (linemap-position-to-end-of-line map 2) 5) + (check-equal? (linemap-position-to-end-of-line map 3) 5) + (check-equal? (linemap-position-to-end-of-line map 4) 5) + (check-equal? (linemap-position-to-end-of-line map 5) 5) + (check-equal? (linemap-position-to-end-of-line map 6) 11) + (check-equal? (linemap-position-to-end-of-line map 7) 11) + (check-equal? (linemap-position-to-end-of-line map 8) 11) + (check-equal? (linemap-position-to-end-of-line map 9) 11) + (check-equal? (linemap-position-to-end-of-line map 10) 11) + (check-equal? (linemap-position-to-end-of-line map 11) 11) (check-equal? (linemap-position-to-end-of-line map 12) 12)) (test-case "multiple blank lines" (define map (string-linemap "a\n\nb")) - (check-equal? (linemap-position-to-end-of-line map 1) 2) + (check-equal? (linemap-position-to-end-of-line map 0) 1) + (check-equal? (linemap-position-to-end-of-line map 1) 1) (check-equal? (linemap-position-to-end-of-line map 2) 2) - (check-equal? (linemap-position-to-end-of-line map 3) 3) - (check-equal? (linemap-position-to-end-of-line map 4) 5) - (check-equal? (linemap-position-to-end-of-line map 5) 5)))) + (check-equal? (linemap-position-to-end-of-line map 3) 4) + (check-equal? (linemap-position-to-end-of-line map 4) 4)))) diff --git a/grimoire/linemap.scrbl b/grimoire/linemap.scrbl index de76b97..0b6cc91 100644 --- a/grimoire/linemap.scrbl +++ b/grimoire/linemap.scrbl @@ -19,12 +19,15 @@ line-oriented view, used by @tech{source groups} and the command-line interface lines a refactoring suggestion modifies, to render string replacements as line-based diffs, and to restrict analysis to the requested lines. -@bold{All positions and line numbers in a linemap are one-based}, matching the conventions of -@racket[syntax-position] and @racket[syntax-line]: the first character of a string is at position -@racket[1], on line @racket[1]. Beware that this is the @emph{opposite} of the convention used by -@tech{string replacements}, which address characters with @emph{zero-based} indices. Converting -between the two worlds requires adding or subtracting one, as discussed in -@secref["string-replacement"]. +@bold{Positions in a linemap are zero-based, but line numbers are one-based.} Positions are +character indices into the string, following the same convention as Racket's string operations and +as @tech{string replacements}: the first character of a string is at position @racket[0]. Line +numbers instead begin at line @racket[1], matching @racket[syntax-line] and the conventions of +code editors --- line numbers are almost exclusively useful in user interfaces, where one-based +numbering is expected. Beware that @racket[syntax-position] and file port positions are +@emph{one-based}, unlike linemap positions. The @racket[syntax-line-range] operation performs that +conversion itself, but positions obtained from syntax objects by other means must be converted +before use with a linemap. The lines of a string are the segments separated by newline characters. The terminating newline is not part of a line's contents, but positions of newline characters belong to the lines they @@ -41,25 +44,25 @@ consists of a single empty line. treated as line separators.} -@defproc[(linemap-position-to-line [map linemap?] [position exact-positive-integer?]) +@defproc[(linemap-position-to-line [map linemap?] [position exact-nonnegative-integer?]) exact-positive-integer?]{ Returns the line number of the line containing @racket[position]. The position of a newline character is considered contained by the line that the newline terminates. Positions beyond the end of the string do not raise an error; they are all treated as belonging to the last line.} -@defproc[(linemap-position-to-start-of-line [map linemap?] [position exact-positive-integer?]) - exact-positive-integer?]{ +@defproc[(linemap-position-to-start-of-line [map linemap?] [position exact-nonnegative-integer?]) + exact-nonnegative-integer?]{ Returns the position of the first character of the line containing @racket[position]. If the string ends with a newline and @racket[position] is on the final, empty line after it, that - line's start position is one past the end of the string.} + line's start position is equal to the length of the string.} -@defproc[(linemap-position-to-end-of-line [map linemap?] [position exact-positive-integer?]) - exact-positive-integer?]{ +@defproc[(linemap-position-to-end-of-line [map linemap?] [position exact-nonnegative-integer?]) + exact-nonnegative-integer?]{ Returns the position just past the last character of the contents of the line containing - @racket[position] --- that is, the position of the line's terminating newline, or one past the - end of the string if the line is the last one.} + @racket[position] --- that is, the position of the line's terminating newline, or the length of + the string if the line is the last one.} @defproc[(syntax-line-range [stx syntax?] [#:linemap map linemap?]) range?]{ diff --git a/private/line-replacement.rkt b/private/line-replacement.rkt index 5e52a29..37d8df2 100644 --- a/private/line-replacement.rkt +++ b/private/line-replacement.rkt @@ -77,16 +77,13 @@ (define new-lmap (string-linemap new-string)) (define start-line - (linemap-position-to-line orig-lmap (add1 (string-replacement-start replacement)))) + (linemap-position-to-line orig-lmap (string-replacement-start replacement))) (define start-pos - (sub1 - (linemap-position-to-start-of-line orig-lmap (add1 (string-replacement-start replacement))))) + (linemap-position-to-start-of-line orig-lmap (string-replacement-start replacement))) (define original-end-pos - (sub1 - (linemap-position-to-end-of-line orig-lmap - (add1 (string-replacement-original-end replacement))))) + (linemap-position-to-end-of-line orig-lmap (string-replacement-original-end replacement))) (define new-end-pos - (sub1 (linemap-position-to-end-of-line new-lmap (add1 (string-replacement-new-end replacement))))) + (linemap-position-to-end-of-line new-lmap (string-replacement-new-end replacement))) (define original-substr (substring original-string start-pos original-end-pos)) (define new-substr (substring new-string start-pos new-end-pos)) diff --git a/private/refactoring-result.rkt b/private/refactoring-result.rkt index 734591f..90ed951 100644 --- a/private/refactoring-result.rkt +++ b/private/refactoring-result.rkt @@ -192,9 +192,9 @@ (define lmap (string-linemap full-orig-code)) (define start (string-replacement-start replacement)) (define end (string-replacement-original-end replacement)) - (define start-column (- (add1 start) (linemap-position-to-start-of-line lmap (add1 start)))) + (define start-column (- start (linemap-position-to-start-of-line lmap start))) (define raw-text (string->immutable-string (substring full-orig-code start end))) - (code-snippet raw-text start-column (linemap-position-to-line lmap (add1 start)))) + (code-snippet raw-text start-column (linemap-position-to-line lmap start))) (define (refactoring-result-new-code result) @@ -203,8 +203,8 @@ (source->string (syntax-replacement-source (refactoring-result-syntax-replacement result)))) (define lmap (string-linemap full-orig-code)) (define start (string-replacement-start replacement)) - (define original-line (linemap-position-to-line lmap (add1 start))) - (define original-column (- (add1 start) (linemap-position-to-start-of-line lmap (add1 start)))) + (define original-line (linemap-position-to-line lmap start)) + (define original-column (- start (linemap-position-to-start-of-line lmap start))) (define refactored-source-code (string-replacement-apply replacement full-orig-code)) (define new-code-string (substring refactored-source-code diff --git a/private/syntax-replacement.rkt b/private/syntax-replacement.rkt index 8dd87c1..3e130ae 100644 --- a/private/syntax-replacement.rkt +++ b/private/syntax-replacement.rkt @@ -267,10 +267,9 @@ (define trailing-text-length (let ([linemap (string-linemap original)] [orig-end (string-replacement-original-end replacement)]) - ;; Convert from 0-based string indices to 1-based positions for linemap - (if (= (linemap-position-to-line linemap (add1 start)) - (linemap-position-to-line linemap (add1 orig-end))) - (- (linemap-position-to-end-of-line linemap (add1 orig-end)) (add1 orig-end)) + (if (= (linemap-position-to-line linemap start) + (linemap-position-to-line linemap orig-end)) + (- (linemap-position-to-end-of-line linemap orig-end) orig-end) 0))) (define allowed-width (- base-allowed-width trailing-text-length)) From cd533afc8a6bc5d03f4d1610cb4b9baa32540eb5 Mon Sep 17 00:00:00 2001 From: Jack Firth Date: Sat, 11 Jul 2026 00:40:29 -0700 Subject: [PATCH 06/13] Document newline normalization in source reading The fix for #272 (Windows \r\n newlines breaking analysis, PR #274) lives in with-input-from-source, which reencodes every source port with newline conversion. That behavior and the invariant it protects were undocumented. Co-Authored-By: Claude Fable 5 --- grimoire/linemap.scrbl | 8 ++++++-- grimoire/source.scrbl | 18 ++++++++++++++++-- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/grimoire/linemap.scrbl b/grimoire/linemap.scrbl index 0b6cc91..ae60597 100644 --- a/grimoire/linemap.scrbl +++ b/grimoire/linemap.scrbl @@ -5,7 +5,8 @@ racket/contract/base rebellion/base/comparator rebellion/base/range - resyntax/grimoire/linemap)) + resyntax/grimoire/linemap + resyntax/grimoire/source)) @title[#:tag "linemap"]{Linemaps} @@ -41,7 +42,10 @@ consists of a single empty line. @defproc[(string-linemap [str string?]) linemap?]{ Constructs a @tech{linemap} of the lines in @racket[str]. Only @racket[#\newline] characters are - treated as line separators.} + treated as line separators. In particular, Windows-style @racket["\r\n"] line endings are not + understood. This never arises in practice, because Resyntax normalizes all newlines to + @racket[#\newline] when reading @tech{source code} --- see @racket[with-input-from-source] for + details on that normalization and why it matters.} @defproc[(linemap-position-to-line [map linemap?] [position exact-nonnegative-integer?]) diff --git a/grimoire/source.scrbl b/grimoire/source.scrbl index 6984a38..9682ce1 100644 --- a/grimoire/source.scrbl +++ b/grimoire/source.scrbl @@ -4,6 +4,7 @@ @(require (for-label racket/base racket/contract/base racket/path + racket/port rebellion/base/immutable-string rebellion/collection/range-set resyntax/grimoire/source @@ -104,13 +105,26 @@ stack of dependent changes to commit in series without actually mutating the fil @defproc[(source->string [code source?]) immutable-string?]{ Returns the full text of @racket[code], reading it from the filesystem if necessary. For @racket[modified-source?] values, this returns the new, updated text rather than the original - unmodified text.} + unmodified text. The returned text has its newlines normalized, as described in + @racket[with-input-from-source].} @defproc[(with-input-from-source [code source?] [proc (-> any)]) any]{ Calls @racket[proc] with @racket[current-input-port] set to a freshly opened input port reading the contents of @racket[code]. For unmodified file sources, this opens a file port. For modified - sources and string sources, this opens a string port without interacting with the filesystem.} + sources and string sources, this opens a string port without interacting with the filesystem. + + The opened port decodes the source as UTF-8 and @bold{normalizes newlines}, as in + @racket[reencode-input-port]: Windows-style @racket["\r\n"] sequences (and other newline + conventions) are converted into single @racket[#\newline] characters. Every operation that reads + a source goes through this normalization, including @racket[source->string] and + @racket[source-read-syntax]. Normalizing consistently is load-bearing: when line counting is + enabled on a port, Racket counts a @racket["\r\n"] sequence as a @emph{single} position, so + without normalization the source locations of syntax objects read from a source would disagree + with the character indices of that source's text. Resyntax relies on the assumption that a syntax + object's position and span identify exactly the range of characters it was read from. Analyzing + code with Windows-style newlines used to violate that assumption and break Resyntax in + hard-to-diagnose ways.} @section{Parsing, Expanding, and Compiling Sources} From fc7f3b7b68cc07e7261a564f11de00a4a234e66c Mon Sep 17 00:00:00 2001 From: Jack Firth Date: Sat, 11 Jul 2026 00:56:36 -0700 Subject: [PATCH 07/13] Normalize newlines when constructing string and modified sources Previously only the port-reading path normalized newlines, so two string sources could be non-equal? yet denote the same text via source->string. Normalizing eagerly in the struct guards removes that corner case as early in the data flow as possible. The guard conversion matches the full set of sequences that reencode-input-port converts, verified empirically: \r\n, \r NEL, \r, NEL, and LS all become \n. Co-Authored-By: Claude Fable 5 --- grimoire/source.rkt | 32 ++++++++++++++++++++++++++++++-- grimoire/source.scrbl | 13 ++++++++++--- 2 files changed, 40 insertions(+), 5 deletions(-) diff --git a/grimoire/source.rkt b/grimoire/source.rkt index 8350f50..112cbe4 100644 --- a/grimoire/source.rkt +++ b/grimoire/source.rkt @@ -53,6 +53,19 @@ ;@---------------------------------------------------------------------------------------------------- +;; Matches every newline sequence that reencode-input-port's newline conversion recognizes. +(define newline-sequence-pattern + (let ([cr (string #\return)] + [lf (string #\newline)] + [nel (string (integer->char #x85))] + [ls (string (integer->char #x2028))]) + (regexp (string-append cr lf "|" cr nel "|" cr "|" nel "|" ls)))) + + +(define (string-normalize-newlines str) + (regexp-replace* newline-sequence-pattern str (string #\newline))) + + (struct source () #:transparent) @@ -66,12 +79,13 @@ (struct string-source unmodified-source (contents) #:transparent - #:guard (λ (contents _) (string->immutable-string contents))) + #:guard (λ (contents _) (string->immutable-string (string-normalize-newlines contents)))) (struct modified-source source (original contents) #:transparent - #:guard (λ (original contents _) (values original (string->immutable-string contents)))) + #:guard (λ (original contents _) + (values original (string->immutable-string (string-normalize-newlines contents))))) (define (source-name src) @@ -137,6 +151,20 @@ (module+ test + (test-case "string and modified sources normalize newlines upon construction" + (define cr (string #\return)) + (define lf (string #\newline)) + (define nel (string (integer->char #x85))) + (define ls (string (integer->char #x2028))) + (define normalized (string-source (string-append "a" lf "b" lf "c"))) + (check-equal? (string-source (string-append "a" cr lf "b" cr lf "c")) normalized) + (check-equal? (string-source (string-append "a" cr "b" cr "c")) normalized) + (check-equal? (string-source (string-append "a" nel "b" cr nel "c")) normalized) + (check-equal? (string-source (string-append "a" ls "b" ls "c")) normalized) + (define base (string-source "base")) + (check-equal? (modified-source base (string-append "a" cr lf "b")) + (modified-source base (string-append "a" lf "b")))) + (test-case "source-read-language" (check-equal? (source-read-language (string-source "#lang racket")) 'racket) (check-equal? (source-read-language (string-source "#lang at-exp racket")) 'at-exp) diff --git a/grimoire/source.scrbl b/grimoire/source.scrbl index 9682ce1..30db79c 100644 --- a/grimoire/source.scrbl +++ b/grimoire/source.scrbl @@ -64,7 +64,10 @@ stack of dependent changes to commit in series without actually mutating the fil @defproc[(string-source [contents string?]) string-source?]{ - Constructs a source string containing @racket[contents] directly.} + Constructs a source string containing @racket[contents] directly. The newlines of + @racket[contents] are normalized at construction time, in the same manner described in + @racket[with-input-from-source]. Normalizing eagerly ensures that two string sources denoting the + same text are @racket[equal?] even if they were constructed with different newline conventions.} @defproc[(modified-source? [v any/c]) boolean?]{ @@ -77,7 +80,8 @@ stack of dependent changes to commit in series without actually mutating the fil Constructs a modified source that replaces the contents of @racket[original] with @racket[new-contents]. This represents a whole-file replacement --- the @emph{complete} contents of @racket[original] are @emph{entirely} swapped out with @racket[new-contents]. Modified sources cannot - represent partial edits on their own.} + represent partial edits on their own. Like @racket[string-source], the newlines of + @racket[new-contents] are normalized at construction time.} @defproc[(source-name [code source?]) (or/c path? symbol?)]{ @@ -124,7 +128,10 @@ stack of dependent changes to commit in series without actually mutating the fil with the character indices of that source's text. Resyntax relies on the assumption that a syntax object's position and span identify exactly the range of characters it was read from. Analyzing code with Windows-style newlines used to violate that assumption and break Resyntax in - hard-to-diagnose ways.} + hard-to-diagnose ways. + + String sources and modified sources also apply this normalization eagerly, when the source value + is constructed, so the port-level conversion only has a visible effect for file sources.} @section{Parsing, Expanding, and Compiling Sources} From fc2bf196747e55b38a3d4cfc890dbc8337ea60c9 Mon Sep 17 00:00:00 2001 From: Jack Firth Date: Sat, 11 Jul 2026 01:02:38 -0700 Subject: [PATCH 08/13] Normalize newlines through reencode-input-port itself Rather than a regexp that reproduces the port's conversion behavior, round- trip the string through an actual newline-converting reencoded port. Slightly more copying, but it can never fall out of sync with what reading a source does. Co-Authored-By: Claude Fable 5 --- grimoire/source.rkt | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/grimoire/source.rkt b/grimoire/source.rkt index 112cbe4..9cb4f2d 100644 --- a/grimoire/source.rkt +++ b/grimoire/source.rkt @@ -53,17 +53,13 @@ ;@---------------------------------------------------------------------------------------------------- -;; Matches every newline sequence that reencode-input-port's newline conversion recognizes. -(define newline-sequence-pattern - (let ([cr (string #\return)] - [lf (string #\newline)] - [nel (string (integer->char #x85))] - [ls (string (integer->char #x2028))]) - (regexp (string-append cr lf "|" cr nel "|" cr "|" nel "|" ls)))) - - +;; Reads str back out through a newline-converting reencoded port, guaranteeing that this +;; normalization can never disagree with the one performed by with-input-from-source. (define (string-normalize-newlines str) - (regexp-replace* newline-sequence-pattern str (string #\newline))) + (define reencoded-in + (reencode-input-port + (open-input-string str) "UTF-8" #false #false 'string-normalize-newlines #true)) + (port->string reencoded-in)) (struct source () #:transparent) From 68dd1673951dadc5b410b0c2b8d6b444631028c9 Mon Sep 17 00:00:00 2001 From: Jack Firth Date: Sat, 11 Jul 2026 01:29:09 -0700 Subject: [PATCH 09/13] Share one reencode-input-port wrapper across all source reading Co-Authored-By: Claude Fable 5 --- grimoire/source.rkt | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/grimoire/source.rkt b/grimoire/source.rkt index 9cb4f2d..a7bc150 100644 --- a/grimoire/source.rkt +++ b/grimoire/source.rkt @@ -53,13 +53,14 @@ ;@---------------------------------------------------------------------------------------------------- -;; Reads str back out through a newline-converting reencoded port, guaranteeing that this -;; normalization can never disagree with the one performed by with-input-from-source. +;; All source text flows through this port wrapper, which decodes it as UTF-8 and converts +;; newline sequences like \r\n into single \n characters. +(define (reencoded-source-input-port in) + (reencode-input-port in "UTF-8" #false #false (object-name in) #true)) + + (define (string-normalize-newlines str) - (define reencoded-in - (reencode-input-port - (open-input-string str) "UTF-8" #false #false 'string-normalize-newlines #true)) - (port->string reencoded-in)) + (port->string (reencoded-source-input-port (open-input-string str)))) (struct source () #:transparent) @@ -93,8 +94,7 @@ (define (with-input-from-source code proc) (define (call-proc-with-reencoded-input in) - (define reencoded-in (reencode-input-port in "UTF-8" #false #false (object-name in) #true)) - (parameterize ([current-input-port reencoded-in]) + (parameterize ([current-input-port (reencoded-source-input-port in)]) (proc))) (match code From 00e44332184dadc435d5730f3d1118c6fb61e985 Mon Sep 17 00:00:00 2001 From: Jack Firth Date: Sat, 11 Jul 2026 01:51:57 -0700 Subject: [PATCH 10/13] Link line-counting docs from newline normalization prose Co-Authored-By: Claude Fable 5 --- grimoire/source.scrbl | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/grimoire/source.scrbl b/grimoire/source.scrbl index 30db79c..0f41351 100644 --- a/grimoire/source.scrbl +++ b/grimoire/source.scrbl @@ -122,13 +122,15 @@ stack of dependent changes to commit in series without actually mutating the fil @racket[reencode-input-port]: Windows-style @racket["\r\n"] sequences (and other newline conventions) are converted into single @racket[#\newline] characters. Every operation that reads a source goes through this normalization, including @racket[source->string] and - @racket[source-read-syntax]. Normalizing consistently is load-bearing: when line counting is - enabled on a port, Racket counts a @racket["\r\n"] sequence as a @emph{single} position, so - without normalization the source locations of syntax objects read from a source would disagree - with the character indices of that source's text. Resyntax relies on the assumption that a syntax - object's position and span identify exactly the range of characters it was read from. Analyzing - code with Windows-style newlines used to violate that assumption and break Resyntax in - hard-to-diagnose ways. + @racket[source-read-syntax]. Normalizing consistently is load-bearing: as described in + @secref["linecol" #:doc '(lib "scribblings/reference/reference.scrbl")], Racket performs this + same conversion whenever it counts line and column numbers, treating a @racket["\r\n"] sequence + as a @emph{single} position --- and line counting must be enabled for syntax objects to receive + line numbers in their source locations. Without normalization, then, the source locations of + syntax objects read from a source would disagree with the character indices of that source's + text. Resyntax relies on the assumption that a syntax object's position and span identify exactly + the range of characters it was read from. Analyzing code with Windows-style newlines used to + violate that assumption and break Resyntax in hard-to-diagnose ways. String sources and modified sources also apply this normalization eagerly, when the source value is constructed, so the port-level conversion only has a visible effect for file sources.} From 51404782619ef302d05f8fc7b0cf008c4695ec8c Mon Sep 17 00:00:00 2001 From: Jack Firth Date: Sat, 11 Jul 2026 14:42:37 -0700 Subject: [PATCH 11/13] Edit linemap overview docs --- grimoire/linemap.scrbl | 47 +++++++++++++++++++++++++++--------------- 1 file changed, 30 insertions(+), 17 deletions(-) diff --git a/grimoire/linemap.scrbl b/grimoire/linemap.scrbl index ae60597..f59b13c 100644 --- a/grimoire/linemap.scrbl +++ b/grimoire/linemap.scrbl @@ -13,27 +13,40 @@ @defmodule[resyntax/grimoire/linemap] A @deftech{linemap} is a precomputed index of a string's line structure that supports converting -between character positions and line numbers. Resyntax straddles two views of source code: the -line-oriented view, used by @tech{source groups} and the command-line interface (see -@secref["cli"]) to select which code to analyze, and the character-oriented view, used by -@tech{string replacements} and syntax object source locations. Linemaps are the bridge between them --- Resyntax uses linemaps to compute which -lines a refactoring suggestion modifies, to render string replacements as line-based diffs, and to -restrict analysis to the requested lines. - -@bold{Positions in a linemap are zero-based, but line numbers are one-based.} Positions are -character indices into the string, following the same convention as Racket's string operations and -as @tech{string replacements}: the first character of a string is at position @racket[0]. Line -numbers instead begin at line @racket[1], matching @racket[syntax-line] and the conventions of -code editors --- line numbers are almost exclusively useful in user interfaces, where one-based -numbering is expected. Beware that @racket[syntax-position] and file port positions are -@emph{one-based}, unlike linemap positions. The @racket[syntax-line-range] operation performs that -conversion itself, but positions obtained from syntax objects by other means must be converted -before use with a linemap. +between character positions and line numbers. Source code is frequently viewed from one of two +perspectives: + +@itemlist[ + @item{A @emph{human} perspective, either reader or writer, who looks at code as a 2D grid composed of + lines and columns.} + + @item{A @emph{machine} perspective, that looks at code as one line sequence of characters (or perhaps + even just plain bytes).}] + +Tools that serve as a human-machine interface for code often have to juggle these two perspectives. +Linemaps are Resyntax's tool for doing so. They are used in various places where human concerns +related to viewing and describing source code and source edits come up, such as displaying or +consuming line-based diffs in the command-line interface. See @secref["cli"] for further details on +that matter. + +@bold{Positions in a linemap are zero-based, but line numbers are one-based.} Positions +are character indices into the string. This follows the same convention as Racket's string operations +such as @racket[string-ref], as well as Resyntax's conventions for @tech{string replacements}. Line +numbers, however, follow the conventions outlined in +@secref["linecol" #:doc '(lib "scribblings/reference/reference.scrbl")]: source files begin at line +@racket[1]. This matches @racket[syntax-line] and the conventions of code editors --- line numbers +are almost exclusively useful in user interfaces, where one-based numbering is expected. + +@bold{Beware that @racket[syntax-position] and file port positions are @emph{one-based}, unlike + linemap positions.} The @racket[syntax-line-range] operation performs that conversion itself, but +positions obtained from syntax objects by other means must be converted before use with a linemap. The lines of a string are the segments separated by newline characters. The terminating newline is not part of a line's contents, but positions of newline characters belong to the lines they terminate. A string that ends with a newline has a final empty line after it, and the empty string -consists of a single empty line. +consists of a single empty line. Note that there are multiple distinct byte sequences that Racket +treats as a newline when reading source code --- for further details on how Resyntax handles these +cases, see the notes in @racket[with-input-from-source]. @defproc[(linemap? [v any/c]) boolean?]{ From 719da3d3c02e16da6716b1910005d3297573b4e2 Mon Sep 17 00:00:00 2001 From: Jack Firth Date: Sat, 11 Jul 2026 14:46:07 -0700 Subject: [PATCH 12/13] Fix typo in linemap docs Co-Authored-By: Claude Fable 5 --- grimoire/linemap.scrbl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/grimoire/linemap.scrbl b/grimoire/linemap.scrbl index f59b13c..42fc607 100644 --- a/grimoire/linemap.scrbl +++ b/grimoire/linemap.scrbl @@ -20,8 +20,8 @@ perspectives: @item{A @emph{human} perspective, either reader or writer, who looks at code as a 2D grid composed of lines and columns.} - @item{A @emph{machine} perspective, that looks at code as one line sequence of characters (or perhaps - even just plain bytes).}] + @item{A @emph{machine} perspective, that looks at code as one linear sequence of characters (or + perhaps even just plain bytes).}] Tools that serve as a human-machine interface for code often have to juggle these two perspectives. Linemaps are Resyntax's tool for doing so. They are used in various places where human concerns From 3d7b37ab12b80a0994f6fe8341b4cee885d68c35 Mon Sep 17 00:00:00 2001 From: Jack Firth Date: Sat, 11 Jul 2026 14:53:52 -0700 Subject: [PATCH 13/13] Fix 1 occurrence of `define-let-to-multi-define` Applies Resyntax's own suggestion from the PR analyzer bot, which the GitHub UI couldn't apply because the suggestion range overlapped deleted lines. Co-Authored-By: Claude Fable 5 --- private/syntax-replacement.rkt | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/private/syntax-replacement.rkt b/private/syntax-replacement.rkt index 3e130ae..6a05271 100644 --- a/private/syntax-replacement.rkt +++ b/private/syntax-replacement.rkt @@ -264,13 +264,12 @@ ;; For single-line replacements, we need to account for trailing text after the replacement ;; to ensure the formatted code doesn't exceed the line length limit. + (define linemap (string-linemap original)) + (define orig-end (string-replacement-original-end replacement)) (define trailing-text-length - (let ([linemap (string-linemap original)] - [orig-end (string-replacement-original-end replacement)]) - (if (= (linemap-position-to-line linemap start) - (linemap-position-to-line linemap orig-end)) - (- (linemap-position-to-end-of-line linemap orig-end) orig-end) - 0))) + (if (= (linemap-position-to-line linemap start) (linemap-position-to-line linemap orig-end)) + (- (linemap-position-to-end-of-line linemap orig-end) orig-end) + 0)) (define allowed-width (- base-allowed-width trailing-text-length)) (define formatted-code-substring