diff --git a/src/extractor/parser/extractConstants.ts b/src/extractor/parser/extractConstants.ts new file mode 100644 index 00000000..d4c3472b --- /dev/null +++ b/src/extractor/parser/extractConstants.ts @@ -0,0 +1,252 @@ +import { Token } from './types.js'; + +/** + * Pre-pass over the merged token stream that builds a flat map of + * module-top-level constant string bindings. + * + * Two declaration shapes are recognised: + * + * const NAME = 'literal' + * const NAME = 'literal' as const + * + * and (flat) string-valued object literals: + * + * const NS = { KEY1: 'literal1', KEY2: 'literal2' } as const + * const NS = { KEY1: 'literal1', KEY2: 'literal2' } + * + * Object literals are flattened, so the second example produces entries + * `NS.KEY1 -> 'literal1'` and `NS.KEY2 -> 'literal2'`. Nested objects + * are intentionally skipped (the existing dynamic-namespace warning + * still fires for those). + * + * The extractor walks tokens, not the TS AST, so it cannot natively + * resolve an identifier passed to `useTranslate(NS)` or `` + * back to its declaration. This helper produces the small symbol table + * the parser substitutes in when it sees a variable reference. + * + * Walking the already-merged token stream (rather than running regex on + * the raw source) means strings, comments, template literals and block + * delimiters are already correctly classified by TextMate. In particular + * the `funcDepth` counter ensures we only capture declarations at module + * top level — a `const X = '...'` nested inside a function body or + * branch is invisible to the capture, which matches the visibility the + * consumer of the identifier would have. + */ + +type State = + | 'Idle' + | 'AfterConst' + | 'AfterName' + | 'InTypeAnnotation' + | 'AfterAssign' + | 'InObjectBody' + | 'InObjectAfterKey' + | 'InObjectAfterColon'; + +type AnyToken = Token; + +type Context = { + result: Map; + state: State; + // Depth of block/control-flow bodies we're currently inside. While + // > 0, top-level const-detection is suspended; only block.begin/end + // are tracked so we know when we resurface. + funcDepth: number; + // Depth of a square-bracket pattern (array literal or array + // destructure) we're currently skipping. Tracked separately from + // funcDepth so list.begin/end never leaks into block depth and vice + // versa. + listDepth: number; + // Depth of the object literal currently being captured (1 = top of + // the object body). Tracked separately from funcDepth so a nested + // object literal value abandons capture without leaking depth. + objectDepth: number; + captureName: string; + captureProps: Array<{ key: string; value: string }>; + currentKey: string; + // Set when a nested block.begin is seen during object capture; any + // properties already collected for this capture are discarded. + abandonedObject: boolean; +}; + +function createContext(): Context { + return { + result: new Map(), + state: 'Idle', + funcDepth: 0, + listDepth: 0, + objectDepth: 0, + captureName: '', + captureProps: [], + currentKey: '', + abandonedObject: false, + }; +} + +function resetCapture(ctx: Context): void { + ctx.state = 'Idle'; + ctx.captureProps = []; + ctx.captureName = ''; + ctx.currentKey = ''; + ctx.abandonedObject = false; +} + +function finalizeObjectCapture(ctx: Context): void { + if (!ctx.abandonedObject) { + // Mirror JS object-literal semantics: a later property with the same + // name overwrites earlier ones, so always assign. + for (const { key, value } of ctx.captureProps) { + ctx.result.set(`${ctx.captureName}.${key}`, value); + } + } + resetCapture(ctx); +} + +/** Track block depth inside a function body / control-flow block. */ +function stepInFunctionBody(ctx: Context, token: AnyToken): void { + if (token.customType === 'block.begin') ctx.funcDepth++; + else if (token.customType === 'block.end') ctx.funcDepth--; +} + +/** Skip an array literal / array destructure pattern at top level. */ +function stepInListSkip(ctx: Context, token: AnyToken): void { + if (token.customType === 'list.begin') ctx.listDepth++; + else if (token.customType === 'list.end') ctx.listDepth--; +} + +/** Walk the body of a `const NS = { ... }` capture. */ +function stepInObjectBody(ctx: Context, token: AnyToken): void { + const c = token.customType; + + if (c === 'block.begin') { + ctx.objectDepth++; + ctx.abandonedObject = true; + return; + } + if (c === 'block.end') { + ctx.objectDepth--; + if (ctx.objectDepth === 0) { + finalizeObjectCapture(ctx); + } + return; + } + if (ctx.abandonedObject) return; + + switch (ctx.state) { + case 'InObjectBody': + if (c === 'object.key') { + ctx.currentKey = token.token; + ctx.state = 'InObjectAfterKey'; + } + break; + case 'InObjectAfterKey': + if (c === 'acessor.doublecolon') { + ctx.state = 'InObjectAfterColon'; + } else { + ctx.state = 'InObjectBody'; + } + break; + case 'InObjectAfterColon': + if (c === 'string') { + ctx.captureProps.push({ key: ctx.currentKey, value: token.token }); + } + ctx.state = 'InObjectBody'; + break; + } +} + +/** Module-top-level state machine — the only place captures originate. */ +function stepAtTopLevel(ctx: Context, token: AnyToken): void { + const c = token.customType; + + switch (ctx.state) { + case 'Idle': + if (c === 'block.begin') { + ctx.funcDepth = 1; + } else if (!c && token.token === 'const') { + ctx.state = 'AfterConst'; + } + // `export` and other prefixes simply stay in Idle until the + // following `const` token is seen. + break; + + case 'AfterConst': + if (c === 'variable') { + ctx.captureName = token.token; + ctx.state = 'AfterName'; + } else if (c === 'block.begin') { + // Object destructure pattern like `const { x } = ...` — skip + // until the matching block.end via funcDepth. + ctx.funcDepth = 1; + ctx.state = 'Idle'; + } else if (c === 'list.begin') { + // Array destructure pattern like `const [x] = ...` — skip + // until the matching list.end via listDepth. Tracked + // separately from funcDepth because stepInFunctionBody only + // consumes block.begin/end and would never see the closing + // bracket, leaving funcDepth permanently > 0. + ctx.listDepth = 1; + ctx.state = 'Idle'; + } else { + ctx.state = 'Idle'; + } + break; + + case 'AfterName': + if (c === 'operator.assignment') { + ctx.state = 'AfterAssign'; + } else if (c === 'acessor.doublecolon') { + ctx.state = 'InTypeAnnotation'; + } else { + ctx.state = 'Idle'; + } + break; + + case 'InTypeAnnotation': + if (c === 'operator.assignment') { + ctx.state = 'AfterAssign'; + } else if (c === 'block.begin') { + // Type-level block like `const X: { foo: 1 } = ...`. Bail out + // of this declaration; the rest is content we can't reason + // about with this state machine. + ctx.funcDepth = 1; + ctx.state = 'Idle'; + } + // Otherwise just skip over the type tokens. + break; + + case 'AfterAssign': + if (c === 'string') { + if (!ctx.result.has(ctx.captureName)) { + ctx.result.set(ctx.captureName, token.token); + } + resetCapture(ctx); + } else if (c === 'block.begin') { + ctx.objectDepth = 1; + ctx.state = 'InObjectBody'; + } else { + ctx.state = 'Idle'; + } + break; + } +} + +export function extractConstants( + tokens: ReadonlyArray +): Map { + const ctx = createContext(); + + for (const token of tokens) { + if (ctx.funcDepth > 0) { + stepInFunctionBody(ctx, token); + } else if (ctx.listDepth > 0) { + stepInListSkip(ctx, token); + } else if (ctx.objectDepth > 0) { + stepInObjectBody(ctx, token); + } else { + stepAtTopLevel(ctx, token); + } + } + + return ctx.result; +} diff --git a/src/extractor/parser/generalMapper.ts b/src/extractor/parser/generalMapper.ts index ff1df7ed..1a4132ac 100644 --- a/src/extractor/parser/generalMapper.ts +++ b/src/extractor/parser/generalMapper.ts @@ -37,6 +37,9 @@ export const generalMapper = (token: Token) => { // variables case 'variable.other.object.ts': case 'variable.other.constant.ts': + case 'variable.other.constant.object.ts': + case 'variable.other.constant.property.ts': + case 'variable.other.property.ts': case 'variable.language.this.ts': return 'variable'; diff --git a/src/extractor/parser/parser.ts b/src/extractor/parser/parser.ts index 56010e04..75c9be18 100644 --- a/src/extractor/parser/parser.ts +++ b/src/extractor/parser/parser.ts @@ -24,6 +24,7 @@ import { closingTagMerger } from './tokenMergers/closingTagMerger.js'; import { typesAsMerger } from './tokenMergers/typesAsMerger.js'; import { typesCastMerger } from './tokenMergers/typesCastMerger.js'; import { customTCallMerger } from './tokenMergers/customTCallMerger.js'; +import { extractConstants } from './extractConstants.js'; export const DEFAULT_BLOCKS = { 'block.begin': ['block.end'], @@ -124,6 +125,7 @@ export const Parser = ({ withLabel, ruleMap, blocks, + constants: extractConstants(filteredIgnored), }; let depth = 0; diff --git a/src/extractor/parser/tree/getValue.ts b/src/extractor/parser/tree/getValue.ts index b0f8461a..fcd54139 100644 --- a/src/extractor/parser/tree/getValue.ts +++ b/src/extractor/parser/tree/getValue.ts @@ -18,6 +18,33 @@ export function getValue( end: ['expression.end'] as T[], }); + case 'variable': { + // Look ahead for a same-file `NAME.PROP` member access. The tokens + // are forward-only, but the only way `variable + acessor.dot + + // variable` shows up in a value position is when the user is + // referencing a constants object — consuming those tokens here + // doesn't disrupt any other parse path. + const next = context.tokens.peek(); + if (next?.customType === 'acessor.dot') { + context.tokens.next(); + const afterDot = context.tokens.peek(); + if (afterDot?.customType === 'variable') { + context.tokens.next(); + const memberKey = `${token.token}.${afterDot.token}`; + const memberResolved = context.constants.get(memberKey); + if (memberResolved !== undefined) { + return { type: 'primitive', line, value: memberResolved }; + } + return { type: 'expr', line, values: [] }; + } + } + const resolved = context.constants.get(token.token); + if (resolved !== undefined) { + return { type: 'primitive', line, value: resolved }; + } + return { type: 'expr', line, values: [] }; + } + default: return { type: 'expr', line, values: [] }; } diff --git a/src/extractor/parser/types.ts b/src/extractor/parser/types.ts index e86352b6..bfcb7a28 100644 --- a/src/extractor/parser/types.ts +++ b/src/extractor/parser/types.ts @@ -89,6 +89,7 @@ export type ParserContext = { withLabel: (fn: (...args: S) => U) => (...args: S) => U; ruleMap: RuleMap; blocks: BlocksType; + constants: Map; }; export type ExtractorInternal = ( diff --git a/test/e2e/push.p3.test.ts b/test/e2e/push.p3.test.ts index d6d2fc6a..b1978492 100644 --- a/test/e2e/push.p3.test.ts +++ b/test/e2e/push.p3.test.ts @@ -169,17 +169,21 @@ describe('project 3', () => { const stored = tolgeeDataToDict(keys.data); - // Keys in the "food" namespace should not be removed - expect(Object.keys(stored)).toEqual([ - 'table', - 'chair', - 'plate', - 'fork', - 'water', - 'salad', - 'tomato', - 'onions', - ]); + // Keys in the "food" namespace should not be removed. The API does + // not guarantee an ordering for the returned keys, so compare as a + // set (sort both sides). + expect(Object.keys(stored).sort()).toEqual( + [ + 'table', + 'chair', + 'plate', + 'fork', + 'water', + 'salad', + 'tomato', + 'onions', + ].sort() + ); }); it('removes other keys (config)', async () => { @@ -216,17 +220,20 @@ describe('project 3', () => { const stored = tolgeeDataToDict(keys.data); - // Keys in the "food" namespace should not be removed - expect(Object.keys(stored)).toEqual([ - 'table', - 'chair', - 'plate', - 'fork', - 'water', - 'salad', - 'tomato', - 'onions', - ]); + // Keys in the "food" namespace should not be removed. Compare as a + // set (see the args-variant above for rationale). + expect(Object.keys(stored).sort()).toEqual( + [ + 'table', + 'chair', + 'plate', + 'fork', + 'water', + 'salad', + 'tomato', + 'onions', + ].sort() + ); }); it("doesn't remove other keys when filtered by namespace", async () => { @@ -270,16 +277,19 @@ describe('project 3', () => { const stored = tolgeeDataToDict(keys.data); - expect(Object.keys(stored)).toEqual([ - 'table', - 'chair', - 'plate', - 'fork', - 'knife', - 'water', - 'salad', - 'tomato', - 'onions', - ]); + // Compare as a set; API ordering of returned keys is not guaranteed. + expect(Object.keys(stored).sort()).toEqual( + [ + 'table', + 'chair', + 'plate', + 'fork', + 'knife', + 'water', + 'salad', + 'tomato', + 'onions', + ].sort() + ); }); }); diff --git a/test/unit/extractor/react/tComponent.test.ts b/test/unit/extractor/react/tComponent.test.ts index 10b9c873..838cc9d1 100644 --- a/test/unit/extractor/react/tComponent.test.ts +++ b/test/unit/extractor/react/tComponent.test.ts @@ -363,4 +363,54 @@ describe.each(['jsx', 'tsx'])(' (.%s)', (ext) => { expect(extracted.keys).toEqual(expectedKeys); }); }); + + describe('top-level const namespace identifiers', () => { + it('resolves a const used as the `ns` prop of ', async () => { + const code = ` + import { T } from '@tolgee/react' + const NS = 'auth' + function Test () { + return + } + `; + + const extracted = await extractReactKeys(code, FILE_NAME); + expect(extracted.warnings).toEqual([]); + expect(extracted.keys).toEqual([ + { keyName: 'key1', namespace: 'auth', line: 5 }, + ]); + }); + + it('resolves an `as const` declaration used as `ns` prop', async () => { + const code = ` + import { T } from '@tolgee/react' + export const NS = 'billing' as const + function Test () { + return + } + `; + + const extracted = await extractReactKeys(code, FILE_NAME); + expect(extracted.warnings).toEqual([]); + expect(extracted.keys).toEqual([ + { keyName: 'key1', namespace: 'billing', line: 5 }, + ]); + }); + + it('resolves member access on a const object used as `ns` prop', async () => { + const code = ` + import { T } from '@tolgee/react' + const NS = { AUTH: 'auth', NAMESPACED: 'namespaced' } as const + function Test () { + return + } + `; + + const extracted = await extractReactKeys(code, FILE_NAME); + expect(extracted.warnings).toEqual([]); + expect(extracted.keys).toEqual([ + { keyName: 'key1', namespace: 'namespaced', line: 5 }, + ]); + }); + }); }); diff --git a/test/unit/extractor/react/useTranslate.test.ts b/test/unit/extractor/react/useTranslate.test.ts index 03c1ca07..7815e4fd 100644 --- a/test/unit/extractor/react/useTranslate.test.ts +++ b/test/unit/extractor/react/useTranslate.test.ts @@ -549,4 +549,172 @@ describe.each(['js', 'ts', 'jsx', 'tsx'])('useTranslate (.%s)', (ext) => { expect(extracted.warnings).toEqual([]); }); }); + + describe('top-level const namespace identifiers', () => { + it('resolves a plain const string to its literal value', async () => { + const code = ` + import '@tolgee/react' + const NS = 'my-namespace' + function Test () { + const { t } = useTranslate(NS) + t('key1') + } + `; + + const extracted = await extractReactKeys(code, FILE_NAME); + expect(extracted.warnings).toEqual([]); + expect(extracted.keys).toEqual([ + { keyName: 'key1', namespace: 'my-namespace', line: 6 }, + ]); + }); + + it('resolves an exported `as const` declaration', async () => { + const code = ` + import '@tolgee/react' + export const NS = 'billing' as const + function Test () { + const { t } = useTranslate(NS) + t('key1') + } + `; + + const extracted = await extractReactKeys(code, FILE_NAME); + expect(extracted.warnings).toEqual([]); + expect(extracted.keys).toEqual([ + { keyName: 'key1', namespace: 'billing', line: 6 }, + ]); + }); + + it('resolves member access on a const object (`as const`)', async () => { + const code = ` + import '@tolgee/react' + const NS = { AUTH: 'auth', BILLING: 'billing' } as const + function Test () { + const { t } = useTranslate(NS.BILLING) + t('key1') + } + `; + + const extracted = await extractReactKeys(code, FILE_NAME); + expect(extracted.warnings).toEqual([]); + expect(extracted.keys).toEqual([ + { keyName: 'key1', namespace: 'billing', line: 6 }, + ]); + }); + + it('resolves member access on a plain const object (no `as const`)', async () => { + const code = ` + import '@tolgee/react' + const NS = { AUTH: 'auth' } + function Test () { + const { t } = useTranslate(NS.AUTH) + t('key1') + } + `; + + const extracted = await extractReactKeys(code, FILE_NAME); + expect(extracted.warnings).toEqual([]); + expect(extracted.keys).toEqual([ + { keyName: 'key1', namespace: 'auth', line: 6 }, + ]); + }); + + it('matches JS last-write-wins for duplicate object keys', async () => { + const code = ` + import '@tolgee/react' + const NS = { K: 'first', K: 'second' } as const + function Test () { + const { t } = useTranslate(NS.K) + t('key1') + } + `; + + const extracted = await extractReactKeys(code, FILE_NAME); + expect(extracted.warnings).toEqual([]); + expect(extracted.keys).toEqual([ + { keyName: 'key1', namespace: 'second', line: 6 }, + ]); + }); + + it('keeps capturing after a top-level array destructure', async () => { + // Regression: `const [a, b] = ...` is a list-destructure pattern. + // It must not leave the walker stuck in skip mode, otherwise the + // subsequent `const NS = ...` would silently fail to be captured. + const code = ` + import '@tolgee/react' + const [first, second] = ['x', 'y'] + const NS = 'after-destructure' + function Test () { + const { t } = useTranslate(NS) + t('key1') + } + `; + + const extracted = await extractReactKeys(code, FILE_NAME); + expect(extracted.warnings).toEqual([]); + expect(extracted.keys).toEqual([ + { keyName: 'key1', namespace: 'after-destructure', line: 7 }, + ]); + }); + + it('ignores const declarations nested inside function bodies', async () => { + // A function-local `const NS` must not shadow what the consumer + // would see at module scope. Here there is no module-level NS, + // so the call site should produce the existing dynamic-namespace + // warning, not be silently resolved to the inner literal. + const code = ` + import '@tolgee/react' + function helper () { + const NS = 'inner-only' + return NS + } + function Test () { + const { t } = useTranslate(NS) + t('key1') + } + `; + + const extracted = await extractReactKeys(code, FILE_NAME); + expect(extracted.warnings).toEqual([ + { warning: 'W_DYNAMIC_NAMESPACE', line: 8 }, + { warning: 'W_UNRESOLVABLE_NAMESPACE', line: 9 }, + ]); + expect(extracted.keys).toEqual([]); + }); + + it('still warns for member access on an unknown property', async () => { + const code = ` + import '@tolgee/react' + const NS = { AUTH: 'auth' } as const + function Test () { + const { t } = useTranslate(NS.BILLING) + t('key1') + } + `; + + const extracted = await extractReactKeys(code, FILE_NAME); + expect(extracted.warnings).toEqual([ + { warning: 'W_DYNAMIC_NAMESPACE', line: 5 }, + { warning: 'W_UNRESOLVABLE_NAMESPACE', line: 6 }, + ]); + expect(extracted.keys).toEqual([]); + }); + + it('still emits a warning for identifiers without a const declaration', async () => { + const code = ` + import '@tolgee/react' + function Test () { + const { t } = useTranslate(unknownNs) + t('key1') + } + `; + + const extracted = await extractReactKeys(code, FILE_NAME); + expect(extracted.warnings).toEqual([ + { warning: 'W_DYNAMIC_NAMESPACE', line: 4 }, + { warning: 'W_UNRESOLVABLE_NAMESPACE', line: 5 }, + ]); + expect(extracted.keys).toEqual([]); + }); + }); });