fix(types): enhance type system with mapped types and reverse-mapped …

Xantham.slnx

@@ -49,6 +49,7 @@
     <File Path="docs/Xantham.Decoder/index.md" />
     <File Path="docs/Xantham.Decoder/names.fsx" />
     <File Path="docs/Xantham.Decoder/runtime.fsx" />
+    <File Path="docs/Xantham.Decoder/source-metadata-migration.md" />
   </Folder>
   <Folder Name="/docs/Xantham.Generator/">
     <File Path="docs/Xantham.Generator/generator-context.md" />

src/Xantham.Common/Common.Types.fs

@@ -64,10 +64,14 @@ type ExportPath = ExportPath of string with
 type Export = Map<ExportPath, ExportValue>
 
 [<Struct>]
-type PackageId = PackageId of name: string * version: string
+type PackageId = PackageId of name: string * version: string with
+    member inline this.Name = let (PackageId (name, _)) = this in name
+    member inline this.Version = let (PackageId (_, version)) = this in version
 
 [<Struct>]
-type SubModuleId = SubModuleId of packageId: PackageId * subModuleName: string
+type SubModuleId = SubModuleId of packageId: PackageId * subModuleName: string with
+    member inline this.PackageId = let (SubModuleId (packageId, _)) = this in packageId
+    member inline this.SubModuleName = let (SubModuleId (_, subModuleName)) = this in subModuleName
 
 [<Struct>]
 type Package = {

src/Xantham.Decoder/Types/Arena.Interner.fs

@@ -469,7 +469,8 @@ type ArenaInterner = {
     ResolvedSubModules: IDictionary<SubModuleId, ResolvedSubModule>
     ResolvedExportPoints: IDictionary<Xantham.ExportPoint, ResolvedExportPoint>
     /// Map from source module path to the list of resolved exports declared in that module.
-    ExportMap: Map<Xantham.Source, LazyResolvedExport>
+    /// Note: Lists should be singletons where `Xantham.Source.IsPackage`.
+    ExportMap: Map<Xantham.Source, LazyResolvedExport list>
     /// <summary>
     /// WARNING: Evaluation of the graph can be expensive.<br/>
     /// It is useful only when used in combination with the resolve type and resolve export
@@ -999,6 +1000,10 @@ module ArenaInterner =
                 | TsExportDeclaration.Function funs -> funs.ValueOrHead.Metadata.Source
                 , (lazyResolveExport key).Value
                 )
+            |> Seq.groupBy fst
+            |> Seq.map (fun (source, values) ->
+                source, values |> Seq.map snd |> List.ofSeq
+                )
             |> Map.ofSeq
         {
             ResolveType = resolve

src/Xantham.Decoder/Utils.fs

@@ -303,11 +303,13 @@ let private compressWithMap (types: Map<TypeKey, TsType>) (compressions: Diction
         | TsType.Union (TsTypeUnion values) ->
             values
             |> List.map swap
+            |> List.sort
             |> TsTypeUnion
             |> TsType.Union
         | TsType.Intersection (TsTypeIntersection values) ->
             values
             |> List.map swap
+            |> List.sort
             |> TsTypeIntersection
             |> TsType.Intersection
         | TsType.IndexedAccess tsIndexAccessType ->

src/Xantham.Fable/Read.fs

@@ -2,6 +2,7 @@
 module Xantham.Fable.Main
 
 open Fable.Core.DynamicExtensions
+open Fable.Core.JsInterop
 open Node
 open Thoth.Json
 open Xantham
@@ -460,6 +461,13 @@ let read (reader: TypeScriptReader) =
     |> Internal.trimTypeReferenceArrayTupleDuplicates
     |> Internal.mergeExports
     |> Internal.selectAndMergeWinnersInDuplicates
+    |> fun result ->
+        reader.program.getRootFileNames().AsArray
+        |> Array.head
+        |> fun file ->
+            fs.unlinkSync(!^file)
+            fs.rmdirSync(!^(path.dirname file))
+        result
 let write (outputDestination: string) (result: EncodedResult) =
     Internal.writeOutput outputDestination result
 let readAndWrite (outputDestination: string) (reader: TypeScriptReader) =

src/Xantham.Fable/Reading/Dispatch/TypeDeclaration.fs

@@ -420,7 +420,10 @@ let dispatch (ctx: TypeScriptReader) (xanTag: XanthamTag) (node: TypeDeclaration
                 sourceTag.SubModuleId.Value
                 |> Source.PackageInternal
             | _ ->
+                #if !FABLE_TEST
+                // This log will reoccur continuously in the test environment.
                 Log.error "Invariant: a declaration was not identified as a lib-es decl, had no export collection, and no submodule id. Defaulting Metadata to Source.LibEs."
+                #endif
                 sourceTag.Guard.Source.fileName
                 |> Node.Api.path.basename
                 |> Source.LibEs

src/Xantham.Fable/Reading/Dispatch/TypeFlagObject.fs

@@ -217,28 +217,40 @@ let dispatch (ctx: TypeScriptReader) (xanTag: XanthamTag) (tag: TypeFlagObject)
     | TypeFlagObject.Mapped mappedType ->
         nameof TypeFlagObject.Mapped |> debugLocation
         // Try enumerating concrete properties first; fall back to a string-index for generic cases.
+        let decl = mappedType.declaration
         let props = ctx.checker.getPropertiesOfType(mappedType).AsArray
         let members =
             if props.Length > 0 then
                 Array.map (propertySymToMemberSlot ctx) props
             else
-                // Fully generic mapped type — emit { [key: string]: any } as a safe fallback
+                let keyType =
+                    mappedType.nameType
+                    |> Option.orElse mappedType.constraintType
+                    |> Option.orElse (mappedType.typeParameter |> Option.bind _.getConstraint())
+                    |> Option.filter (_.TypeKey >> (<>) mappedType.TypeKey)
+                    |> Option.map (pushTypeToStack ctx >> _.TypeSignal)
+                let valueType =
+                    mappedType.templateType
+                    |> Option.filter (_.TypeKey >> (<>) mappedType.TypeKey)
+                    |> Option.map (pushTypeToStack ctx >> _.TypeSignal)
+                let isReadOnly = decl.readonlyToken.IsSome
+                let isOptional = decl.questionToken.IsSome
                 [|
                     {
                         SIndexSignatureBuilder.Parameters =
                             [|
                                 {
                                     SParameterBuilder.Name = "key"
-                                    IsOptional = false
+                                    IsOptional = isOptional
                                     IsSpread = false
-                                    Type = TypeSignal.ofKey TypeKindPrimitive.String.TypeKey
+                                    Type = keyType |> Option.defaultValue (TypeSignal.ofKey TypeKindPrimitive.String.TypeKey)
                                     Documentation = []
                                 }
                                 |> ValueSome
                                 |> Signal.source
                             |]
-                        Type = TypeSignal.ofKey TypeKindPrimitive.Any.TypeKey
-                        IsReadOnly = false
+                        Type = valueType |> Option.defaultValue (TypeSignal.ofKey TypeKindPrimitive.Any.TypeKey)
+                        IsReadOnly = isReadOnly
                     }
                     |> SMemberBuilder.IndexSignature
                     |> ValueSome
@@ -252,8 +264,8 @@ let dispatch (ctx: TypeScriptReader) (xanTag: XanthamTag) (tag: TypeFlagObject)
         mappedType.TypeKey
         |> setTypeKeyForTag xanTag
 
-    | TypeFlagObject.Instantiated objType ->
-        nameof TypeFlagObject.Instantiated |> debugLocation
+    | TypeFlagObject.ReverseMapped objType ->
+        nameof TypeFlagObject.ReverseMapped |> debugLocation
         // A generic type applied to concrete arguments — same shape as Reference.
         // If the target equals itself (uninstantiated), forward to the declaration instead.
         let typeRef = unbox<Ts.TypeReference> objType

src/Xantham.Fable/Reading/Prelude.fs

@@ -382,6 +382,8 @@ let pushToStack (ctx: TypeScriptReader) (tag: XanthamTag) =
     tag.Builder |> ignore
     tag.TypeSignal |> ignore
     ctx.stack.Push tag
+let pushTypeToStack (ctx: TypeScriptReader) (node: Ts.Type) =
+    ctx.CreateXanthamTag node |> fst |> stackPushAndThen ctx id
 let arrayHasModifier (modifier: Modifiers -> bool) (arr: ResizeArray<Ts.Modifier>) =
     arr.AsArray |> Array.exists (Modifiers.Create >> modifier)
 let optionArrayHasModifier (modifier: Modifiers -> bool) (arr: ResizeArray<Ts.Modifier> option) =

src/Xantham.Fable/Types/ReactiveBuilders.fs

@@ -491,19 +491,22 @@ type SConditionalTypeBuilder = {
           True = this.True.Value; False = this.False.Value }
 
 
+// TypeUnion and TypeIntersection are commutative; order the keys before assembling.
+// The decoder already has a phase which will dedupe if the order is the same.
+
 /// Signal-based equivalent of <c>TsTypeUnionBuilder</c>, builds to <see cref="T:Xantham.TsTypeUnion"/>.
 type STypeUnionBuilder = {
     Types: TypeSignal array
 } with
     member this.Build() : TsTypeUnion =
-        TsTypeUnion (this.Types |> Array.map _.Value |> Array.toList)
+        TsTypeUnion (this.Types |> Array.map _.Value |> Array.sort |> Array.toList)
 
 /// Signal-based equivalent of <c>TsTypeIntersectionBuilder</c>, builds to <see cref="T:Xantham.TsTypeIntersection"/>.
 type STypeIntersectionBuilder = {
     Types: TypeSignal array
 } with
     member this.Build() : TsTypeIntersection =
-        TsTypeIntersection (this.Types |> Array.map _.Value |> Array.toList)
+        TsTypeIntersection (this.Types |> Array.map _.Value |> Array.sort |> Array.toList)
 
 /// Signal-based equivalent of <c>TsTupleElementTypeBuilder</c>, builds to <see cref="T:Xantham.TsTupleElementType"/>.
 type STupleElementTypeBuilder = {

src/Xantham.Fable/Types/Reader.fs

@@ -9,30 +9,37 @@ open Xantham.Fable.Types
 open Xantham.Fable.Types.Tracer
 
 let private commonCompilerOptions = jsOptions<Ts.CompilerOptions>(fun c ->
+    // c.traceResolution <- Some true
     c.moduleResolution <- Some Ts.ModuleResolutionKind.Bundler
     c.target <- Some Ts.ScriptTarget.Latest
     c.skipLibCheck <- Some true
     c.declaration <- Some true
     c.emitDeclarationOnly <- Some true
+    c.resolveJsonModule <- Some true
     // Without this, unions with null and undefined are reduced out when resolving
     // a type node to a type. This increases workaround logic.
     c.strictNullChecks <- Some true
     c.resolvePackageJsonExports <- Some true
     c.resolvePackageJsonImports <- Some true)
 
-let private createProgram (entryFile: string): Ts.Program =
-    let entryFile = String.normalizePath entryFile
-    ts.createProgram(jsOptions<Ts.CreateProgramOptions> (fun o ->
-        o.rootNames <- ResizeArray [| entryFile |]
-        o.options <- commonCompilerOptions
-        ))
-
-let private createProgramForFiles (entryFiles: string array): Ts.Program =
+let private createProgramForFiles (entryFiles: string array) =
     let entryFiles = entryFiles |> Array.map String.normalizePath
-    ts.createProgram(jsOptions<Ts.CreateProgramOptions>(fun o ->
-        o.rootNames <- ResizeArray entryFiles
-        o.options <- commonCompilerOptions
-        ))
+    let xanthamTempDir = Node.Api.fs.mkdtempSync("xantham_")
+    let tempFilePath = Node.Api.path.join(xanthamTempDir, "temp.d.ts")
+    Node.Api.fs.writeFileSync(tempFilePath, String.concat "\n" <| [
+        for entryFile in entryFiles do
+            "import * as _ from '" + entryFile + "';"
+    ])
+    {|
+        TempFilePath = tempFilePath
+        Program =
+            ts.createProgram(jsOptions<Ts.CreateProgramOptions>(fun o ->
+                o.rootNames <- ResizeArray [
+                    tempFilePath
+                ]
+                o.options <- commonCompilerOptions
+                ))
+    |}
 
 [<ReferenceEquality>]
 type TypeScriptReader = {
@@ -45,7 +52,9 @@ type TypeScriptReader = {
     ExportCache: Dictionary<IdentityKey, ExportStore>
     MemberCache: Dictionary<IdentityKey, MemberStore>
     LibCache: HashSet<IdentityKey>
+    TempFilePath: string
 } with
+    member inline this.tempFilePath = this.TempFilePath
     member inline this.stack = this.Stack
     member inline this.entryFile = this.EntryFiles |> Array.head
     member inline this.entryFiles = this.EntryFiles
@@ -65,7 +74,10 @@ module TypeScriptReader =
         let signalCache = Dictionary()
         let memberCache = Dictionary()
         let libCache = HashSet()
-        let program = createProgramForFiles entryFiles
+        let tempFile, program =
+            let programResult = createProgramForFiles entryFiles
+            programResult.TempFilePath, programResult.Program
+
         let checker = program.getTypeChecker()
         let exportCache = Dictionary()
         #if DEBUG && !FABLE_TEST
@@ -81,6 +93,7 @@ module TypeScriptReader =
             LibCache = libCache
             MemberCache = memberCache
             ExportCache = exportCache
+            TempFilePath = tempFile
         }
     let inline create (entryFile: string) =
         createFor [| entryFile |]

src/Xantham.Fable/Types/SourceTag.fs

@@ -458,6 +458,7 @@ type Ts.Program with
             | _ -> ()
             )
         this.getSourceFiles().AsArray
+        |> Array.filter (_.fileName >> (<>) this.getRootFileNames().AsArray[0])
         |> Array.map (fun sf -> SourceTag.CreateValue(this, sf))
         |> Array.filter _.Value.IsAmbient
         |> Array.iter (fun tag ->
@@ -467,7 +468,11 @@ type Ts.Program with
                 tag.Guard.PackageJsonContent
                 |> ValueOption.bind (_.name >> Option.toValueOption)
                 |> ValueOption.defaultWith (fun () ->
+                    #if !FABLE_TEST
                     failwith "Invariant: Ambient source has no associated package name."
+                    #else
+                    Node.Api.path.basename tag.Guard.Source.fileName
+                    #endif
                     )
             let packageVersion =
                 tag.Guard.PackageJsonContent

src/Xantham.Fable/Types/XanTagKind.fs

@@ -180,6 +180,11 @@ module private Internal =
             Ts.TypeFlags.NumberLiteral >-> TypeFlagLiteral.Number
             Ts.TypeFlags.StringLiteral >-> TypeFlagLiteral.String
         |]
+    /// <summary>
+    /// <a href="https://github.com/microsoft/TypeScript/blob/v5.9.3/src/compiler/types.ts#L6513">
+    /// Unique identifiers for type of object.
+    /// </a>
+    /// </summary>
     let typeFlagObjectKindSet: (Ts.ObjectFlags * (obj -> TypeFlagObject))[] =
         [|
             Ts.ObjectFlags.Class >-> TypeFlagObject.Class
@@ -188,7 +193,7 @@ module private Internal =
             Ts.ObjectFlags.Reference >-> TypeFlagObject.Reference
             Ts.ObjectFlags.Anonymous >-> TypeFlagObject.Anonymous
             Ts.ObjectFlags.Mapped >-> TypeFlagObject.Mapped
-            Ts.ObjectFlags.Instantiated >-> TypeFlagObject.Instantiated
+            Ts.ObjectFlags.ReverseMapped >-> TypeFlagObject.ReverseMapped
             Ts.ObjectFlags.EvolvingArray >-> TypeFlagObject.EvolvingArray
         |]
     #nowarn 40
@@ -1167,7 +1172,7 @@ type TypeFlagObject =
     /// <c>checker.getPropertiesOfType</c> and <c>checker.getSignaturesOfType</c> to enumerate members.
     /// Example: the type <c>{ x: number; y: number }</c>.
     /// </remarks>
-    | Anonymous of Ts.ObjectType
+    | Anonymous of AnonymousType
     /// <summary>
     /// A mapped type at the checker layer: <c>{ [K in keyof T]: T[K] }</c>.
     /// </summary>
@@ -1177,7 +1182,7 @@ type TypeFlagObject =
     /// instantiations. Inspect <c>mappedType.declaration</c> to navigate back to the AST node.
     /// Example: the type produced by <c>{ [K in keyof T]?: T[K] }</c>.
     /// </remarks>
-    | Mapped of Ts.ObjectType
+    | Mapped of MappedType
     /// <summary>
     /// An instantiated object type — a generic type applied to concrete type arguments.
     /// </summary>
@@ -1186,7 +1191,7 @@ type TypeFlagObject =
     /// a generic interface or class is specialised. The target type and type arguments can be accessed
     /// via the <c>Ts.TypeReference</c> cast on the object. Example: <c>Foo&lt;string&gt;</c> resolved.
     /// </remarks>
-    | Instantiated of Ts.ObjectType
+    | ReverseMapped of ReverseMappedType
     /// <summary>
     /// A type reference object type — a generic type applied with explicit type arguments, at the
     /// checker layer.

src/Xantham.Fable/Utils/TypeScript.Extensions.fs

@@ -51,7 +51,44 @@ type Ts.SourceFile with
 type Ts.Type with
     [<EmitProperty "id">]
     member inline this.TypeKey: TypeKey = jsNative
+
+type TypeMapKind =
+    | Simple = 0
+    | Array = 1
+    | Deferred = 2
+    | Function = 3
+    | Composite = 4
+    | Merged = 5
+[<TypeScriptTaggedUnion "kind"; RequireQualifiedAccess>]
+type TypeMapper =
+    | [<CompiledValue(TypeMapKind.Simple)>] Simple of source: Ts.Type * target: Ts.Type
+    | [<CompiledValue(TypeMapKind.Array)>] Array of sources: Ts.Type array * targets: Ts.Type array option
+    | [<CompiledValue(TypeMapKind.Deferred)>] Deferred of sources: Ts.Type array * targets: (unit -> Ts.Type) array
+    | [<CompiledValue(TypeMapKind.Function)>] Function of func: (Ts.Type -> Ts.Type) * debugInfo: (unit -> string) option
+    | [<CompiledValue(TypeMapKind.Composite)>] Composite of mapper1: TypeMapper * mapper2: TypeMapper
+    | [<CompiledValue(TypeMapKind.Merged)>] Merged of mapper1: TypeMapper * mapper2: TypeMapper
 
+type AnonymousType =
+    inherit Ts.ObjectType
+    abstract target: AnonymousType option
+    abstract mapper: TypeMapper option
+    abstract instantiations: JS.Map<string, Ts.Type> option
+type MappedType =
+    inherit AnonymousType
+    abstract declaration: Ts.MappedTypeNode
+    abstract typeParameter: Ts.TypeParameter option
+    abstract constraintType: Ts.Type option
+    abstract nameType: Ts.Type option
+    abstract templateType: Ts.Type option
+    abstract modifiersType: Ts.Type option
+    abstract resolvedApparentType: Ts.Type option
+    abstract containsError: bool option
+type ReverseMappedType =
+    inherit AnonymousType
+    abstract source: Ts.Type
+    abstract mappedType: MappedType
+    abstract constraintType: Ts.IndexType
+
 type HasTypeArguments =
     | CallExpression of Ts.CallExpression
     | NewExpression of Ts.NewExpression