Optimized find all references and reduced memory usage in VS (#8339)

* Added ItemKey.fsi/fsi. Added blank SemanticClassification.fs/fsi.

* Raise disposed exception

* Re-worked semantic classification. Renamed ItemKeyReader to ItemKeyStore. Exposing ItemKeyStore/Builder

* Fixing build

* Storing semantic classification

* Caching semantic classification

* Wiring it up

* Need to fix lexing

* Added experimental lexing API to handle find all refs syntactic classification from allocating a lot

* Added System.Memory

* Using Span to check equality without allocating

* Allocate less

* Fixing build. Reducing more allocations and not using lex filter on lexing tokens.

* Remove langversion

* Fixed record find all refs

* Fixing test

* Partial match for active pattern

* Feedback changes

* Added comment on TcResolutionsExtensions

* Creating view accessor when needed in ItemKey. Fixed UnionCase find all refs.

* Added comment on warning

* Added Range.comparer. Moving opens to top of file

* More feedback changes

* Added comment on sliding expiration

Author: TIHan

eng/Versions.props

@@ -96,6 +96,7 @@
     <SystemThreadingThreadPoolVersion>4.3.0</SystemThreadingThreadPoolVersion>
     <SystemValueTupleVersion>4.5.0</SystemValueTupleVersion>
     <SystemBuffersVersion>4.5.0</SystemBuffersVersion>
+    <SystemMemoryVersion>4.5.3</SystemMemoryVersion>
     <!-- Roslyn packages -->
     <MicrosoftCodeAnalysisEditorFeaturesVersion>$(RoslynVersion)</MicrosoftCodeAnalysisEditorFeaturesVersion>
     <MicrosoftCodeAnalysisEditorFeaturesTextVersion>$(RoslynVersion)</MicrosoftCodeAnalysisEditorFeaturesTextVersion>

fcs/FSharp.Compiler.Service.Tests/SemanticColorizationServiceTests.fs

@@ -30,7 +30,7 @@ type SemanticClassificationServiceTests() =
     let checker = FSharpChecker.Create()
     let perfOptions = { LanguageServicePerformanceOptions.Default with AllowStaleCompletionResults = false }
 
-    let getRanges (source: string) : (Range.range * SemanticClassificationType) list =
+    let getRanges (source: string) : struct (Range.range * SemanticClassificationType) list =
         asyncMaybe {
 
             let! _, _, checkFileResults = checker.ParseAndCheckDocument(filePath, 0, SourceText.From(source), projectOptions, perfOptions, "")
@@ -45,7 +45,7 @@ type SemanticClassificationServiceTests() =
         let ranges = getRanges fileContents
         let line = text.Lines.GetLinePosition (fileContents.IndexOf(marker) + marker.Length - 1)
         let markerPos = Range.mkPos (Range.Line.fromZ line.Line) (line.Character + marker.Length - 1)
-        match ranges |> List.tryFind (fun (range, _) -> Range.rangeContainsPos range markerPos) with
+        match ranges |> List.tryFind (fun struct (range, _) -> Range.rangeContainsPos range markerPos) with
         | None -> Assert.Fail("Cannot find colorization data for end of marker")
         | Some(_, ty) -> Assert.AreEqual(classificationType, FSharpClassificationTypes.getClassificationTypeName ty, "Classification data doesn't match for end of marker")
 
@@ -54,7 +54,7 @@ type SemanticClassificationServiceTests() =
         let ranges = getRanges fileContents
         let line = text.Lines.GetLinePosition (fileContents.IndexOf(marker) + marker.Length - 1)
         let markerPos = Range.mkPos (Range.Line.fromZ line.Line) (line.Character + marker.Length - 1)
-        let anyData = ranges |> List.exists (fun (range, sct) -> Range.rangeContainsPos range markerPos && ((FSharpClassificationTypes.getClassificationTypeName sct) = classificationType))
+        let anyData = ranges |> List.exists (fun struct (range, sct) -> Range.rangeContainsPos range markerPos && ((FSharpClassificationTypes.getClassificationTypeName sct) = classificationType))
         Assert.False(anyData, "Classification data was found when it wasn't expected.")
 
     [<TestCase("(*1*)", FSharpClassificationTypes.ValueType)>]

fcs/FSharp.Compiler.Service/FSharp.Compiler.Service.fsproj

@@ -576,6 +576,18 @@
     <Compile Include="$(FSharpSourcesRoot)\fsharp\service\Reactor.fs">
       <Link>Service/Reactor.fs</Link>
     </Compile>
+    <Compile Include="$(FSharpSourcesRoot)\fsharp\service\SemanticClassification.fsi">
+      <Link>Service/SemanticClassification.fsi</Link>
+    </Compile>
+    <Compile Include="$(FSharpSourcesRoot)\fsharp\service\SemanticClassification.fs">
+      <Link>Service/SemanticClassification.fs</Link>
+    </Compile>
+    <Compile Include="$(FSharpSourcesRoot)\fsharp\service\ItemKey.fsi">
+      <Link>Service/ItemKey.fsi</Link>
+    </Compile>
+    <Compile Include="$(FSharpSourcesRoot)\fsharp\service\ItemKey.fs">
+      <Link>Service/ItemKey.fs</Link>
+    </Compile>
     <Compile Include="$(FSharpSourcesRoot)\fsharp\service\IncrementalBuild.fsi">
       <Link>Service/IncrementalBuild.fsi</Link>
     </Compile>
@@ -684,6 +696,7 @@
     <PackageReference Include="System.Collections.Immutable" Version="1.5.0" />
     <PackageReference Include="System.Reflection.Metadata" Version="1.6.0" />
     <PackageReference Include="System.Buffers" Version="4.5.0" />
+    <PackageReference Include="System.Memory" Version="4.5.3" />
   </ItemGroup>
   <ItemGroup Condition="'$(TargetFramework)' == 'netstandard2.0'">
     <PackageReference Include="System.Diagnostics.Process" Version="4.1.0" />

src/fsharp/NameResolution.fs

@@ -1757,6 +1757,9 @@ type TcResultsSinkImpl(g, ?sourceText: ISourceText) =
     member this.GetOpenDeclarations() =
         capturedOpenDeclarations |> Seq.distinctBy (fun x -> x.Range, x.AppliedScope, x.IsOwnNamespace) |> Seq.toArray
 
+    member this.GetFormatSpecifierLocations() =
+        capturedFormatSpecifierLocations.ToArray()
+
     interface ITypecheckResultsSink with
         member sink.NotifyEnvWithScope(m, nenv, ad) =
             if allowedRange m then

src/fsharp/NameResolution.fsi

@@ -438,6 +438,9 @@ type internal TcResultsSinkImpl =
     /// Get all open declarations reported to the sink
     member GetOpenDeclarations : unit -> OpenDeclaration[]
 
+    /// Get the format specifier locations
+    member GetFormatSpecifierLocations : unit -> (range * int)[]
+
     interface ITypecheckResultsSink
 
 /// An abstract type for reporting the results of name resolution and type checking, and which allows

src/fsharp/lexhelp.fs

@@ -36,8 +36,8 @@ type LightSyntaxStatus(initial:bool,warn:bool) =
 
 /// Manage lexer resources (string interning)
 [<Sealed>]
-type LexResourceManager() =
-    let strings = new System.Collections.Generic.Dictionary<string, Parser.token>(1024)
+type LexResourceManager(?capacity: int) =
+    let strings = new System.Collections.Generic.Dictionary<string, Parser.token>(defaultArg capacity 1024)
     member x.InternIdentifierToken(s) = 
         match strings.TryGetValue s with
         | true, res -> res

src/fsharp/lexhelp.fsi

@@ -25,7 +25,7 @@ type LightSyntaxStatus =
 
 [<Sealed>]
 type LexResourceManager =
-    new : unit -> LexResourceManager
+    new : ?capacity: int -> LexResourceManager
 
 type lexargs =
     { defines: string list

src/fsharp/range.fs

@@ -5,6 +5,7 @@ module FSharp.Compiler.Range
 
 open System
 open System.IO
+open System.Collections.Generic
 open System.Collections.Concurrent
 open Microsoft.FSharp.Core.Printf
 open FSharp.Compiler.AbstractIL.Internal.Library
@@ -380,4 +381,9 @@ module Range =
 
     let toFileZ (m:range) = m.FileName, toZ m
 
+    let comparer = 
+        { new IEqualityComparer<range> with 
+            member _.Equals(x1, x2) = equals x1 x2 
+            member _.GetHashCode o = o.GetHashCode() }
+
 

src/fsharp/range.fsi

@@ -200,3 +200,6 @@ module Range =
 
     /// Convert a range from one-based line counting (used internally in the F# compiler and in F# error messages) to zero-based line counting (used by Visual Studio)
     val toFileZ : range -> string * Range01
+
+    /// Equality comparer for range.
+    val comparer : IEqualityComparer<range>

src/fsharp/service/FSharpCheckerResults.fs

@@ -109,24 +109,6 @@ type GetPreciseCompletionListFromExprTypingsResult =
     | Some of (ItemWithInst list * DisplayEnv * range) * TType
 
 type Names = string list 
-
-[<RequireQualifiedAccess>]
-type SemanticClassificationType =
-    | ReferenceType
-    | ValueType
-    | UnionCase
-    | Function
-    | Property
-    | MutableVar
-    | Module
-    | Printf
-    | ComputationExpression
-    | IntrinsicFunction
-    | Enumeration
-    | Interface
-    | TypeArgument
-    | Operator
-    | Disposable
 
 /// A TypeCheckInfo represents everything we get back from the typecheck of a file.
 /// It acts like an in-memory database about the file.
@@ -155,8 +137,6 @@ type internal TypeCheckInfo
            openDeclarations: OpenDeclaration[]) = 
 
     let textSnapshotInfo = defaultArg textSnapshotInfo null
-    let (|CNR|) (cnr:CapturedNameResolution) =
-        (cnr.Pos, cnr.Item, cnr.ItemOccurence, cnr.DisplayEnv, cnr.NameResolutionEnv, cnr.AccessorDomain, cnr.Range)
 
     // These strings are potentially large and the editor may choose to hold them for a while.
     // Use this cache to fold together data tip text results that are the same. 
@@ -1262,133 +1242,8 @@ type internal TypeCheckInfo
     member __.GetFormatSpecifierLocationsAndArity() = 
          sSymbolUses.GetFormatSpecifierLocationsAndArity()
 
-    member __.GetSemanticClassification(range: range option) : (range * SemanticClassificationType) [] =
-      ErrorScope.Protect Range.range0 
-       (fun () -> 
-        let (|LegitTypeOccurence|_|) = function
-            | ItemOccurence.UseInType
-            | ItemOccurence.UseInAttribute
-            | ItemOccurence.Use _
-            | ItemOccurence.Binding _
-            | ItemOccurence.Pattern _ -> Some()
-            | _ -> None
-
-        let (|OptionalArgumentAttribute|_|) ttype =
-            match ttype with
-            | TType.TType_app(tref, _) when tref.Stamp = g.attrib_OptionalArgumentAttribute.TyconRef.Stamp -> Some()
-            | _ -> None
-
-        let (|KeywordIntrinsicValue|_|) (vref: ValRef) =
-            if valRefEq g g.raise_vref vref ||
-               valRefEq g g.reraise_vref vref ||
-               valRefEq g g.typeof_vref vref ||
-               valRefEq g g.typedefof_vref vref ||
-               valRefEq g g.sizeof_vref vref ||
-               valRefEq g g.nameof_vref vref
-            then Some()
-            else None
-        
-        let (|EnumCaseFieldInfo|_|) (rfinfo : RecdFieldInfo) =
-            match rfinfo.TyconRef.TypeReprInfo with
-            | TFSharpObjectRepr x ->
-                match x.fsobjmodel_kind with
-                | TTyconEnum -> Some ()
-                | _ -> None
-            | _ -> None
-
-        let resolutions =
-            match range with
-            | Some range ->
-                sResolutions.CapturedNameResolutions
-                |> Seq.filter (fun cnr -> rangeContainsPos range cnr.Range.Start || rangeContainsPos range cnr.Range.End)
-            | None -> 
-                sResolutions.CapturedNameResolutions :> seq<_>
-
-        let isDisposableTy (ty: TType) =
-            protectAssemblyExplorationNoReraise false false (fun () -> Infos.ExistsHeadTypeInEntireHierarchy g amap range0 ty g.tcref_System_IDisposable)
-
-        let isStructTyconRef (tyconRef: TyconRef) = 
-            let ty = generalizedTyconRef tyconRef
-            let underlyingTy = stripTyEqnsAndMeasureEqns g ty
-            isStructTy g underlyingTy
-
-        let isValRefMutable (vref: ValRef) =
-            // Mutable values, ref cells, and non-inref byrefs are mutable.
-            vref.IsMutable
-            || Tastops.isRefCellTy g vref.Type
-            || (Tastops.isByrefTy g vref.Type && not (Tastops.isInByrefTy g vref.Type))
-
-        let isRecdFieldMutable (rfinfo: RecdFieldInfo) =
-            (rfinfo.RecdField.IsMutable && rfinfo.LiteralValue.IsNone)
-            || Tastops.isRefCellTy g rfinfo.RecdField.FormalType
-
-        resolutions
-        |> Seq.choose (fun cnr ->
-            match cnr with
-            // 'seq' in 'seq { ... }' gets colored as keywords
-            | CNR(_, (Item.Value vref), ItemOccurence.Use, _, _, _, m) when valRefEq g g.seq_vref vref ->
-                Some (m, SemanticClassificationType.ComputationExpression)
-            | CNR(_, (Item.Value vref), _, _, _, _, m) when isValRefMutable vref ->
-                Some (m, SemanticClassificationType.MutableVar)
-            | CNR(_, Item.Value KeywordIntrinsicValue, ItemOccurence.Use, _, _, _, m) ->
-                Some (m, SemanticClassificationType.IntrinsicFunction)
-            | CNR(_, (Item.Value vref), _, _, _, _, m) when isFunction g vref.Type ->
-                if valRefEq g g.range_op_vref vref || valRefEq g g.range_step_op_vref vref then 
-                    None
-                elif vref.IsPropertyGetterMethod || vref.IsPropertySetterMethod then
-                    Some (m, SemanticClassificationType.Property)
-                elif IsOperatorName vref.DisplayName then
-                    Some (m, SemanticClassificationType.Operator)
-                else
-                    Some (m, SemanticClassificationType.Function)
-            | CNR(_, Item.RecdField rfinfo, _, _, _, _, m) when isRecdFieldMutable rfinfo ->
-                Some (m, SemanticClassificationType.MutableVar)
-            | CNR(_, Item.RecdField rfinfo, _, _, _, _, m) when isFunction g rfinfo.FieldType ->
-               Some (m, SemanticClassificationType.Function)
-            | CNR(_, Item.RecdField EnumCaseFieldInfo, _, _, _, _, m) ->
-                Some (m, SemanticClassificationType.Enumeration)
-            | CNR(_, Item.MethodGroup _, _, _, _, _, m) ->
-                Some (m, SemanticClassificationType.Function)
-            // custom builders, custom operations get colored as keywords
-            | CNR(_, (Item.CustomBuilder _ | Item.CustomOperation _), ItemOccurence.Use, _, _, _, m) ->
-                Some (m, SemanticClassificationType.ComputationExpression)
-            // types get colored as types when they occur in syntactic types or custom attributes
-            // type variables get colored as types when they occur in syntactic types custom builders, custom operations get colored as keywords
-            | CNR(_, Item.Types (_, [OptionalArgumentAttribute]), LegitTypeOccurence, _, _, _, _) -> None
-            | CNR(_, Item.CtorGroup(_, [MethInfo.FSMeth(_, OptionalArgumentAttribute, _, _)]), LegitTypeOccurence, _, _, _, _) -> None
-            | CNR(_, Item.Types(_, types), LegitTypeOccurence, _, _, _, m) when types |> List.exists (isInterfaceTy g) -> 
-                Some (m, SemanticClassificationType.Interface)
-            | CNR(_, Item.Types(_, types), LegitTypeOccurence, _, _, _, m) when types |> List.exists (isStructTy g) -> 
-                Some (m, SemanticClassificationType.ValueType)
-            | CNR(_, Item.Types(_, TType_app(tyconRef, TType_measure _ :: _) :: _), LegitTypeOccurence, _, _, _, m) when isStructTyconRef tyconRef ->
-                Some (m, SemanticClassificationType.ValueType)
-            | CNR(_, Item.Types(_, types), LegitTypeOccurence, _, _, _, m) when types |> List.exists isDisposableTy ->
-                Some (m, SemanticClassificationType.Disposable)
-            | CNR(_, Item.Types _, LegitTypeOccurence, _, _, _, m) -> 
-                Some (m, SemanticClassificationType.ReferenceType)
-            | CNR(_, (Item.TypeVar _ ), LegitTypeOccurence, _, _, _, m) ->
-                Some (m, SemanticClassificationType.TypeArgument)
-            | CNR(_, Item.UnqualifiedType tyconRefs, LegitTypeOccurence, _, _, _, m) ->
-                if tyconRefs |> List.exists (fun tyconRef -> tyconRef.Deref.IsStructOrEnumTycon) then
-                    Some (m, SemanticClassificationType.ValueType)
-                else Some (m, SemanticClassificationType.ReferenceType)
-            | CNR(_, Item.CtorGroup(_, minfos), LegitTypeOccurence, _, _, _, m) ->
-                if minfos |> List.exists (fun minfo -> isStructTy g minfo.ApparentEnclosingType) then
-                    Some (m, SemanticClassificationType.ValueType)
-                else Some (m, SemanticClassificationType.ReferenceType)
-            | CNR(_, Item.ExnCase _, LegitTypeOccurence, _, _, _, m) ->
-                Some (m, SemanticClassificationType.ReferenceType)
-            | CNR(_, Item.ModuleOrNamespaces refs, LegitTypeOccurence, _, _, _, m) when refs |> List.exists (fun x -> x.IsModule) ->
-                Some (m, SemanticClassificationType.Module)
-            | CNR(_, (Item.ActivePatternCase _ | Item.UnionCase _ | Item.ActivePatternResult _), _, _, _, _, m) ->
-                Some (m, SemanticClassificationType.UnionCase)
-            | _ -> None)
-        |> Seq.toArray
-        |> Array.append (sSymbolUses.GetFormatSpecifierLocationsAndArity() |> Array.map (fun m -> fst m, SemanticClassificationType.Printf))
-       ) 
-       (fun msg -> 
-           Trace.TraceInformation(sprintf "FCS: recovering from error in GetSemanticClassification: '%s'" msg)
-           Array.empty)
+    member __.GetSemanticClassification(range: range option) : struct (range * SemanticClassificationType) [] =
+        sResolutions.GetSemanticClassification(g, amap, sSymbolUses.GetFormatSpecifierLocationsAndArity(), range)
 
     /// The resolutions in the file
     member __.ScopeResolutions = sResolutions