From d856384de34f8a1e47762a7d452f33af3b88a3cc Mon Sep 17 00:00:00 2001 From: nielsbosma Date: Tue, 23 Jun 2026 19:50:03 +0200 Subject: [PATCH] Improve manifest reach, detection precision, and language accuracy Code-level follow-ups to the detector-coverage work, all gated on the stack-hash quality bar (move a real slot or remove a false positive). 21 new tests; no existing fixture detection changed. Manifest reach & precision - Apply the per-manifest dependency cap only to the reported output; the rule engine now sees the full set (was silently dropping hash-significant deps). - PyPiParser reads PEP-735 [dependency-groups], any *requirements*.txt, setup.py install_requires/extras, and conda environment.yml (incl. nested pip:). - Transitive guard: pip-compile lockfiles are detected and their transitive closure marked Transitive; framework/db/orm supported only by transitive deps drop to Low so they never fabricate a hash slot. - New PaketParser (paket.dependencies/.references -> nuget deps). - MSBuild following: merge PackageReferences/properties from shared .props/.targets (recursive, cycle-guarded). Detection - WinForms via a new MSBuild `properties` MatchSpec facet (UseWindowsForms=true). - New built-in PrismaDetector: schema.prisma datasource provider -> database slot. Language accuracy - Drop .h from the Objective-C detector and .m from MATLAB; .fx -> HLSL (not FLUX); OpenCL added to the classifier priority list (beats Common Lisp on .cl). - Binary-content guard so blobs (e.g. a .p pickle) are not counted as source. - Exclude *.min.js / *.min.css from language statistics. --- .../ComponentDetectorTests.cs | 26 +++ .../LanguageClassifierTests.cs | 17 ++ .../ManifestParserTests.cs | 118 +++++++++++++ .../PrismaDetectorTests.cs | 32 ++++ src/Ivy.StackAnalyzer.Test/RuleEngineTests.cs | 35 +++- .../Snapshots/django-spa.verified.yaml | 2 +- .../Snapshots/go-service.verified.yaml | 2 +- .../next-dotnet-monorepo.verified.yaml | 2 +- .../Snapshots/numpy-style-lib.verified.yaml | 2 +- src/Ivy.StackAnalyzer/AnalyzerOptions.cs | 4 +- .../Components/ComponentContext.cs | 4 + .../Components/ComponentDetector.cs | 84 ++++++++- .../Detection/PrismaDetector.cs | 60 +++++++ src/Ivy.StackAnalyzer/Detection/RuleEngine.cs | 61 +++++-- .../Manifests/IManifestParser.cs | 6 + .../Manifests/ManifestParserRegistry.cs | 1 + .../Manifests/NuGetParser.cs | 8 + .../Manifests/PaketParser.cs | 66 +++++++ src/Ivy.StackAnalyzer/Manifests/PyPiParser.cs | 166 ++++++++++++++++-- src/Ivy.StackAnalyzer/Models/DataModels.cs | 10 ++ src/Ivy.StackAnalyzer/Pipeline.cs | 14 +- .../Scanning/LanguageClassifier.cs | 23 ++- src/Ivy.StackAnalyzer/StackDetection.cs | 2 +- .../data/detectors/dotnet.yml | 10 ++ .../data/detectors/language.yml | 8 +- src/Ivy.StackAnalyzer/data/languages.yml | 2 +- src/Ivy.StackAnalyzer/data/vendor.yml | 2 + 27 files changed, 723 insertions(+), 44 deletions(-) create mode 100644 src/Ivy.StackAnalyzer.Test/PrismaDetectorTests.cs create mode 100644 src/Ivy.StackAnalyzer/Detection/PrismaDetector.cs create mode 100644 src/Ivy.StackAnalyzer/Manifests/PaketParser.cs diff --git a/src/Ivy.StackAnalyzer.Test/ComponentDetectorTests.cs b/src/Ivy.StackAnalyzer.Test/ComponentDetectorTests.cs index 1a7bc1f..b57789d 100644 --- a/src/Ivy.StackAnalyzer.Test/ComponentDetectorTests.cs +++ b/src/Ivy.StackAnalyzer.Test/ComponentDetectorTests.cs @@ -36,6 +36,32 @@ public void Workspace_root_members_and_auxiliary_flags() Assert.False(byPath["apps/web"].IsAuxiliary); } + [Fact] + public void Follows_msbuild_import_for_shared_packagereferences() + { + using var repo = new TempRepo(); + repo.Write("tests/Common.props", """ + + + + + + """) + .Write("tests/MyTests/MyTests.csproj", """ + + + + + + + """); + + var comp = Detect(repo).Single(c => c.RelativePath == "tests/MyTests"); + // The directly-declared dep and the one imported from ../Common.props both surface. + Assert.Contains(comp.Dependencies, d => d.Dependency.Name == "NUnit"); + Assert.Contains(comp.Dependencies, d => d.Dependency.Name == "NUnit3TestAdapter"); + } + [Fact] public void Nearest_root_attribution_keeps_nested_files_with_child() { diff --git a/src/Ivy.StackAnalyzer.Test/LanguageClassifierTests.cs b/src/Ivy.StackAnalyzer.Test/LanguageClassifierTests.cs index 40b22b5..739ad80 100644 --- a/src/Ivy.StackAnalyzer.Test/LanguageClassifierTests.cs +++ b/src/Ivy.StackAnalyzer.Test/LanguageClassifierTests.cs @@ -24,12 +24,29 @@ public class LanguageClassifierTests [InlineData("Dockerfile", "Dockerfile")] // by filename [InlineData("go.mod", "Go Module")] // by filename [InlineData("styles/site.css", "CSS")] + [InlineData("shaders/water.fx", "HLSL")] // not InfluxData FLUX + [InlineData("kernels/conv.cl", "OpenCL")] // not Common Lisp / Cool public void Resolves_expected_language(string path, string expected) { var c = Classifier.Classify(File(path)); Assert.Equal(expected, c.Language); } + [Fact] + public void Binary_content_is_not_classified_as_source() + { + // A file whose extension maps to a language but whose bytes are binary + // (e.g. a Python pickle) must not be counted as code. + var tmp = Path.Combine(Path.GetTempPath(), "ivy-bin-" + Guid.NewGuid().ToString("N") + ".py"); + System.IO.File.WriteAllBytes(tmp, [0x80, 0x03, 0x00, 0x01, 0x02]); + try + { + var c = Classifier.Classify(File("model.py", tmp)); + Assert.Null(c.Language); + } + finally { System.IO.File.Delete(tmp); } + } + [Fact] public void Unknown_extension_is_unclassified() { diff --git a/src/Ivy.StackAnalyzer.Test/ManifestParserTests.cs b/src/Ivy.StackAnalyzer.Test/ManifestParserTests.cs index 2e87ec1..1a6c839 100644 --- a/src/Ivy.StackAnalyzer.Test/ManifestParserTests.cs +++ b/src/Ivy.StackAnalyzer.Test/ManifestParserTests.cs @@ -81,6 +81,124 @@ public void PyPi_requirements_txt() Assert.DoesNotContain(m.Dependencies, d => d.Name.StartsWith('-')); } + [Fact] + public void PyPi_pep735_dependency_groups() + { + const string toml = """ + [project] + name = "svc" + dependencies = ["django>=5"] + [dependency-groups] + dev = ["pytest>=8", {include-group = "lint"}] + lint = ["ruff"] + """; + var m = new PyPiParser().Parse("pyproject.toml", toml); + Assert.Contains(m.Dependencies, d => d.Name == "pytest" && d.Scope == DependencyScope.Dev); + Assert.Contains(m.Dependencies, d => d.Name == "ruff"); + Assert.DoesNotContain(m.Dependencies, d => d.Name == "include-group"); + } + + [Theory] + [InlineData("dev-requirements.txt")] + [InlineData("requirements-dev.txt")] + [InlineData("test-requirements.txt")] + public void PyPi_parses_alternate_requirements_names(string fileName) + { + var parser = new PyPiParser(); + Assert.True(parser.CanParse(fileName)); + var m = parser.Parse(fileName, "pytest==8.0\nblack\n"); + Assert.Contains(m.Dependencies, d => d.Name == "pytest"); + } + + [Fact] + public void PyPi_setup_py_install_requires_and_extras() + { + const string setup = """ + from setuptools import setup + setup(name="svc", + install_requires=["flask>=3", "requests"], + extras_require={"dev": ["pytest"]}) + """; + var m = new PyPiParser().Parse("setup.py", setup); + Assert.Contains(m.Dependencies, d => d.Name == "flask"); + Assert.Contains(m.Dependencies, d => d.Name == "requests"); + Assert.Contains(m.Dependencies, d => d.Name == "pytest" && d.Scope == DependencyScope.Optional); + } + + [Fact] + public void PyPi_conda_environment_yaml() + { + const string env = """ + name: ml + channels: [conda-forge] + dependencies: + - python=3.10 + - pytorch + - conda-forge::numpy + - pip: + - transformers==4.0 + """; + var m = new PyPiParser().Parse("environment.yml", env); + Assert.Contains(m.Dependencies, d => d.Name == "pytorch"); + Assert.Contains(m.Dependencies, d => d.Name == "numpy"); + Assert.Contains(m.Dependencies, d => d.Name == "transformers"); + Assert.DoesNotContain(m.Dependencies, d => d.Name == "python"); + } + + [Fact] + public void PyPi_pip_compile_marks_transitive_deps() + { + const string compiled = """ + # This file is autogenerated by pip-compile + fastapi==0.110.0 + # via -r requirements.in + starlette==0.36.0 + # via fastapi + """; + var m = new PyPiParser().Parse("requirements.txt", compiled); + Assert.Contains(m.Dependencies, d => d.Name == "fastapi" && d.Scope == DependencyScope.Runtime); + Assert.Contains(m.Dependencies, d => d.Name == "starlette" && d.Scope == DependencyScope.Transitive); + } + + [Fact] + public void Paket_dependencies_and_references() + { + const string deps = """ + source https://api.nuget.org/v3/index.json + nuget FSharp.Core ~> 6.0 + nuget Elmish + + group Test + nuget NUnit3TestAdapter + """; + var m = new PaketParser().Parse("paket.dependencies", deps); + Assert.Equal("nuget", m.Ecosystem); + Assert.Contains(m.Dependencies, d => d.Name == "FSharp.Core"); + Assert.Contains(m.Dependencies, d => d.Name == "Elmish" && d.Scope == DependencyScope.Runtime); + Assert.Contains(m.Dependencies, d => d.Name == "NUnit3TestAdapter" && d.Scope == DependencyScope.Dev); + + const string refs = "FSharp.Core\nElmish\ngroup Test\n NUnit3TestAdapter\n"; + var r = new PaketParser().Parse("Proj/paket.references", refs); + Assert.Contains(r.Dependencies, d => d.Name == "Elmish"); + Assert.Contains(r.Dependencies, d => d.Name == "NUnit3TestAdapter" && d.Scope == DependencyScope.Dev); + } + + [Fact] + public void NuGet_reads_msbuild_properties() + { + const string xml = """ + + + WinExe + true + + + """; + var m = new NuGetParser().Parse("App/App.csproj", xml); + Assert.Equal("true", m.Properties["UseWindowsForms"]); + Assert.Equal("WinExe", m.Properties["OutputType"]); + } + [Fact] public void GoMod_require_block_and_indirect() { diff --git a/src/Ivy.StackAnalyzer.Test/PrismaDetectorTests.cs b/src/Ivy.StackAnalyzer.Test/PrismaDetectorTests.cs new file mode 100644 index 0000000..499a509 --- /dev/null +++ b/src/Ivy.StackAnalyzer.Test/PrismaDetectorTests.cs @@ -0,0 +1,32 @@ +using Ivy.StackAnalyzer.Detection; +using Xunit; + +namespace Ivy.StackAnalyzer.Test; + +public class PrismaDetectorTests +{ + [Theory] + [InlineData("postgresql", "PostgreSQL")] + [InlineData("mysql", "MySQL")] + [InlineData("sqlite", "SQLite")] + [InlineData("mongodb", "MongoDB")] + public void Maps_datasource_provider_to_database(string provider, string expected) + { + var schema = $$""" + datasource db { + provider = "{{provider}}" + url = env("DATABASE_URL") + } + """; + Assert.Equal(expected, PrismaDetector.DatabaseFor(schema)); + } + + [Fact] + public void Ignores_generator_provider_and_dynamic_provider() + { + // A generator block's provider ("prisma-client-js") is not a database. + Assert.Null(PrismaDetector.DatabaseFor("""generator client { provider = "prisma-client-js" }""")); + // A provider read from an env var is not a literal engine. + Assert.Null(PrismaDetector.DatabaseFor("""datasource db { provider = env("DB_PROVIDER") }""")); + } +} diff --git a/src/Ivy.StackAnalyzer.Test/RuleEngineTests.cs b/src/Ivy.StackAnalyzer.Test/RuleEngineTests.cs index df8079f..ef56f1e 100644 --- a/src/Ivy.StackAnalyzer.Test/RuleEngineTests.cs +++ b/src/Ivy.StackAnalyzer.Test/RuleEngineTests.cs @@ -14,7 +14,8 @@ private static ComponentContext Ctx( IEnumerable? sdks = null, IEnumerable? extensions = null, IEnumerable? envVars = null, - IEnumerable? scripts = null) + IEnumerable? scripts = null, + IEnumerable>? properties = null) { var names = (fileNames ?? []).ToHashSet(StringComparer.OrdinalIgnoreCase); return new ComponentContext @@ -25,6 +26,7 @@ private static ComponentContext Ctx( Languages = [], Dependencies = (deps ?? []).ToList(), Sdks = (sdks ?? []).ToList(), + Properties = new Dictionary(properties ?? [], StringComparer.OrdinalIgnoreCase), IsWorkspaceRoot = false, IsAuxiliary = false, FileNames = names, @@ -37,6 +39,8 @@ private static ComponentContext Ctx( private static EcosystemDependency Npm(string name) => new("npm", new Dependency(name, null, DependencyScope.Runtime)); private static EcosystemDependency NuGet(string name) => new("nuget", new Dependency(name, null, DependencyScope.Runtime)); + private static EcosystemDependency Pypi(string name, DependencyScope scope = DependencyScope.Runtime) + => new("pypi", new Dependency(name, null, scope)); private static readonly DataStore Data = DataStore.Load(); @@ -126,6 +130,35 @@ public void Strong_match_keeps_declared_confidence() Assert.Equal(Confidence.High, react.Confidence); } + [Fact] + public void Transitive_only_slot_tech_is_dropped_to_low() + { + // A framework/db/orm whose only support is a transitive dep (e.g. Django + // pulled into a CLI's pip-compile lockfile) must not enter a hash slot. + var engine = new RuleEngine(Data); + var hit = engine.Detect(Ctx(deps: [Pypi("django", DependencyScope.Transitive)])) + .FirstOrDefault(t => t.Name == "Django"); + Assert.NotNull(hit); + Assert.Equal(Confidence.Low, hit!.Confidence); + } + + [Fact] + public void Direct_slot_tech_keeps_high_confidence() + { + var engine = new RuleEngine(Data); + var hit = engine.Detect(Ctx(deps: [Pypi("django", DependencyScope.Runtime)])) + .Single(t => t.Name == "Django"); + Assert.Equal(Confidence.High, hit.Confidence); + } + + [Fact] + public void Detects_windows_forms_by_msbuild_property() + { + var engine = new RuleEngine(Data); + var result = engine.Detect(Ctx(properties: [new("UseWindowsForms", "true")])); + Assert.Contains(result, t => t.Name == "Windows Forms" && t.Category == TechCategory.Framework); + } + [Theory] [InlineData("SUPABASE_URL", "Supabase")] [InlineData("SLACK_WEBHOOK_URL", "Slack")] diff --git a/src/Ivy.StackAnalyzer.Test/Snapshots/django-spa.verified.yaml b/src/Ivy.StackAnalyzer.Test/Snapshots/django-spa.verified.yaml index 12ad061..82ed23f 100644 --- a/src/Ivy.StackAnalyzer.Test/Snapshots/django-spa.verified.yaml +++ b/src/Ivy.StackAnalyzer.Test/Snapshots/django-spa.verified.yaml @@ -151,6 +151,6 @@ readme: in `frontend/`. metadata: analyzerVersion: 0.1.0 - rulesLoaded: 778 + rulesLoaded: 779 languageDefsLoaded: 815 ignoredDirectories: [] diff --git a/src/Ivy.StackAnalyzer.Test/Snapshots/go-service.verified.yaml b/src/Ivy.StackAnalyzer.Test/Snapshots/go-service.verified.yaml index 13dc9dd..d83c0e6 100644 --- a/src/Ivy.StackAnalyzer.Test/Snapshots/go-service.verified.yaml +++ b/src/Ivy.StackAnalyzer.Test/Snapshots/go-service.verified.yaml @@ -62,6 +62,6 @@ infrastructure: - Dockerfile metadata: analyzerVersion: 0.1.0 - rulesLoaded: 778 + rulesLoaded: 779 languageDefsLoaded: 815 ignoredDirectories: [] diff --git a/src/Ivy.StackAnalyzer.Test/Snapshots/next-dotnet-monorepo.verified.yaml b/src/Ivy.StackAnalyzer.Test/Snapshots/next-dotnet-monorepo.verified.yaml index ab50add..744c959 100644 --- a/src/Ivy.StackAnalyzer.Test/Snapshots/next-dotnet-monorepo.verified.yaml +++ b/src/Ivy.StackAnalyzer.Test/Snapshots/next-dotnet-monorepo.verified.yaml @@ -220,6 +220,6 @@ readme: backed by Postgres. metadata: analyzerVersion: 0.1.0 - rulesLoaded: 778 + rulesLoaded: 779 languageDefsLoaded: 815 ignoredDirectories: [] diff --git a/src/Ivy.StackAnalyzer.Test/Snapshots/numpy-style-lib.verified.yaml b/src/Ivy.StackAnalyzer.Test/Snapshots/numpy-style-lib.verified.yaml index ffc248d..39dfa0e 100644 --- a/src/Ivy.StackAnalyzer.Test/Snapshots/numpy-style-lib.verified.yaml +++ b/src/Ivy.StackAnalyzer.Test/Snapshots/numpy-style-lib.verified.yaml @@ -53,6 +53,6 @@ technologies: infrastructure: [] metadata: analyzerVersion: 0.1.0 - rulesLoaded: 778 + rulesLoaded: 779 languageDefsLoaded: 815 ignoredDirectories: [] diff --git a/src/Ivy.StackAnalyzer/AnalyzerOptions.cs b/src/Ivy.StackAnalyzer/AnalyzerOptions.cs index 7458918..c78295d 100644 --- a/src/Ivy.StackAnalyzer/AnalyzerOptions.cs +++ b/src/Ivy.StackAnalyzer/AnalyzerOptions.cs @@ -17,7 +17,9 @@ public sealed record AnalyzerOptions /// Maximum number of README lines captured in the excerpt. public int MaxReadmeLines { get; init; } = 120; - /// Maximum number of dependencies kept per parsed manifest. + /// Maximum number of dependencies kept per parsed manifest in the + /// reported output. Detection always runs against the full set, so this + /// only bounds report size — it never hides a technology from the rule engine. public int MaxDependenciesPerManifest { get; init; } = 100; /// Extra directories scanned for user-supplied language / detector data files. diff --git a/src/Ivy.StackAnalyzer/Components/ComponentContext.cs b/src/Ivy.StackAnalyzer/Components/ComponentContext.cs index bd88ecb..b0f7d62 100644 --- a/src/Ivy.StackAnalyzer/Components/ComponentContext.cs +++ b/src/Ivy.StackAnalyzer/Components/ComponentContext.cs @@ -28,6 +28,10 @@ public sealed class ComponentContext /// MSBuild SDK attributes seen in this component's project files. public IReadOnlyList Sdks { get; init; } = []; + /// MSBuild build properties merged across this component's project files. + public IReadOnlyDictionary Properties { get; init; } + = new Dictionary(StringComparer.OrdinalIgnoreCase); + /// Bare file names present anywhere in this component. public required IReadOnlySet FileNames { get; init; } diff --git a/src/Ivy.StackAnalyzer/Components/ComponentDetector.cs b/src/Ivy.StackAnalyzer/Components/ComponentDetector.cs index cd331b1..83e871a 100644 --- a/src/Ivy.StackAnalyzer/Components/ComponentDetector.cs +++ b/src/Ivy.StackAnalyzer/Components/ComponentDetector.cs @@ -1,3 +1,4 @@ +using System.Xml.Linq; using Ivy.StackAnalyzer.Manifests; using Ivy.StackAnalyzer.Scanning; @@ -69,7 +70,7 @@ public IReadOnlyList Detect(IReadOnlyList file if (DirOf(f.File.RelativePath) != root) continue; var parser = _parsers.Resolve(f.File.FileName); if (parser is null) continue; - var parsed = ParseSafely(parser, f.File); + var parsed = ParseManifest(parser, f.File); if (parsed is not null) manifests.Add(parsed); } @@ -77,15 +78,23 @@ public IReadOnlyList Detect(IReadOnlyList file // (it is the guaranteed fallback), but skip empty non-root roots. if (root != "" && compFiles.Count == 0 && manifests.Count == 0) continue; + // Feed the FULL dependency set to detection. Truncating here (it used to + // Take(MaxDependenciesPerManifest)) silently dropped hash-significant techs + // whose dep happened to sit past the cap (e.g. Tailwind in a 189-dep + // package.json). The cap is applied only to the *reported* manifest list + // (see Pipeline), never to what the rule engine sees. var deps = new List(); foreach (var m in manifests) - { - var kept = m.Dependencies.Take(_options.MaxDependenciesPerManifest); - foreach (var d in kept) deps.Add(new EcosystemDependency(m.Ecosystem, d)); - } + foreach (var d in m.Dependencies) + deps.Add(new EcosystemDependency(m.Ecosystem, d)); var sdks = manifests.Where(m => !string.IsNullOrEmpty(m.Sdk)).Select(m => m.Sdk!).Distinct().ToList(); + var properties = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var m in manifests) + foreach (var (k, v) in m.Properties) + properties[k] = v; + bool isWorkspaceRoot = workspaceRootDirs.Contains(root) || manifests.Any(m => m.Workspaces.Count > 0); @@ -103,6 +112,7 @@ public IReadOnlyList Detect(IReadOnlyList file Languages = languages, Dependencies = deps, Sdks = sdks, + Properties = properties, IsWorkspaceRoot = isWorkspaceRoot, IsAuxiliary = IsAuxiliary(root), SizeBytes = compFiles.Sum(f => f.File.Length), @@ -121,17 +131,75 @@ public IReadOnlyList Detect(IReadOnlyList file // A pathologically large "manifest" must not be slurped into memory. private const long MaxManifestBytes = 16 * 1024 * 1024; - private static ParsedManifest? ParseSafely(IManifestParser parser, ScannedFile file) + private static ParsedManifest? ParseManifest(IManifestParser parser, ScannedFile file) { if (file.Length > MaxManifestBytes) return null; + string content; + ParsedManifest parsed; try { - var content = File.ReadAllText(file.FullPath); - return parser.Parse(file.RelativePath, content); + content = File.ReadAllText(file.FullPath); + parsed = parser.Parse(file.RelativePath, content); } // Defense-in-depth: no single malformed/locked manifest can abort the run. catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or FormatException or InvalidOperationException or System.Xml.XmlException) { return null; } + + // MSBuild project files may pull PackageReferences from shared `.props`/ + // `.targets` via — follow those so deps declared in a + // central file (e.g. tests/UnitTest.props) are not invisible. + if (!IsMsBuildProject(file.FileName)) return parsed; + var visited = new HashSet(StringComparer.OrdinalIgnoreCase); + try { visited.Add(Path.GetFullPath(file.FullPath)); } catch { /* keep going */ } + var deps = new List(parsed.Dependencies); + var props = new Dictionary(parsed.Properties, StringComparer.OrdinalIgnoreCase); + FollowImports(file.FullPath, content, visited, deps, props); + return parsed with { Dependencies = deps, Properties = props }; + } + + private static bool IsMsBuildProject(string fileName) + => fileName.EndsWith(".csproj", StringComparison.OrdinalIgnoreCase) + || fileName.EndsWith(".fsproj", StringComparison.OrdinalIgnoreCase) + || fileName.EndsWith(".vbproj", StringComparison.OrdinalIgnoreCase); + + // Recursively merge PackageReferences + properties from ed MSBuild files. + // Only literal relative/absolute paths are followed (skip MSBuild `$(...)` props). + private static void FollowImports( + string baseFullPath, string content, + HashSet visited, List deps, Dictionary props) + { + XDocument doc; + try { doc = XDocument.Parse(content); } + catch (System.Xml.XmlException) { return; } + var baseDir = Path.GetDirectoryName(baseFullPath); + if (baseDir is null) return; + + foreach (var imp in doc.Descendants().Where(e => e.Name.LocalName == "Import")) + { + var proj = (string?)imp.Attribute("Project"); + if (string.IsNullOrWhiteSpace(proj) || proj.Contains("$(")) continue; + + string full; + try { full = Path.GetFullPath(Path.Combine(baseDir, proj.Replace('\\', '/'))); } + catch { continue; } + if (!visited.Add(full) || !File.Exists(full)) continue; + + string impContent; + try + { + if (new FileInfo(full).Length > MaxManifestBytes) continue; + impContent = File.ReadAllText(full); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { continue; } + + ParsedManifest pm; + try { pm = new NuGetParser().Parse(full, impContent); } + catch (System.Xml.XmlException) { continue; } + deps.AddRange(pm.Dependencies); + foreach (var (k, v) in pm.Properties) props[k] = v; + + FollowImports(full, impContent, visited, deps, props); + } } private static IReadOnlySet ReadEnvVarNames(List files) diff --git a/src/Ivy.StackAnalyzer/Detection/PrismaDetector.cs b/src/Ivy.StackAnalyzer/Detection/PrismaDetector.cs new file mode 100644 index 0000000..5abde3b --- /dev/null +++ b/src/Ivy.StackAnalyzer/Detection/PrismaDetector.cs @@ -0,0 +1,60 @@ +using System.Text.RegularExpressions; +using Ivy.StackAnalyzer.Components; + +namespace Ivy.StackAnalyzer.Detection; + +/// +/// Surfaces the database engine declared in a Prisma schema's datasource +/// block (provider = "postgresql"). Common Prisma stacks have no raw driver +/// dependency and keep the connection string in an env var, so the DB is knowable +/// only from schema.prisma — which data rules cannot read into. +/// +public sealed partial class PrismaDetector : ITechnologyDetector +{ + public IEnumerable Detect(ComponentContext ctx) + { + foreach (var f in ctx.Files) + { + if (!string.Equals(f.File.FileName, "schema.prisma", StringComparison.OrdinalIgnoreCase)) + continue; + string content; + try { content = File.ReadAllText(f.File.FullPath); } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { continue; } + + var db = DatabaseFor(content); + if (db is not null) + yield return new DetectedTechnology( + db, TechCategory.Database, + $"prisma datasource provider in {f.File.RelativePath}", + Confidence.High, ctx.RelativePath); + } + } + + /// Maps the provider of a Prisma datasource block to a + /// canonical database name, or null if absent / dynamic / unknown. + public static string? DatabaseFor(string schemaPrismaContent) + { + // Scope to the datasource block — `generator` blocks also have a `provider` + // (e.g. "prisma-client-js") that must not be mistaken for a database. + var ds = DatasourceBlockRegex().Match(schemaPrismaContent); + if (!ds.Success) return null; + var prov = ProviderRegex().Match(ds.Groups["body"].Value); + if (!prov.Success) return null; + return prov.Groups["p"].Value.ToLowerInvariant() switch + { + "postgresql" => "PostgreSQL", + "mysql" => "MySQL", + "sqlite" => "SQLite", + "sqlserver" => "Microsoft SQL Server", + "mongodb" => "MongoDB", + "cockroachdb" => "CockroachDB", + _ => null, + }; + } + + [GeneratedRegex(@"datasource\s+[A-Za-z0-9_]+\s*\{(?[^}]*)\}", RegexOptions.Singleline)] + private static partial Regex DatasourceBlockRegex(); + + [GeneratedRegex(@"provider\s*=\s*""(?

[^""]+)""")] + private static partial Regex ProviderRegex(); +} diff --git a/src/Ivy.StackAnalyzer/Detection/RuleEngine.cs b/src/Ivy.StackAnalyzer/Detection/RuleEngine.cs index a76e369..3812aaa 100644 --- a/src/Ivy.StackAnalyzer/Detection/RuleEngine.cs +++ b/src/Ivy.StackAnalyzer/Detection/RuleEngine.cs @@ -19,32 +19,40 @@ public RuleEngine(DataStore data) : this(data.Rules) { } public RuleEngine(IReadOnlyList rules) => _rules = rules; + private static bool IsHashSlot(TechCategory c) + => c is TechCategory.Framework or TechCategory.Database or TechCategory.Orm; + public IReadOnlyList Detect(ComponentContext ctx) { - var matched = new List<(RuleDef Rule, string Evidence, bool Strong)>(); + var matched = new List<(RuleDef Rule, string Evidence, bool Strong, bool Direct)>(); foreach (var rule in _rules) { var m = Match(rule, ctx); - if (m is not null) matched.Add((rule, m.Value.Evidence, m.Value.Strong)); + if (m is not null) matched.Add((rule, m.Value.Evidence, m.Value.Strong, m.Value.Direct)); } // Apply supersedes: drop any tech that a present tech supersedes. var presentIds = matched.Select(m => m.Rule.Id).ToHashSet(StringComparer.OrdinalIgnoreCase); var superseded = new HashSet(StringComparer.OrdinalIgnoreCase); - foreach (var (rule, _, _) in matched) + foreach (var (rule, _, _, _) in matched) foreach (var s in rule.Supersedes) if (presentIds.Contains(s)) superseded.Add(s); var result = new List(); - foreach (var (rule, evidence, strong) in matched) + foreach (var (rule, evidence, strong, direct) in matched) { if (superseded.Contains(rule.Id)) continue; + var category = CategoryMap.Parse(rule.Category); // A dotenv-only match (env-var names are scaffolding, not proof of use) // is downgraded to Low so hash/digest consumers can drop the noise. var confidence = strong ? CategoryMap.ParseConfidence(rule.Confidence) : Confidence.Low; + // A hash-slot tech (framework/db/orm) supported only by transitive deps + // (e.g. FastAPI pulled in by a CLI's pip-compile lockfile) is not a real + // stack choice — drop it to Low so it never enters a hash slot. + if (!direct && IsHashSlot(category)) confidence = Confidence.Low; result.Add(new DetectedTechnology( rule.Name, - CategoryMap.Parse(rule.Category), + category, evidence, confidence, ctx.RelativePath)); @@ -57,21 +65,27 @@ public IReadOnlyList Detect(ComponentContext ctx) } ///

- /// Returns evidence text + whether a strong facet matched (anything other - /// than dotenv), or null if the rule doesn't match. + /// Returns evidence text, whether a strong facet matched (anything other + /// than dotenv), and whether any strong facet was direct (a non-dep facet, + /// or a dep that is not transitive); or null if the rule doesn't match. /// - private (string Evidence, bool Strong)? Match(RuleDef rule, ComponentContext ctx) + private (string Evidence, bool Strong, bool Direct)? Match(RuleDef rule, ComponentContext ctx) { var m = rule.Match; var evidence = new List(); + bool direct = false; // a non-transitive dep, or any non-dep facet, matched // Dependencies (exact) foreach (var d in m.Deps) { - if (ctx.Dependencies.Any(x => + var hit = ctx.Dependencies.FirstOrDefault(x => Eco(x.Ecosystem, d.Ecosystem) && - string.Equals(x.Dependency.Name, d.Name, StringComparison.OrdinalIgnoreCase))) + string.Equals(x.Dependency.Name, d.Name, StringComparison.OrdinalIgnoreCase)); + if (hit is not null) + { evidence.Add($"{d.Ecosystem} dep '{d.Name}'"); + if (hit.Dependency.Scope != DependencyScope.Transitive) direct = true; + } } // Dependency prefixes @@ -80,7 +94,11 @@ public IReadOnlyList Detect(ComponentContext ctx) var hit = ctx.Dependencies.FirstOrDefault(x => Eco(x.Ecosystem, p.Ecosystem) && x.Dependency.Name.StartsWith(p.Prefix, StringComparison.OrdinalIgnoreCase)); - if (hit is not null) evidence.Add($"{p.Ecosystem} dep '{hit.Dependency.Name}'"); + if (hit is not null) + { + evidence.Add($"{p.Ecosystem} dep '{hit.Dependency.Name}'"); + if (hit.Dependency.Scope != DependencyScope.Transitive) direct = true; + } } // Dependency regex @@ -89,14 +107,29 @@ public IReadOnlyList Detect(ComponentContext ctx) var rx = GetRegex(r.Pattern); var hit = ctx.Dependencies.FirstOrDefault(x => Eco(x.Ecosystem, r.Ecosystem) && rx.IsMatch(x.Dependency.Name)); - if (hit is not null) evidence.Add($"{r.Ecosystem} dep '{hit.Dependency.Name}'"); + if (hit is not null) + { + evidence.Add($"{r.Ecosystem} dep '{hit.Dependency.Name}'"); + if (hit.Dependency.Scope != DependencyScope.Transitive) direct = true; + } } + // Non-dependency facets below (sdk/files/paths/extensions/scripts) are all + // direct, first-party evidence — a present config file or SDK is never + // "transitive". If any matches, the rule is directly supported. + int beforeNonDep = evidence.Count; + // SDK attribute foreach (var sdk in m.Sdk) if (ctx.Sdks.Any(s => string.Equals(s, sdk, StringComparison.OrdinalIgnoreCase))) evidence.Add($"Sdk={sdk}"); + // Build properties (e.g. MSBuild UseWindowsForms=true) + foreach (var p in m.Properties) + if (ctx.Properties.TryGetValue(p.Name, out var v) + && string.Equals(v, p.Value, StringComparison.OrdinalIgnoreCase)) + evidence.Add($"{p.Name}={v}"); + // Files (exact name) foreach (var f in m.Files) if (ctx.FileNames.Contains(f)) evidence.Add(f); @@ -131,6 +164,8 @@ public IReadOnlyList Detect(ComponentContext ctx) if (hit is not null) evidence.Add($"script '{hit}'"); } + if (evidence.Count > beforeNonDep) direct = true; + // Every facet above is a strong signal; dotenv (below) is weak. bool strong = evidence.Count > 0; @@ -140,7 +175,7 @@ public IReadOnlyList Detect(ComponentContext ctx) evidence.Add($"env {prefix}*"); if (evidence.Count == 0) return null; - return (string.Join("; ", evidence.Distinct().Take(4)), strong); + return (string.Join("; ", evidence.Distinct().Take(4)), strong, direct); } private static bool Eco(string a, string b) => string.Equals(a, b, StringComparison.OrdinalIgnoreCase); diff --git a/src/Ivy.StackAnalyzer/Manifests/IManifestParser.cs b/src/Ivy.StackAnalyzer/Manifests/IManifestParser.cs index 9c80dc3..f1e4a20 100644 --- a/src/Ivy.StackAnalyzer/Manifests/IManifestParser.cs +++ b/src/Ivy.StackAnalyzer/Manifests/IManifestParser.cs @@ -21,6 +21,12 @@ public sealed record ParsedManifest /// MSBuild Sdk attribute for *.csproj / *.fsproj. public string? Sdk { get; init; } + /// MSBuild PropertyGroup scalar properties (e.g. + /// UseWindowsForms=true). Some frameworks have no package marker and are + /// only knowable from a build property. + public IReadOnlyDictionary Properties { get; init; } + = new Dictionary(StringComparer.OrdinalIgnoreCase); + public ManifestFile ToManifestFile() => new(Path, Ecosystem, Dependencies); } diff --git a/src/Ivy.StackAnalyzer/Manifests/ManifestParserRegistry.cs b/src/Ivy.StackAnalyzer/Manifests/ManifestParserRegistry.cs index f79c344..9632b78 100644 --- a/src/Ivy.StackAnalyzer/Manifests/ManifestParserRegistry.cs +++ b/src/Ivy.StackAnalyzer/Manifests/ManifestParserRegistry.cs @@ -15,6 +15,7 @@ public static IReadOnlyList Default() => [ new NpmParser(), new NuGetParser(), + new PaketParser(), new PyPiParser(), new GoModParser(), new CargoParser(), diff --git a/src/Ivy.StackAnalyzer/Manifests/NuGetParser.cs b/src/Ivy.StackAnalyzer/Manifests/NuGetParser.cs index 0e3ca10..28f8b04 100644 --- a/src/Ivy.StackAnalyzer/Manifests/NuGetParser.cs +++ b/src/Ivy.StackAnalyzer/Manifests/NuGetParser.cs @@ -19,6 +19,7 @@ public ParsedManifest Parse(string relativePath, string content) { var deps = new List(); string? sdk = null; + var properties = new Dictionary(StringComparer.OrdinalIgnoreCase); try { var doc = XDocument.Parse(content); @@ -27,6 +28,12 @@ public ParsedManifest Parse(string relativePath, string content) { sdk = (string?)root.Attribute("Sdk"); + // Scalar properties from every (last value wins). + foreach (var pg in root.Descendants().Where(e => e.Name.LocalName == "PropertyGroup")) + foreach (var prop in pg.Elements()) + if (!prop.HasElements) + properties[prop.Name.LocalName] = prop.Value.Trim(); + foreach (var pr in root.Descendants().Where(e => e.Name.LocalName is "PackageReference" or "PackageVersion" or "GlobalPackageReference")) { @@ -47,6 +54,7 @@ public ParsedManifest Parse(string relativePath, string content) Ecosystem = "nuget", Dependencies = deps, Sdk = sdk, + Properties = properties, }; } } diff --git a/src/Ivy.StackAnalyzer/Manifests/PaketParser.cs b/src/Ivy.StackAnalyzer/Manifests/PaketParser.cs new file mode 100644 index 0000000..77b9e08 --- /dev/null +++ b/src/Ivy.StackAnalyzer/Manifests/PaketParser.cs @@ -0,0 +1,66 @@ +namespace Ivy.StackAnalyzer.Manifests; + +/// +/// Parses Paket manifests (paket.dependencies / paket.references), +/// the dependency manager many F#/.NET repos use instead of in-project +/// PackageReferences. Emits NuGet-ecosystem dependencies so the existing +/// .NET detectors fire. Deps under a group whose name contains "test" or +/// "build" are marked . +/// +public sealed class PaketParser : IManifestParser +{ + public bool CanParse(string fileName) + => string.Equals(fileName, "paket.dependencies", StringComparison.OrdinalIgnoreCase) + || string.Equals(fileName, "paket.references", StringComparison.OrdinalIgnoreCase); + + public ParsedManifest Parse(string relativePath, string content) + { + var fileName = relativePath[(relativePath.LastIndexOf('/') + 1)..]; + bool isDependencies = fileName.Equals("paket.dependencies", StringComparison.OrdinalIgnoreCase); + var deps = new List(); + var scope = DependencyScope.Runtime; + + foreach (var raw in content.Split('\n')) + { + var line = raw.Trim(); + if (line.Length == 0 || line.StartsWith("//") || line.StartsWith('#')) continue; + + // `group Test` switches the active scope for subsequent lines. + if (line.StartsWith("group ", StringComparison.OrdinalIgnoreCase)) + { + var groupName = line[6..].Trim(); + scope = IsDevGroup(groupName) ? DependencyScope.Dev : DependencyScope.Runtime; + continue; + } + + string? name = isDependencies ? PackageFromDependencyLine(line) : PackageFromReferenceLine(line); + if (!string.IsNullOrEmpty(name)) deps.Add(new Dependency(name!, null, scope)); + } + + return new ParsedManifest { Path = relativePath, Ecosystem = "nuget", Dependencies = deps }; + } + + private static bool IsDevGroup(string name) + => name.Contains("test", StringComparison.OrdinalIgnoreCase) + || name.Contains("build", StringComparison.OrdinalIgnoreCase); + + // paket.dependencies: `nuget FSharp.Core ~> 6.0`. Only nuget sources are NuGet deps. + private static string? PackageFromDependencyLine(string line) + { + if (!line.StartsWith("nuget ", StringComparison.OrdinalIgnoreCase)) return null; + var rest = line[6..].Trim(); + var sp = rest.IndexOfAny([' ', '\t']); + return sp < 0 ? rest : rest[..sp]; + } + + // paket.references: a bare package name per line (settings may follow). Skip + // file references and framework directives. + private static string? PackageFromReferenceLine(string line) + { + if (line.StartsWith("File:", StringComparison.OrdinalIgnoreCase) + || line.StartsWith("framework:", StringComparison.OrdinalIgnoreCase) + || line.StartsWith("redirects:", StringComparison.OrdinalIgnoreCase)) return null; + var sp = line.IndexOfAny([' ', '\t']); + return sp < 0 ? line : line[..sp]; + } +} diff --git a/src/Ivy.StackAnalyzer/Manifests/PyPiParser.cs b/src/Ivy.StackAnalyzer/Manifests/PyPiParser.cs index 12de54e..5d21676 100644 --- a/src/Ivy.StackAnalyzer/Manifests/PyPiParser.cs +++ b/src/Ivy.StackAnalyzer/Manifests/PyPiParser.cs @@ -4,14 +4,30 @@ namespace Ivy.StackAnalyzer.Manifests; -/// Parses Python manifests: pyproject.toml, requirements*.txt, Pipfile. +/// +/// Parses Python manifests: pyproject.toml (PEP 621, Poetry, PEP 735 +/// dependency-groups), Pipfile, setup.py (install_requires), +/// conda environment.yml, and any *requirements*.txt. pip-compile +/// lockfiles are recognised and their transitive closure is marked +/// so it cannot fabricate a hash slot. +/// public sealed partial class PyPiParser : IManifestParser { public bool CanParse(string fileName) => string.Equals(fileName, "pyproject.toml", StringComparison.OrdinalIgnoreCase) || string.Equals(fileName, "Pipfile", StringComparison.OrdinalIgnoreCase) - || (fileName.StartsWith("requirements", StringComparison.OrdinalIgnoreCase) - && fileName.EndsWith(".txt", StringComparison.OrdinalIgnoreCase)); + || string.Equals(fileName, "setup.py", StringComparison.OrdinalIgnoreCase) + || IsCondaEnv(fileName) + || IsRequirementsTxt(fileName); + + // dev-requirements.txt, requirements-dev.txt, test-requirements.txt, requirements.txt, … + private static bool IsRequirementsTxt(string f) + => f.EndsWith(".txt", StringComparison.OrdinalIgnoreCase) + && f.Contains("requirements", StringComparison.OrdinalIgnoreCase); + + private static bool IsCondaEnv(string f) + => string.Equals(f, "environment.yml", StringComparison.OrdinalIgnoreCase) + || string.Equals(f, "environment.yaml", StringComparison.OrdinalIgnoreCase); public ParsedManifest Parse(string relativePath, string content) { @@ -20,6 +36,8 @@ public ParsedManifest Parse(string relativePath, string content) { "pyproject.toml" => ParsePyproject(content), "pipfile" => ParsePipfile(content), + "setup.py" => ParseSetupPy(content), + "environment.yml" or "environment.yaml" => ParseCondaEnv(content), _ => ParseRequirements(content), }; return new ParsedManifest { Path = relativePath, Ecosystem = "pypi", Dependencies = deps }; @@ -54,6 +72,15 @@ private static List ParsePyproject(string content) if (g is TomlTable gt && gt.TryGetValue("dependencies", out var gd) && gd is TomlTable gdt) AddFromTable(gdt, DependencyScope.Dev, deps, skipPython: true); } + + // PEP 735: [dependency-groups] dev = ["pytest", {include-group = "test"}] + // (the modern uv / pip convention). String values are PEP 508 specs; + // {include-group = ...} tables are references to other groups -> skip. + if (model.TryGetValue("dependency-groups", out var dg) && dg is TomlTable dgt) + foreach (var group in dgt.Values) + if (group is TomlArray dga) + foreach (var item in dga) + if (item is string s) AddFromSpecifier(s, DependencyScope.Dev, deps); return deps; } @@ -70,21 +97,128 @@ private static List ParsePipfile(string content) return deps; } + // setup.py: extract install_requires=[...] and extras_require={...: [...]}. + private static List ParseSetupPy(string content) + { + var deps = new List(); + foreach (Match block in RequiresBlockRegex().Matches(content)) + { + var scope = block.Groups["kind"].Value.StartsWith("install", StringComparison.OrdinalIgnoreCase) + ? DependencyScope.Runtime : DependencyScope.Optional; + foreach (Match lit in StringLiteralRegex().Matches(block.Groups["body"].Value)) + AddFromSpecifier(lit.Groups["v"].Value, scope, deps); + } + return deps; + } + + // conda environment.yml: top-level `dependencies:` list, plus a nested `- pip:` list. + private static List ParseCondaEnv(string content) + { + var deps = new List(); + bool inDeps = false; + int depsIndent = -1; + foreach (var raw in content.Split('\n')) + { + var line = raw.Replace("\t", " ").TrimEnd(); + var trimmed = line.Trim(); + if (trimmed.Length == 0 || trimmed.StartsWith('#')) continue; + int indent = line.Length - line.TrimStart(' ').Length; + + if (!inDeps) + { + if (trimmed is "dependencies:" || trimmed.StartsWith("dependencies:", StringComparison.Ordinal)) + { inDeps = true; depsIndent = indent; } + continue; + } + + // A non-list key at or above the `dependencies:` indent ends the block. + if (indent <= depsIndent && !trimmed.StartsWith('-')) { inDeps = false; continue; } + if (!trimmed.StartsWith('-')) continue; + + var entry = trimmed[1..].Trim(); + if (entry.Length == 0 || entry.StartsWith("pip:", StringComparison.OrdinalIgnoreCase)) continue; + + // Strip a conda channel prefix (`conda-forge::numpy`) and skip the interpreter pin. + var sep = entry.IndexOf("::", StringComparison.Ordinal); + if (sep >= 0) entry = entry[(sep + 2)..]; + entry = entry.Trim().Trim('"', '\''); + if (entry.Length == 0) continue; + if (entry.Equals("python", StringComparison.OrdinalIgnoreCase) + || entry.StartsWith("python=", StringComparison.OrdinalIgnoreCase) + || entry.StartsWith("python ", StringComparison.OrdinalIgnoreCase)) continue; + + AddFromSpecifier(entry, DependencyScope.Runtime, deps); + } + return deps; + } + private static List ParseRequirements(string content) { + // pip-compile output lists the full transitive closure as if it were direct. + // When we recognise that format, only deps requested via `-r`/`-c`/`-e`/`.in` + // stay Runtime; everything else is marked Transitive so it cannot fabricate a + // framework/db/orm hash slot. + bool compiled = LooksCompiled(content); var deps = new List(); + Dependency? pending = null; // the dep whose `# via` block we're reading + bool pendingDirect = false; + + void Flush() + { + if (pending is null) return; + var scope = !compiled || pendingDirect ? DependencyScope.Runtime : DependencyScope.Transitive; + deps.Add(pending with { Scope = scope }); + pending = null; + } + foreach (var raw in content.Split('\n')) { - var line = raw.Trim(); - if (line.Length == 0 || line.StartsWith('#') || line.StartsWith('-')) continue; + var line = raw.TrimEnd(); + var trimmed = line.Trim(); + if (trimmed.Length == 0) { continue; } + + // A `# via` continuation line belongs to the previous dep. + if (trimmed.StartsWith('#')) + { + if (pending is not null && ViaIsDirect(trimmed)) pendingDirect = true; + continue; + } + if (trimmed.StartsWith('-')) continue; // options like -r, --hash on their own line + + Flush(); + // Strip an inline comment introduced by whitespace + '#'. + var inlineDirect = false; var hash = CommentRegex().Match(line); - if (hash.Success) line = line[..hash.Index].Trim(); - AddFromSpecifier(line, DependencyScope.Runtime, deps); + if (hash.Success) + { + var comment = line[hash.Index..]; + inlineDirect = ViaIsDirect(comment); + line = line[..hash.Index].Trim(); + } + var parsed = ParseSpecifier(line); + if (parsed is null) continue; + pending = parsed; + pendingDirect = inlineDirect; } + Flush(); return deps; } + // Heuristics for a pip-compile / pip-tools generated lockfile. + private static bool LooksCompiled(string content) + => content.Contains("# via", StringComparison.Ordinal) + || content.Contains("pip-compile", StringComparison.OrdinalIgnoreCase) + || content.Contains("uv pip compile", StringComparison.OrdinalIgnoreCase); + + // A `# via` target that references a requirements input (`-r foo.in`, `-c …`, + // `-e .`, or a `*.in` file) means the package was directly requested. + private static bool ViaIsDirect(string comment) + => comment.Contains("-r ", StringComparison.Ordinal) + || comment.Contains("-c ", StringComparison.Ordinal) + || comment.Contains("-e ", StringComparison.Ordinal) + || comment.Contains(".in", StringComparison.OrdinalIgnoreCase); + private static void AddFromTable(TomlTable table, DependencyScope scope, List into, bool skipPython) { foreach (var (key, value) in table) @@ -102,15 +236,21 @@ TomlTable t when t.TryGetValue("version", out var v) => v?.ToString(), private static void AddFromSpecifier(string? spec, DependencyScope scope, List into) { - if (string.IsNullOrWhiteSpace(spec)) return; + var dep = ParseSpecifier(spec); + if (dep is not null) into.Add(dep with { Scope = scope }); + } + + private static Dependency? ParseSpecifier(string? spec) + { + if (string.IsNullOrWhiteSpace(spec)) return null; // Drop an environment marker (PEP 508): "foo>=1; python_version<'3.9'" -> "foo>=1". var semi = spec.IndexOf(';'); if (semi >= 0) spec = spec[..semi]; var m = NameRegex().Match(spec.Trim()); - if (!m.Success) return; + if (!m.Success) return null; var name = m.Groups["name"].Value; var version = m.Groups["ver"].Success ? m.Groups["ver"].Value.Trim() : null; - into.Add(new Dependency(name, version, scope)); + return new Dependency(name, version, DependencyScope.Runtime); } private static string? NormalizeVersion(string? v) @@ -121,4 +261,10 @@ private static void AddFromSpecifier(string? spec, DependencyScope scope, Listinstall_requires|extras_require|tests_require)\s*=\s*[\[{](?[^\]}]*)[\]}]", RegexOptions.Singleline)] + private static partial Regex RequiresBlockRegex(); + + [GeneratedRegex(@"['""](?[^'""]+)['""]")] + private static partial Regex StringLiteralRegex(); } diff --git a/src/Ivy.StackAnalyzer/Models/DataModels.cs b/src/Ivy.StackAnalyzer/Models/DataModels.cs index 2938bf3..463a89a 100644 --- a/src/Ivy.StackAnalyzer/Models/DataModels.cs +++ b/src/Ivy.StackAnalyzer/Models/DataModels.cs @@ -28,6 +28,13 @@ public sealed class DepPrefixRef public string Prefix { get; set; } = ""; } +/// A build-property match (e.g. MSBuild UseWindowsForms=true). +public sealed class PropertyRef +{ + public string Name { get; set; } = ""; + public string Value { get; set; } = ""; +} + /// /// The superset matcher. Any populated facet contributes; a rule matches when /// any of its facets matches the component (logical OR across facets). @@ -44,6 +51,9 @@ public sealed class MatchSpec public List Sdk { get; set; } = []; public List PathGlobs { get; set; } = []; + /// MSBuild build-property matches (name + expected value). + public List Properties { get; set; } = []; + /// Regexes matched against run-script command strings (e.g. package.json /// scripts values). Detects tools invoked only via a script, such as bun test. public List ScriptsRegex { get; set; } = []; diff --git a/src/Ivy.StackAnalyzer/Pipeline.cs b/src/Ivy.StackAnalyzer/Pipeline.cs index ab97115..ebf3402 100644 --- a/src/Ivy.StackAnalyzer/Pipeline.cs +++ b/src/Ivy.StackAnalyzer/Pipeline.cs @@ -36,7 +36,10 @@ private static StackDetection Run(string repoPath, AnalyzerOptions options, Canc var parsers = new ManifestParserRegistry(); var classifier = new LanguageClassifier(data); var ruleEngine = new RuleEngine(data); - var detectors = options.AdditionalDetectors; // code escape hatch (PLAN.md §7c) + // Built-in code detectors run alongside the data rules and any user-supplied + // ones (code escape hatch, PLAN.md §7c). + IReadOnlyList detectors = + [new PrismaDetector(), .. options.AdditionalDetectors]; // 1. Walk var scan = new FileSystemScanner(data, options).Scan(fullPath, ct); @@ -60,7 +63,7 @@ private static StackDetection Run(string repoPath, AnalyzerOptions options, Canc { RelativePath = ctx.RelativePath, Languages = ctx.Languages, - Manifests = ctx.Manifests.Select(m => m.ToManifestFile()).ToList(), + Manifests = ctx.Manifests.Select(m => CapForReport(m, options.MaxDependenciesPerManifest)).ToList(), Technologies = techs, FileCount = ctx.FileCount, SizeBytes = ctx.SizeBytes, @@ -109,6 +112,13 @@ private static StackDetection Run(string repoPath, AnalyzerOptions options, Canc }; } + // The dependency cap bounds the size of the *reported* manifest only; detection + // already ran against the full set (see ComponentDetector). + private static ManifestFile CapForReport(ParsedManifest m, int cap) + => m.Dependencies.Count <= cap + ? m.ToManifestFile() + : new ManifestFile(m.Path, m.Ecosystem, m.Dependencies.Take(cap).ToList()); + private static IEnumerable Dedupe(IEnumerable techs) => techs .GroupBy(t => (t.Name, t.Category)) diff --git a/src/Ivy.StackAnalyzer/Scanning/LanguageClassifier.cs b/src/Ivy.StackAnalyzer/Scanning/LanguageClassifier.cs index a836eaa..3ff1e63 100644 --- a/src/Ivy.StackAnalyzer/Scanning/LanguageClassifier.cs +++ b/src/Ivy.StackAnalyzer/Scanning/LanguageClassifier.cs @@ -28,6 +28,11 @@ public ClassifiedFile Classify(ScannedFile file) if (name is null) return new ClassifiedFile { File = file }; + // A binary blob whose extension happens to map to a language (e.g. a Python + // pickle `.p` → Gnuplot) must not be counted as source code. + if (IsBinary(file.FullPath)) + return new ClassifiedFile { File = file }; + return new ClassifiedFile { File = file, @@ -36,6 +41,22 @@ public ClassifiedFile Classify(ScannedFile file) }; } + // Cheap binary sniff: a NUL byte in the first few KB is a strong binary signal + // (UTF-8/16 text without a BOM never contains a lone NUL in normal content). + private static bool IsBinary(string fullPath) + { + try + { + using var fs = File.OpenRead(fullPath); + Span buf = stackalloc byte[4096]; + int n = fs.Read(buf); + for (int i = 0; i < n; i++) + if (buf[i] == 0) return true; + return false; + } + catch { return false; } + } + private string? ResolveLanguage(ScannedFile file) { // 1. Exact filename (Dockerfile, Makefile, go.mod, ...) @@ -64,7 +85,7 @@ public ClassifiedFile Classify(ScannedFile file) private static readonly string[] CommonPriority = [ "Markdown", "JSON", "YAML", "TypeScript", "TSX", "JavaScript", "JSX", - "Python", "C#", "F#", "Java", "Kotlin", "Go", "Rust", "C", "C++", + "Python", "C#", "F#", "Java", "Kotlin", "Go", "Rust", "C", "C++", "OpenCL", "Ruby", "PHP", "Swift", "Scala", "HTML", "CSS", "SCSS", "Sass", "Less", "Vue", "Svelte", "Dart", "Elixir", "Shell", "PowerShell", "SQL", "XML", "Objective-C", "Dockerfile", "Text", diff --git a/src/Ivy.StackAnalyzer/StackDetection.cs b/src/Ivy.StackAnalyzer/StackDetection.cs index 2081882..947f388 100644 --- a/src/Ivy.StackAnalyzer/StackDetection.cs +++ b/src/Ivy.StackAnalyzer/StackDetection.cs @@ -118,6 +118,6 @@ public enum TechCategory Documentation, } -public enum DependencyScope { Runtime, Dev, Peer, Optional } +public enum DependencyScope { Runtime, Dev, Peer, Optional, Transitive } public enum Confidence { Low, Medium, High } diff --git a/src/Ivy.StackAnalyzer/data/detectors/dotnet.yml b/src/Ivy.StackAnalyzer/data/detectors/dotnet.yml index 600d838..4377a62 100644 --- a/src/Ivy.StackAnalyzer/data/detectors/dotnet.yml +++ b/src/Ivy.StackAnalyzer/data/detectors/dotnet.yml @@ -61,6 +61,16 @@ sdk: ["Microsoft.NET.Sdk.Worker"] confidence: high +# WinForms has no NuGet package and no XAML; its only reliable marker is the +# MSBuild property true. +- id: windows-forms + name: Windows Forms + category: framework + match: + properties: + - { name: UseWindowsForms, value: "true" } + confidence: high + - id: avalonia name: Avalonia category: framework diff --git a/src/Ivy.StackAnalyzer/data/detectors/language.yml b/src/Ivy.StackAnalyzer/data/detectors/language.yml index 37a4275..f1b717e 100644 --- a/src/Ivy.StackAnalyzer/data/detectors/language.yml +++ b/src/Ivy.StackAnalyzer/data/detectors/language.yml @@ -119,17 +119,21 @@ match: extensions: [.lua] confidence: high +# `.m` is shared by MATLAB and Objective-C; in code repos it is overwhelmingly +# Objective-C, so MATLAB keeps only its unambiguous `.matlab`. - id: matlab name: MATLAB category: language match: - extensions: [.matlab, .m] + extensions: [.matlab] confidence: high +# `.h` is a C/C++ header, not Objective-C — claiming it produced a false +# high-confidence Objective-C on every C repo. Keep `.m`/`.mm` only. - id: objectivec name: Objective-C category: language match: - extensions: [.m, .mm, .h] + extensions: [.m, .mm] confidence: high - id: perl name: Perl diff --git a/src/Ivy.StackAnalyzer/data/languages.yml b/src/Ivy.StackAnalyzer/data/languages.yml index a8bc20e..ced4498 100644 --- a/src/Ivy.StackAnalyzer/data/languages.yml +++ b/src/Ivy.StackAnalyzer/data/languages.yml @@ -195,7 +195,7 @@ Euphoria: { type: programming, extensions: [.e, .ex], interpreters: [eui, euiw] "F*": { type: programming, extensions: [.fst, .fsti] } FIGlet Font: { type: data, extensions: [.flf] } FIRRTL: { type: programming, extensions: [.fir] } -FLUX: { type: programming, extensions: [.fx, .flux] } +FLUX: { type: programming, extensions: [.flux] } Factor: { type: programming, extensions: [.factor], filenames: [.factor-boot-rc, .factor-rc] } Fancy: { type: programming, extensions: [.fy, .fancypack], filenames: [Fakefile] } Fantom: { type: programming, extensions: [.fan] } diff --git a/src/Ivy.StackAnalyzer/data/vendor.yml b/src/Ivy.StackAnalyzer/data/vendor.yml index 6a2088e..b0d3ca8 100644 --- a/src/Ivy.StackAnalyzer/data/vendor.yml +++ b/src/Ivy.StackAnalyzer/data/vendor.yml @@ -1,6 +1,8 @@ # Vendored / generated path patterns seeded from github-linguist (MIT), plus common build dirs. # Each entry is a .NET regex tested against forward-slash relative paths. patterns: + # Minified / bundled assets — generated, not authored; would inflate JS/CSS shares. + - "(^|/)[^/]+\\.min\\.(js|css)$" # Generated lockfiles — excluded so they do not skew language statistics. - "(^|/)package-lock\\.json$" - "(^|/)npm-shrinkwrap\\.json$"