diff --git a/internal/pypi/pypi.go b/internal/pypi/pypi.go index 46b5757..3565b59 100644 --- a/internal/pypi/pypi.go +++ b/internal/pypi/pypi.go @@ -331,6 +331,20 @@ func (p *pyprojectParser) Parse(filename string, content []byte) ([]core.Depende }) } + // PEP 621 optional dependencies + for groupName, groupDeps := range pyproject.Project.OptionalDependencies { + scope := optionalGroupScope(groupName) + for _, dep := range groupDeps { + name, version := parsePEP508(dep) + deps = append(deps, core.Dependency{ + Name: name, + Version: version, + Scope: scope, + Direct: true, + }) + } + } + return deps, nil } @@ -388,6 +402,19 @@ func parsePEP508(dep string) (string, string) { return name, "" } +// optionalGroupScope maps a PEP 621 optional-dependency group name to a scope. +// Well-known dev/test group names get their own scope; everything else is optional. +func optionalGroupScope(groupName string) core.Scope { + switch strings.ToLower(groupName) { + case "dev", "development", "develop", "lint": + return core.Development + case "test", "testing", "tests": + return core.Test + default: + return core.Optional + } +} + // poetryLockParser parses poetry.lock files. type poetryLockParser struct{} @@ -665,30 +692,16 @@ func (p *setupPyParser) Parse(filename string, content []byte) ([]core.Dependenc // Parse extras_require if match := extrasRequireRegex.FindStringSubmatch(contentStr); match != nil { - // Find the group names and their requirements - groupContent := match[1] - // Simple parsing: find all quoted strings that look like requirements - for _, req := range quotedStringRegex.FindAllStringSubmatch(groupContent, -1) { - if len(req) >= 2 { - reqStr := req[1] - // Skip group names (they don't contain version specifiers) - if strings.ContainsAny(reqStr, ">=<~!=") || !strings.ContainsAny(reqStr, "-_.") { - continue - } - // This is a potential group name, skip - if !strings.Contains(reqStr, ">=") && !strings.Contains(reqStr, "==") && - !strings.Contains(reqStr, "<=") && !strings.Contains(reqStr, "~=") { - // Check if it looks like a package name - if len(reqStr) < 20 && strings.ContainsAny(reqStr, "abcdefghijklmnopqrstuvwxyz") { - name, version := parseSetupRequirement(reqStr) - deps = append(deps, core.Dependency{ - Name: name, - Version: version, - Scope: core.Development, - Direct: true, - }) - } - } + for groupName, groupDeps := range parseExtrasRequire(match[1]) { + scope := optionalGroupScope(groupName) + for _, req := range groupDeps { + name, version := parseSetupRequirement(req) + deps = append(deps, core.Dependency{ + Name: name, + Version: version, + Scope: scope, + Direct: true, + }) } } } @@ -711,6 +724,26 @@ func parseSetupRequirement(req string) (string, string) { return req, "" } +// extrasRequireGroupRegex matches a group key and its list value inside extras_require, +// e.g. 'testing': ['pkg1', 'pkg2>=1.0'] +var extrasRequireGroupRegex = regexp.MustCompile(`['"]([^'"]+)['"]\s*:\s*\[([^\]]*)\]`) + +// parseExtrasRequire parses the inner content of a setup.py extras_require dict +// and returns a map of group name to list of requirement strings. +func parseExtrasRequire(content string) map[string][]string { + groups := make(map[string][]string) + for _, match := range extrasRequireGroupRegex.FindAllStringSubmatch(content, -1) { + groupName := match[1] + listContent := match[2] + for _, req := range quotedStringRegex.FindAllStringSubmatch(listContent, -1) { + if len(req) >= 2 { + groups[groupName] = append(groups[groupName], req[1]) + } + } + } + return groups +} + // pylockTomlParser parses pylock.toml files (PEP 665). type pylockTomlParser struct{} diff --git a/internal/pypi/pypi_test.go b/internal/pypi/pypi_test.go index 37a81d6..08eb099 100644 --- a/internal/pypi/pypi_test.go +++ b/internal/pypi/pypi_test.go @@ -967,6 +967,107 @@ func TestPipdeptreeJSON(t *testing.T) { } } +func TestPyprojectPEP621OptionalDeps(t *testing.T) { + content, err := os.ReadFile("../../testdata/pypi/pep621-optional/pyproject.toml") + if err != nil { + t.Fatalf("failed to read fixture: %v", err) + } + + parser := &pyprojectParser{} + deps, err := parser.Parse("pyproject.toml", content) + if err != nil { + t.Fatalf("Parse failed: %v", err) + } + + // 1 runtime + 1 lint + 2 dev + 2 testing + 3 modeling = 9 + if len(deps) != 9 { + t.Fatalf("expected 9 dependencies, got %d", len(deps)) + } + + type depKey struct { + name string + version string + scope core.Scope + } + + depMap := make(map[string]depKey) + for _, d := range deps { + depMap[d.Name] = depKey{d.Name, d.Version, d.Scope} + } + + expected := []depKey{ + {"sqlmodel", ">=0.0.16,<0.0.17", core.Runtime}, + {"pre-commit", ">=2.20.0", core.Development}, + {"ipython", "", core.Development}, + {"alembic", ">=1,<2", core.Development}, + {"pytest", ">=7,<8", core.Test}, + {"pytest-cov", "", core.Test}, + {"numpy", ">=1,<3", core.Optional}, + {"pandas", ">=1,<3", core.Optional}, + {"scipy", ">=1,<2", core.Optional}, + } + + for _, exp := range expected { + dep, ok := depMap[exp.name] + if !ok { + t.Errorf("expected %s dependency", exp.name) + continue + } + if dep.version != exp.version { + t.Errorf("%s version = %q, want %q", exp.name, dep.version, exp.version) + } + if dep.scope != exp.scope { + t.Errorf("%s scope = %v, want %v", exp.name, dep.scope, exp.scope) + } + } +} + +func TestSetupPyExtrasRequire(t *testing.T) { + content, err := os.ReadFile("../../testdata/pypi/setup.py") + if err != nil { + t.Fatalf("failed to read fixture: %v", err) + } + + parser := &setupPyParser{} + deps, err := parser.Parse("setup.py", content) + if err != nil { + t.Fatalf("Parse failed: %v", err) + } + + // Find the extras_require deps by scope + var testDeps []core.Dependency + for _, d := range deps { + if d.Scope == core.Test { + testDeps = append(testDeps, d) + } + } + + if len(testDeps) != 7 { + t.Fatalf("expected 7 test-scope deps from extras_require, got %d", len(testDeps)) + } + + testDepMap := make(map[string]core.Dependency) + for _, d := range testDeps { + testDepMap[d.Name] = d + } + + expectedTestDeps := []string{ + "django-responsediff", + "flake8", + "pep8", + "pytest", + "pytest-django", + "pytest-cov", + "codecov", + } + + for _, name := range expectedTestDeps { + if _, ok := testDepMap[name]; !ok { + t.Errorf("expected %s in extras_require testing deps", name) + } + } +} + func TestPipenvGraphJSON(t *testing.T) { content, err := os.ReadFile("../../testdata/pypi/pipenv.graph.json") if err != nil { diff --git a/testdata/pypi/pep621-optional/pyproject.toml b/testdata/pypi/pep621-optional/pyproject.toml new file mode 100644 index 0000000..c595dd3 --- /dev/null +++ b/testdata/pypi/pep621-optional/pyproject.toml @@ -0,0 +1,29 @@ +[build-system] +requires = ["setuptools>=65", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "rs-graph" +description = "Exploring research software" +requires-python = ">=3.11" +dependencies = [ + "sqlmodel>=0.0.16,<0.0.17", +] + +[project.optional-dependencies] +lint = [ + "pre-commit>=2.20.0", +] +dev = [ + "ipython", + "alembic[tz]>=1,<2", +] +testing = [ + "pytest>=7,<8", + "pytest-cov", +] +modeling = [ + "numpy>=1,<3", + "pandas>=1,<3", + "scipy>=1,<2", +]