Populate package and dependency attributes in inspect_packages (#1180)

* Populate package and depenedency attributes in inspect_packages

Properly populate dependency attributes like for_package, datasource_id
and datafile_resource for DiscoveredDependencies and codebase_resources
in DiscoveredPackages found in inspect_packages pipeline.

Signed-off-by: Ayan Sinha Mahapatra <ayansmahapatra@gmail.com>

* Adress review comments

Signed-off-by: Ayan Sinha Mahapatra <ayansmahapatra@gmail.com>

---------

Signed-off-by: Ayan Sinha Mahapatra <ayansmahapatra@gmail.com>

Author: AyanSinhaMahapatra

CHANGELOG.rst

@@ -23,6 +23,11 @@ v34.5.0 (unreleased)
   symbol and string using tree-sitter.
   https://github.com/nexB/scancode.io/pull/1181
 
+- Fix `inspect_packages` pipeline to properly link discovered packages and dependencies to
+  codebase resources of package manifests where they were found. Also correctly assign
+  the datasource_ids atrribute for packages and dependencies.
+  https://github.com/nexB/scancode.io/pull/1180/
+
 v34.4.0 (2024-04-22)
 --------------------
 

scanpipe/models.py

@@ -435,7 +435,7 @@ class UpdateMixin:
 
     def update(self, **kwargs):
         """
-        Update this resource with the provided ``kwargs`` values.
+        Update this instance with the provided ``kwargs`` values.
         The full ``save()`` process will be triggered, including signals, and the
         ``update_fields`` is automatically set.
         """
@@ -3404,6 +3404,7 @@ def create_from_data(
         dependency_data,
         for_package=None,
         datafile_resource=None,
+        datasource_id=None,
         strip_datafile_path_root=False,
     ):
         """
@@ -3449,6 +3450,9 @@ def create_from_data(
                     datafile_path = "/".join(segments[1:])
                 datafile_resource = project.codebaseresources.get(path=datafile_path)
 
+        if datasource_id:
+            dependency_data["datasource_id"] = datasource_id
+
         # Set purl fields from `purl`
         purl = dependency_data.get("purl")
         purl_mapping = PackageURL.from_string(purl).to_dict()

scanpipe/pipes/__init__.py

@@ -180,6 +180,7 @@ def update_or_create_package(project, package_data, codebase_resources=None):
     package_data = _clean_package_data(package_data)
     # No values for package_uid requires to be empty string for proper queryset lookup
     package_uid = package_data.get("package_uid") or ""
+    datasource_id = package_data.get("datasource_id") or ""
 
     package = DiscoveredPackage.objects.get_or_none(
         project=project,
@@ -192,8 +193,14 @@ def update_or_create_package(project, package_data, codebase_resources=None):
     else:
         package = DiscoveredPackage.create_from_data(project, package_data)
 
-    if package and codebase_resources:
-        package.add_resources(codebase_resources)
+    if package:
+        if datasource_id and datasource_id not in package.datasource_ids:
+            datasource_ids = package.datasource_ids.copy()
+            datasource_ids.append(datasource_id)
+            package.update(datasource_ids=datasource_ids)
+
+        if codebase_resources:
+            package.add_resources(codebase_resources)
 
     return package
 
@@ -210,7 +217,12 @@ def create_local_files_package(project, defaults, codebase_resources=None):
 
 
 def update_or_create_dependency(
-    project, dependency_data, for_package=None, strip_datafile_path_root=False
+    project,
+    dependency_data,
+    for_package=None,
+    datafile_resource=None,
+    datasource_id=None,
+    strip_datafile_path_root=False,
 ):
     """
     Get, update or create a DiscoveredDependency then returns it.
@@ -241,6 +253,8 @@ def update_or_create_dependency(
             project,
             dependency_data,
             for_package=for_package,
+            datafile_resource=datafile_resource,
+            datasource_id=datasource_id,
             strip_datafile_path_root=strip_datafile_path_root,
         )
 

scanpipe/pipes/scancode.py

@@ -478,11 +478,23 @@ def process_package_data(project):
 
             package_data = pd.to_dict()
             dependencies = package_data.pop("dependencies")
-            for dep in dependencies:
-                pipes.update_or_create_dependency(project, dep)
 
+            package = None
             if pd.purl:
-                pipes.update_or_create_package(project, package_data)
+                package = pipes.update_or_create_package(
+                    project=project,
+                    package_data=package_data,
+                    codebase_resources=[resource],
+                )
+
+            for dep in dependencies:
+                pipes.update_or_create_dependency(
+                    project=project,
+                    dependency_data=dep,
+                    for_package=package,
+                    datafile_resource=resource,
+                    datasource_id=pd.datasource_id,
+                )
 
 
 def get_packages_with_purl_from_resources(project):

scanpipe/tests/test_pipelines.py

@@ -613,6 +613,16 @@ def test_scanpipe_inspect_packages_creates_packages_npm(self):
         self.assertEqual(1, project1.discoveredpackages.count())
         self.assertEqual(1, project1.discovereddependencies.count())
 
+        package = project1.discoveredpackages.get()
+        dependency = project1.discovereddependencies.get()
+
+        self.assertEqual(1, package.codebase_resources.count())
+        self.assertEqual("pkg:npm/is-npm@1.0.0", dependency.for_package.purl)
+        self.assertEqual(package.datasource_ids, [dependency.datasource_id])
+        self.assertEqual(
+            package.codebase_resources.get().path, dependency.datafile_resource.path
+        )
+
     def test_scanpipe_inspect_packages_creates_packages_pypi(self):
         pipeline_name = "inspect_packages"
         project1 = Project.objects.create(name="Analysis")