fix(project create): optimize release state handling on update

Instead of restoring all release states after project update in a big
loop, SW360 REST API also allows to pass full release information during
release update. For large projects, this is much faster and avoids
timeouts or even crashes due to REST API rate limiting.

This also respects states provided in the SBOM which will overwrite
existing states.

Fixes #121

Author: gernot-h

ChangeLog.md

@@ -15,6 +15,8 @@
 * `bom map` fix: In few cases with --nocache, it added mixed matches to output
   BOM, now we assure that only the best mapping results are added.
 * `project createbom` stores release relations (`CONTAINED`, `SIDE_BY_SIDE` etc.) as capycli:projectRelation
+* `project update`: optimized handling of release mainline state and release relation. Now states
+  provided in the SBOM are used and slowdowns/crashes introduced in 2.7.0 (#121) fixed again.
 
 ## 2.7.0
 

capycli/project/create_project.py

@@ -33,9 +33,9 @@ def __init__(self, onlyUpdateProject: bool = False) -> None:
         self.onlyUpdateProject = onlyUpdateProject
         self.project_mainline_state: str = ""
 
-    def bom_to_release_list(self, sbom: Bom) -> List[str]:
-        """Creates a list with linked releases"""
-        linkedReleases = []
+    def bom_to_release_list(self, sbom: Bom) -> Dict[str, Any]:
+        """Creates a list with linked releases from the SBOM."""
+        linkedReleases: Dict[str, Any] = {}
 
         for cx_comp in sbom.components:
             rid = CycloneDxSupport.get_property_value(cx_comp, CycloneDxSupport.CDX_PROP_SW360ID)
@@ -45,31 +45,39 @@ def bom_to_release_list(self, sbom: Bom) -> List[str]:
                     + ", " + str(cx_comp.version))
                 continue
 
-            linkedReleases.append(rid)
+            linkedReleases[rid] = {}
+
+            mainlineState = CycloneDxSupport.get_property_value(cx_comp, CycloneDxSupport.CDX_PROP_PROJ_STATE)
+            if mainlineState:
+                linkedReleases[rid]["mainlineState"] = mainlineState
+            relation = CycloneDxSupport.get_property_value(cx_comp, CycloneDxSupport.CDX_PROP_PROJ_RELATION)
+            if relation:
+                # No typo. In project structure, it's "relation", while release update API uses "releaseRelation".
+                linkedReleases[rid]["releaseRelation"] = relation
 
         return linkedReleases
 
-    def get_release_project_mainline_states(self, project: Optional[Dict[str, Any]]) -> List[Dict[str, Any]]:
-        pms: List[Dict[str, Any]] = []
+    def merge_project_mainline_states(self, data: Dict[str, Any], project: Optional[Dict[str, Any]]) -> None:
         if not project:
-            return pms
+            return
 
         if "linkedReleases" not in project:
-            return pms
+            return
 
         for release in project["linkedReleases"]:  # NOT ["sw360:releases"]
             pms_release = release.get("release", "")
             if not pms_release:
                 continue
+            pms_release = pms_release.split("/")[-1]
+            if pms_release not in data:
+                continue
             pms_state = release.get("mainlineState", "OPEN")
             pms_relation = release.get("relation", "UNKNOWN")
-            pms_entry: Dict[str, Any] = {}
-            pms_entry["release"] = pms_release
-            pms_entry["mainlineState"] = pms_state
-            pms_entry["new_relation"] = pms_relation
-            pms.append(pms_entry)
 
-        return pms
+            if "mainlineState" not in data[pms_release]:
+                data[pms_release]["mainlineState"] = pms_state
+            if "releaseRelation" not in data[pms_release]:
+                data[pms_release]["releaseRelation"] = pms_relation
 
     def update_project(self, project_id: str, project: Optional[Dict[str, Any]],
                        sbom: Bom, project_info: Dict[str, Any]) -> None:
@@ -79,7 +87,7 @@ def update_project(self, project_id: str, project: Optional[Dict[str, Any]],
             sys.exit(ResultCode.RESULT_ERROR_ACCESSING_SW360)
 
         data = self.bom_to_release_list(sbom)
-        pms = self.get_release_project_mainline_states(project)
+        self.merge_project_mainline_states(data, project)
 
         ignore_update_elements = ["name", "version"]
         # remove elements from list because they are handled separately
@@ -90,13 +98,17 @@ def update_project(self, project_id: str, project: Optional[Dict[str, Any]],
         try:
             print_text("  " + str(len(data)) + " releases in SBOM")
 
+            update_mode = self.onlyUpdateProject
             if project and "_embedded" in project and "sw360:releases" in project["_embedded"]:
                 print_text(
                     "  " + str(len(project["_embedded"]["sw360:releases"])) +
                     " releases in project before update")
+            else:
+                # Workaround for SW360 API bug: add releases will hang forever for empty projects
+                update_mode = False
 
             # note: type in sw360python, 1.4.0 is wrong - we are using the correct one!
-            result = self.client.update_project_releases(data, project_id, add=self.onlyUpdateProject)  # type: ignore
+            result = self.client.update_project_releases(data, project_id, add=update_mode)  # type: ignore
             if not result:
                 print_red("  Error updating project releases!")
             project = self.client.get_project(project_id)
@@ -114,20 +126,6 @@ def update_project(self, project_id: str, project: Optional[Dict[str, Any]],
                 if not result2:
                     print_red("  Error updating project!")
 
-            if pms and project:
-                print_text("  Restoring original project mainline states...")
-                for pms_entry in pms:
-                    update_release = False
-                    for r in project.get("linkedReleases", []):
-                        if r["release"] == pms_entry["release"]:
-                            update_release = True
-                            break
-
-                    if update_release:
-                        rid = self.client.get_id_from_href(pms_entry["release"])
-                        self.client.update_project_release_relationship(
-                            project_id, rid, pms_entry["mainlineState"], pms_entry["new_relation"], "")
-
         except SW360Error as swex:
             if swex.response is None:
                 print_red("  Unknown error: " + swex.message)

tests/fixtures/sbom_for_create_project.json

@@ -66,6 +66,10 @@
           "name": "siemens:primaryLanguage",
           "value": "Python"
         },
+        {
+          "name": "capycli:projectRelation",
+          "value": "DYNAMICALLY_LINKED"
+        },
         {
           "name": "siemens:sw360Id",
           "value": "a5cae39f39db4e2587a7d760f59ce3d0"
@@ -79,4 +83,4 @@
       "dependsOn": []
     }
   ]
-}
+}

tests/test_create_project.py

@@ -8,7 +8,7 @@
 
 import json
 import os
-from typing import Any, Dict, List, Tuple
+from typing import Any, Dict, Tuple
 
 import responses
 import responses.matchers
@@ -44,7 +44,7 @@ def match(request: Any) -> Tuple[bool, str]:
     return match
 
 
-def update_release_matcher(releases: List[str]) -> Any:
+def update_release_matcher(releases: Dict[str, Any]) -> Any:
     """
     Matches the updated releases.
 
@@ -66,10 +66,15 @@ def match(request: Any) -> Tuple[bool, str]:
             reason = ("Number of releases does not match, got " + str(len(json_body)) +
                       " expected: " + str(len(releases)))
         else:
-            for rel in releases:
+            for rel, rel_data in releases.items():
                 if rel not in request_body:
                     result = False
                     reason = ("Release " + rel + " not found in: " + request_body)
+                if rel_data != json_body[rel]:
+                    result = False
+                    reason = ("Release[" + rel + "] = '" + str(json_body[rel]) + "' does not match expected " +
+                              str(rel_data))
+                    break
 
         return result, reason
     return match
@@ -484,7 +489,8 @@ def test_project_update(self) -> None:
                 }
             },
             match=[
-                update_release_matcher(["a5cae39f39db4e2587a7d760f59ce3d0"])
+                update_release_matcher({"a5cae39f39db4e2587a7d760f59ce3d0": {
+                    "releaseRelation": "DYNAMICALLY_LINKED"}})
             ],
             status=201,
             content_type="application/json",
@@ -645,7 +651,10 @@ def test_project_copy_from(self) -> None:
                 }
             },
             match=[
-                update_release_matcher(["a5cae39f39db4e2587a7d760f59ce3d0"])
+                update_release_matcher({"a5cae39f39db4e2587a7d760f59ce3d0": {
+                    "mainlineState": "SPECIFIC",  # from project 007
+                    "releaseRelation": "DYNAMICALLY_LINKED"  # from SBOM
+                }})
             ],
             status=201,
             content_type="application/json",
@@ -791,7 +800,8 @@ def xtest_project_update_old_version(self) -> None:
                 }
             },
             match=[
-                update_release_matcher(["a5cae39f39db4e2587a7d760f59ce3d0"])
+                update_release_matcher({"a5cae39f39db4e2587a7d760f59ce3d0": {
+                    "releaseRelation": "DYNAMICALLY_LINKED"}})
             ],
             status=201,
             content_type="application/json",