Update scancode-toolkit with faster package scan

pyproject.toml

@@ -58,22 +58,19 @@ dependencies = [
   # Docker
   "container-inspector==33.0.0",
   # ScanCode-toolkit
-  "scancode-toolkit[packages]==32.4.1",
+  "scancode-toolkit[packages]==32.5.0",
   "extractcode[full]==31.0.0",
-  "commoncode==32.3.0",
+  "commoncode==32.4.2",
   "Beautifulsoup4[chardet]==4.13.4",
   "packageurl-python==0.17.6",
-  # Workaround issue https://github.com/aboutcode-org/scancode.io/issues/1795
-  "fingerprints==1.2.3",
-  "normality==2.6.1",
   # FetchCode
   "fetchcode==0.8.0",
   "fetchcode-container==1.2.3.210512; sys_platform == 'linux'",
   # Inspectors
   "elf-inspector==0.0.3",
   "go-inspector==0.5.0",
-  "rust-inspector==0.1.0",
-  "binary-inspector==0.1.2",
+  "rust-inspector==0.2.1",
+  "binary-inspector==0.2.0",
   "python-inspector==0.15.0",
   "source-inspector==0.7.0; sys_platform != 'darwin' and platform_machine != 'arm64'",
   "aboutcode-toolkit==11.1.1",
@@ -101,9 +98,7 @@ dependencies = [
   # AboutCode pipeline
   "aboutcode.pipeline==0.2.1",
   # ScoreCode
-  "scorecode==0.0.4",
-  # Workaround issue https://github.com/aboutcode-org/scancode.io/issues/1885
-  "click==8.2.1"
+  "scorecode==0.0.4"
 ]
 
 [project.optional-dependencies]

scanpipe/pipelines/analyze_root_filesystem.py

@@ -94,7 +94,11 @@ def flag_uninteresting_codebase_resources(self):
 
     def scan_for_application_packages(self):
         """Scan unknown resources for packages information."""
-        scancode.scan_for_application_packages(self.project, progress_logger=self.log)
+        scancode.scan_for_application_packages(
+            project=self.project,
+            compiled=True,
+            progress_logger=self.log,
+        )
 
     def match_not_analyzed_to_system_packages(self):
         """

scanpipe/pipelines/inspect_packages.py

@@ -49,10 +49,19 @@ def steps(cls):
             cls.collect_and_create_codebase_resources,
             cls.flag_empty_files,
             cls.flag_ignored_resources,
+            cls.scan_binaries,
             cls.scan_for_application_packages,
             cls.resolve_dependencies,
         )
 
+    @optional_step("Compiled")
+    def scan_binaries(self):
+        """
+        Scan compiled binaries for package and dependency related data'
+        Currently supported compiled binaries: Go, Rust.
+        """
+        self.scan_binaries = True
+
     def scan_for_application_packages(self):
         """
         Scan resources for package information to add DiscoveredPackage
@@ -61,6 +70,7 @@ def scan_for_application_packages(self):
         scancode.scan_for_application_packages(
             project=self.project,
             assemble=True,
+            compiled=self.scan_binaries or False,
             package_only=True,
             progress_logger=self.log,
         )

scanpipe/pipes/scancode.py

@@ -250,7 +250,13 @@ def scan_file(location, with_threading=True, min_license_score=0, **kwargs):
     return _scan_resource(location, scanners, with_threading=with_threading)
 
 
-def scan_for_package_data(location, with_threading=True, package_only=False, **kwargs):
+def scan_for_package_data(
+    location,
+    with_threading=True,
+    package_only=False,
+    compiled=False,
+    **kwargs,
+):
     """
     Run a package scan on provided `location` using the scancode-toolkit direct API.
 
@@ -259,6 +265,7 @@ def scan_for_package_data(location, with_threading=True, package_only=False, **k
     scancode_get_packages = partial(
         scancode_api.get_package_data,
         package_only=package_only,
+        compiled=compiled,
     )
     scanners = [
         Scanner("package_data", scancode_get_packages),
@@ -409,6 +416,7 @@ def scan_for_files(project, resource_qs=None, progress_logger=None):
 def scan_for_application_packages(
     project,
     assemble=True,
+    compiled=False,
     package_only=False,
     resource_qs=None,
     progress_logger=logger.info,
@@ -431,6 +439,7 @@ def scan_for_application_packages(
 
     scan_func_kwargs = {
         "package_only": package_only,
+        "compiled": compiled,
     }
 
     # Collect detected Package data and save it to the CodebaseResource it was

scanpipe/tests/data/asgiref/asgiref-3.3.0.spdx.json

@@ -1,11 +1,11 @@
 {
   "spdxVersion": "SPDX-2.3",
   "dataLicense": "CC0-1.0",
-  "SPDXID": "SPDXRef-DOCUMENT-3282ba3d-f525-4b74-9008-919108846d33",
+  "SPDXID": "SPDXRef-DOCUMENT-92fe63d9-1d53-4b63-b19a-85022fb7a3f3",
   "name": "scancodeio_asgiref",
-  "documentNamespace": "https://scancode.io/spdxdocs/3282ba3d-f525-4b74-9008-919108846d33",
+  "documentNamespace": "https://scancode.io/spdxdocs/92fe63d9-1d53-4b63-b19a-85022fb7a3f3",
   "documentDescribes": [
-    "SPDXRef-scancodeio-project-3282ba3d-f525-4b74-9008-919108846d33"
+    "SPDXRef-scancodeio-project-92fe63d9-1d53-4b63-b19a-85022fb7a3f3"
   ],
   "creationInfo": {
     "created": "2000-01-01T01:02:03Z",
@@ -17,7 +17,7 @@
   "packages": [
     {
       "name": "asgiref",
-      "SPDXID": "SPDXRef-scancodeio-project-3282ba3d-f525-4b74-9008-919108846d33",
+      "SPDXID": "SPDXRef-scancodeio-project-92fe63d9-1d53-4b63-b19a-85022fb7a3f3",
       "downloadLocation": "NOASSERTION",
       "licenseDeclared": "NOASSERTION",
       "licenseConcluded": "NOASSERTION",
@@ -26,7 +26,7 @@
     },
     {
       "name": "asgiref",
-      "SPDXID": "SPDXRef-scancodeio-discoveredpackage-b0467b22-5e83-4b0d-91e2-15121a8cf075",
+      "SPDXID": "SPDXRef-scancodeio-discoveredpackage-543a3583-3a13-4b5d-a039-c6bc4072de35",
       "downloadLocation": "NOASSERTION",
       "licenseDeclared": "BSD-3-Clause",
       "licenseConcluded": "BSD-3-Clause",
@@ -46,7 +46,7 @@
     },
     {
       "name": "asgiref",
-      "SPDXID": "SPDXRef-scancodeio-discoveredpackage-856001ca-aabf-4c6a-8ac2-cc1317f3e05d",
+      "SPDXID": "SPDXRef-scancodeio-discoveredpackage-b2913908-3a31-4460-b330-a74d14b5ee24",
       "downloadLocation": "NOASSERTION",
       "licenseDeclared": "BSD-3-Clause",
       "licenseConcluded": "BSD-3-Clause",
@@ -66,7 +66,7 @@
     },
     {
       "name": "pytest",
-      "SPDXID": "SPDXRef-scancodeio-discovereddependency-ef046d41-ae47-42c3-838c-def8bc956723",
+      "SPDXID": "SPDXRef-scancodeio-discovereddependency-19138025-32e9-4060-ab38-622b27493b6c",
       "downloadLocation": "NOASSERTION",
       "licenseDeclared": "NOASSERTION",
       "licenseConcluded": "NOASSERTION",
@@ -82,7 +82,7 @@
     },
     {
       "name": "pytest",
-      "SPDXID": "SPDXRef-scancodeio-discovereddependency-31fbc19e-c309-4463-8312-390ef734bf78",
+      "SPDXID": "SPDXRef-scancodeio-discovereddependency-c6282019-1112-43f8-a27a-658cc0c5dcf4",
       "downloadLocation": "NOASSERTION",
       "licenseDeclared": "NOASSERTION",
       "licenseConcluded": "NOASSERTION",
@@ -98,7 +98,7 @@
     },
     {
       "name": "pytest-asyncio",
-      "SPDXID": "SPDXRef-scancodeio-discovereddependency-3f928a8b-4505-4a7b-ad3a-a589dec836db",
+      "SPDXID": "SPDXRef-scancodeio-discovereddependency-52135390-385f-4fc0-8b0a-38f28c1040dd",
       "downloadLocation": "NOASSERTION",
       "licenseDeclared": "NOASSERTION",
       "licenseConcluded": "NOASSERTION",
@@ -114,7 +114,7 @@
     },
     {
       "name": "pytest-asyncio",
-      "SPDXID": "SPDXRef-scancodeio-discovereddependency-892b2af3-a9f0-4544-9874-583307d2d387",
+      "SPDXID": "SPDXRef-scancodeio-discovereddependency-01b33ae9-cb5f-442b-bfac-af8e40a1bf8c",
       "downloadLocation": "NOASSERTION",
       "licenseDeclared": "NOASSERTION",
       "licenseConcluded": "NOASSERTION",
@@ -132,33 +132,33 @@
   "files": [],
   "relationships": [
     {
-      "spdxElementId": "SPDXRef-scancodeio-project-3282ba3d-f525-4b74-9008-919108846d33",
-      "relatedSpdxElement": "SPDXRef-scancodeio-discoveredpackage-b0467b22-5e83-4b0d-91e2-15121a8cf075",
+      "spdxElementId": "SPDXRef-scancodeio-project-92fe63d9-1d53-4b63-b19a-85022fb7a3f3",
+      "relatedSpdxElement": "SPDXRef-scancodeio-discoveredpackage-543a3583-3a13-4b5d-a039-c6bc4072de35",
       "relationshipType": "DEPENDS_ON"
     },
     {
-      "spdxElementId": "SPDXRef-scancodeio-project-3282ba3d-f525-4b74-9008-919108846d33",
-      "relatedSpdxElement": "SPDXRef-scancodeio-discoveredpackage-856001ca-aabf-4c6a-8ac2-cc1317f3e05d",
+      "spdxElementId": "SPDXRef-scancodeio-project-92fe63d9-1d53-4b63-b19a-85022fb7a3f3",
+      "relatedSpdxElement": "SPDXRef-scancodeio-discoveredpackage-b2913908-3a31-4460-b330-a74d14b5ee24",
       "relationshipType": "DEPENDS_ON"
     },
     {
-      "spdxElementId": "SPDXRef-scancodeio-discovereddependency-ef046d41-ae47-42c3-838c-def8bc956723",
-      "relatedSpdxElement": "SPDXRef-scancodeio-discoveredpackage-b0467b22-5e83-4b0d-91e2-15121a8cf075",
+      "spdxElementId": "SPDXRef-scancodeio-discovereddependency-19138025-32e9-4060-ab38-622b27493b6c",
+      "relatedSpdxElement": "SPDXRef-scancodeio-discoveredpackage-543a3583-3a13-4b5d-a039-c6bc4072de35",
       "relationshipType": "DEPENDENCY_OF"
     },
     {
-      "spdxElementId": "SPDXRef-scancodeio-discovereddependency-31fbc19e-c309-4463-8312-390ef734bf78",
-      "relatedSpdxElement": "SPDXRef-scancodeio-discoveredpackage-856001ca-aabf-4c6a-8ac2-cc1317f3e05d",
+      "spdxElementId": "SPDXRef-scancodeio-discovereddependency-c6282019-1112-43f8-a27a-658cc0c5dcf4",
+      "relatedSpdxElement": "SPDXRef-scancodeio-discoveredpackage-b2913908-3a31-4460-b330-a74d14b5ee24",
       "relationshipType": "DEPENDENCY_OF"
     },
     {
-      "spdxElementId": "SPDXRef-scancodeio-discovereddependency-3f928a8b-4505-4a7b-ad3a-a589dec836db",
-      "relatedSpdxElement": "SPDXRef-scancodeio-discoveredpackage-b0467b22-5e83-4b0d-91e2-15121a8cf075",
+      "spdxElementId": "SPDXRef-scancodeio-discovereddependency-52135390-385f-4fc0-8b0a-38f28c1040dd",
+      "relatedSpdxElement": "SPDXRef-scancodeio-discoveredpackage-543a3583-3a13-4b5d-a039-c6bc4072de35",
       "relationshipType": "DEPENDENCY_OF"
     },
     {
-      "spdxElementId": "SPDXRef-scancodeio-discovereddependency-892b2af3-a9f0-4544-9874-583307d2d387",
-      "relatedSpdxElement": "SPDXRef-scancodeio-discoveredpackage-856001ca-aabf-4c6a-8ac2-cc1317f3e05d",
+      "spdxElementId": "SPDXRef-scancodeio-discovereddependency-01b33ae9-cb5f-442b-bfac-af8e40a1bf8c",
+      "relatedSpdxElement": "SPDXRef-scancodeio-discoveredpackage-b2913908-3a31-4460-b330-a74d14b5ee24",
       "relationshipType": "DEPENDENCY_OF"
     }
   ],