Add support for Gradle module metadata

src/packagedcode/__init__.py

@@ -25,6 +25,7 @@
 from packagedcode import freebsd
 from packagedcode import godeps
 from packagedcode import golang
+from packagedcode import gradle
 from packagedcode import haxe
 from packagedcode import maven
 from packagedcode import misc
@@ -56,6 +57,7 @@
     bower.BowerJsonHandler,
 
     build_gradle.BuildGradleHandler,
+    gradle.GradleModuleHandler,
 
     build.AutotoolsConfigureHandler,
     build.BazelBuildHandler,

src/packagedcode/gradle.py

@@ -0,0 +1,122 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# ScanCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/nexB/scancode-toolkit for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+import json
+
+from packageurl import PackageURL
+
+from packagedcode import models
+
+
+class GradleModuleHandler(models.DatafileHandler):
+    datasource_id = 'gradle_module'
+    path_patterns = ('*.module',)
+    default_package_type = 'maven'
+    description = 'Gradle module metadata file'
+
+    @classmethod
+    def parse(cls, location, package_only=False):
+        try:
+            with open(location, 'r') as f:
+                data = json.load(f)
+        except Exception:
+            return
+
+        if not data or not isinstance(data, dict):
+            return
+
+        component = data.get('component', {})
+        if not component:
+            return
+
+        namespace = component.get('group', '')
+        namespace = namespace if namespace else None
+        name = component.get('module', '')
+        name = name if name else None
+        version = component.get('version', '')
+        version = version if version else None
+
+        created_by = data.get('createdBy', {})
+        gradle_version = created_by.get('gradle', {}).get('version')
+
+        variants = data.get('variants', [])
+        seen_deps = {}
+        files = []
+
+        for variant in variants:
+            variant_name = variant.get('name', '')
+            attributes = variant.get('attributes', {})
+            usage = attributes.get('org.gradle.usage', variant_name)
+
+            is_runtime = 'runtime' in usage.lower()
+            scope = 'runtime' if is_runtime else 'api'
+
+            for dep in variant.get('dependencies', []):
+                dep_namespace = dep.get('group', '')
+                dep_namespace = dep_namespace if dep_namespace else None
+                dep_name = dep.get('module', '')
+                dep_version_info = dep.get('version', {})
+
+                if isinstance(dep_version_info, dict):
+                    dep_version = dep_version_info.get('requires') or \
+                                  dep_version_info.get('prefers') or \
+                                  dep_version_info.get('strictly', '')
+                else:
+                    dep_version = str(dep_version_info) if dep_version_info else None
+
+                if not dep_version:
+                    dep_version = None
+
+                if not dep_name:
+                    continue
+
+                key = (dep_namespace, dep_name, dep_version)
+                if key not in seen_deps:
+                    seen_deps[key] = models.DependentPackage(
+                        purl=str(PackageURL(
+                            type='maven',
+                            namespace=dep_namespace,
+                            name=dep_name,
+                            version=dep_version
+                        )),
+                        extracted_requirement=dep_version,
+                        scope=scope,
+                        is_runtime=is_runtime,
+                        is_optional=False,
+                    )
+
+            # Extract files from the first variant that has them
+            if not files and variant.get('files'):
+                files = variant.get('files', [])
+
+        extra_data = {}
+        if gradle_version:
+            extra_data['gradle_version'] = gradle_version
+        format_version = data.get('formatVersion')
+        if format_version:
+            extra_data['format_version'] = format_version
+
+        package_data = dict(
+            datasource_id=cls.datasource_id,
+            type=cls.default_package_type,
+            namespace=namespace,
+            name=name,
+            version=version,
+            dependencies=list(seen_deps.values()),
+            extra_data=extra_data,
+        )
+
+        # Store file checksums on the PackageData if available from the first variant files
+        for f in files:
+            for algo in ('sha1', 'sha256', 'sha512', 'md5'):
+                if f.get(algo):
+                    package_data[algo] = f.get(algo)
+            break
+
+        yield models.PackageData.from_data(package_data, package_only)

tests/packagedcode/data/gradle/module/dependency-management-plugin-1.1.3.module

@@ -0,0 +1,42 @@
+{
+  "formatVersion": "1.1",
+  "component": {
+    "group": "io.spring.gradle",
+    "module": "dependency-management-plugin",
+    "version": "1.1.3",
+    "attributes": {
+      "org.gradle.status": "release"
+    }
+  },
+  "createdBy": {
+    "gradle": {
+      "version": "7.6.1"
+    }
+  },
+  "variants": [
+    {
+      "name": "apiElements",
+      "attributes": {
+        "org.gradle.category": "library",
+        "org.gradle.usage": "java-api"
+      },
+      "dependencies": [
+        {
+          "group": "org.springframework.boot",
+          "module": "spring-boot",
+          "version": {
+            "requires": "3.1.0"
+          }
+        }
+      ],
+      "files": [
+        {
+          "name": "dependency-management-plugin-1.1.3.jar",
+          "url": "dependency-management-plugin-1.1.3.jar",
+          "sha1": "3209385654a7e661d68de95a5ea8fc11d8ce015e",
+          "md5": "abc123"
+        }
+      ]
+    }
+  ]
+}

tests/packagedcode/data/gradle/module/material-1.9.0.module

@@ -0,0 +1,74 @@
+{
+  "formatVersion": "1.1",
+  "component": {
+    "group": "com.google.android.material",
+    "module": "material",
+    "version": "1.9.0",
+    "attributes": {
+      "org.gradle.status": "release"
+    }
+  },
+  "createdBy": {
+    "gradle": {
+      "version": "7.3.3"
+    }
+  },
+  "variants": [
+    {
+      "name": "releaseApiElements",
+      "attributes": {
+        "org.gradle.category": "library",
+        "org.gradle.usage": "java-api"
+      },
+      "dependencies": [
+        {
+          "group": "androidx.annotation",
+          "module": "annotation",
+          "version": {
+            "requires": "1.2.0"
+          }
+        },
+        {
+          "group": "androidx.appcompat",
+          "module": "appcompat",
+          "version": {
+            "requires": "1.5.0"
+          }
+        }
+      ],
+      "files": [
+        {
+          "name": "material-1.9.0.aar",
+          "url": "material-1.9.0.aar",
+          "sha512": "7630aacb9e3073b2064397ed080b8d5bf7db06ba2022d6c927e05b7d53c5787d",
+          "sha256": "6cc2359979269e4d9eddce7d84682d2bb06a35a14edce806bf0da6e8d4d31806",
+          "sha1": "08f4a93a381be223a5bbaacd46eaab92381ab6a8",
+          "md5": "3287103cfb083fb998a35ef8a1983c58"
+        }
+      ]
+    },
+    {
+      "name": "releaseRuntimeElements",
+      "attributes": {
+        "org.gradle.category": "library",
+        "org.gradle.usage": "java-runtime"
+      },
+      "dependencies": [
+        {
+          "group": "com.google.errorprone",
+          "module": "error_prone_annotations",
+          "version": {
+            "requires": "2.15.0"
+          }
+        },
+        {
+          "group": "androidx.annotation",
+          "module": "annotation",
+          "version": {
+            "requires": "1.2.0"
+          }
+        }
+      ]
+    }
+  ]
+}

tests/packagedcode/data/gradle/module/no-deps.module

@@ -0,0 +1,16 @@
+{
+  "formatVersion": "1.1",
+  "component": {
+    "group": "org.example",
+    "module": "standalone",
+    "version": "2.0.0"
+  },
+  "variants": [
+    {
+      "name": "apiElements",
+      "attributes": {},
+      "dependencies": [],
+      "files": []
+    }
+  ]
+}

tests/packagedcode/data/gradle/module/simple.module

@@ -0,0 +1,9 @@
+{
+  "formatVersion": "1.1",
+  "component": {
+    "group": "com.example",
+    "module": "mylib",
+    "version": "1.0.0"
+  },
+  "variants": []
+}

tests/packagedcode/test_gradle_module.py

@@ -0,0 +1,112 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# ScanCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/nexB/scancode-toolkit for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+import io
+import json
+import os.path
+
+import pytest
+
+from commoncode import fileutils
+from commoncode import text
+from commoncode import testcase
+
+from packagedcode import gradle
+from packagedcode import models
+from scancode.cli_test_utils import check_json_scan
+from scancode.cli_test_utils import run_scan_click
+from scancode_config import REGEN_TEST_FIXTURES
+
+
+def parse_module(location=None):
+    """
+    Return a PackageData mapping from the Gradle module file at location.
+    """
+    packages = list(gradle.GradleModuleHandler.parse(location=location))
+    if not packages:
+        return {}
+    package = packages[0]
+    return package.to_dict()
+
+
+class TestGradleModule(testcase.FileBasedTesting):
+    test_data_dir = os.path.join(os.path.dirname(__file__), 'data')
+
+    def test_gradle_module_is_datafile(self):
+        test_dir = self.get_test_loc('gradle/module')
+        material_module = os.path.join(test_dir, 'material-1.9.0.module')
+        simple_module = os.path.join(test_dir, 'simple.module')
+
+        assert gradle.GradleModuleHandler.is_datafile(material_module)
+        assert gradle.GradleModuleHandler.is_datafile(simple_module)
+
+        # Create a fake JSON file
+        fake_json = self.get_temp_file('json')
+        with open(fake_json, 'w') as f:
+            f.write('{"foo": "bar"}')
+        assert not gradle.GradleModuleHandler.is_datafile(fake_json)
+
+    def test_parse_material_module(self):
+        test_loc = self.get_test_loc('gradle/module/material-1.9.0.module')
+        result = parse_module(test_loc)
+
+        assert result.get('type') == 'maven'
+        assert result.get('namespace') == 'com.google.android.material'
+        assert result.get('name') == 'material'
+        assert result.get('version') == '1.9.0'
+
+        deps = result.get('dependencies', [])
+        assert len(deps) == 3
+
+        purls = [d.get('purl') for d in deps]
+        assert 'pkg:maven/androidx.annotation/annotation@1.2.0' in purls
+        assert 'pkg:maven/androidx.appcompat/appcompat@1.5.0' in purls
+        assert 'pkg:maven/com.google.errorprone/error_prone_annotations@2.15.0' in purls
+
+        extra = result.get('extra_data', {})
+        assert extra.get('gradle_version') == '7.3.3'
+        assert extra.get('format_version') == '1.1'
+
+    def test_parse_simple_module(self):
+        test_loc = self.get_test_loc('gradle/module/simple.module')
+        result = parse_module(test_loc)
+
+        assert result.get('namespace') == 'com.example'
+        assert result.get('name') == 'mylib'
+        assert result.get('version') == '1.0.0'
+        assert result.get('dependencies') == []
+
+    def test_parse_no_deps_module(self):
+        test_loc = self.get_test_loc('gradle/module/no-deps.module')
+        # Does not crash
+        result = parse_module(test_loc)
+
+        assert result.get('namespace') == 'org.example'
+        assert result.get('name') == 'standalone'
+        assert result.get('dependencies') == []
+
+    def test_parse_spring_module(self):
+        test_loc = self.get_test_loc('gradle/module/dependency-management-plugin-1.1.3.module')
+        result = parse_module(test_loc)
+
+        assert result.get('namespace') == 'io.spring.gradle'
+        assert result.get('name') == 'dependency-management-plugin'
+        assert result.get('version') == '1.1.3'
+
+        deps = result.get('dependencies', [])
+        assert len(deps) == 1
+        assert deps[0].get('purl') == 'pkg:maven/org.springframework.boot/spring-boot@3.1.0'
+
+    def test_dependencies_deduplicated(self):
+        test_loc = self.get_test_loc('gradle/module/material-1.9.0.module')
+        result = parse_module(test_loc)
+
+        deps = result.get('dependencies', [])
+        purls = [d.get('purl') for d in deps]
+        assert len(purls) == len(set(purls))