Add initial support for affected_by_commits and fixed_by_commits

Signed-off-by: ziad hany <ziadhany2016@gmail.com>

Author: ziadhany

vulnerabilities/importer.py

@@ -37,6 +37,7 @@
 from vulnerabilities.severity_systems import ScoringSystem
 from vulnerabilities.utils import classproperty
 from vulnerabilities.utils import get_reference_id
+from vulnerabilities.utils import is_commit
 from vulnerabilities.utils import is_cve
 from vulnerabilities.utils import nearest_patched_package
 from vulnerabilities.utils import purl_to_dict
@@ -194,6 +195,57 @@ def from_url(cls, url):
         return cls(url=url)
 
 
+@dataclasses.dataclass(eq=True)
+@functools.total_ordering
+class Commit:
+    commit_hash: str
+    vcs_url: str
+
+    commit_rank: Optional[int] = 0
+    commit_author: Optional[str] = None
+    commit_message: Optional[str] = None
+    # commit_date: Optional[datetime] = None
+
+    def __post_init__(self):
+        if not self.commit_hash:
+            raise ValueError("Commit must have a non-empty commit_hash.")
+        if not self.vcs_url:
+            raise ValueError("Commit must have a non-empty vcs_url.")
+        if not isinstance(self.commit_hash, str):
+            self.commit_hash = str(self.commit_hash)
+
+    def _cmp_key(self):
+        return (self.commit_rank, self.commit_hash, self.vcs_url)
+
+    def __lt__(self, other):
+        if not isinstance(other, Commit):
+            return NotImplemented
+        return self.commit_rank < other.commit_rank
+
+    def to_dict(self) -> dict:
+        """Return a normalized dictionary representation of the commit."""
+        return {
+            "commit_hash": self.commit_hash,
+            "vcs_url": self.vcs_url,
+            "commit_rank": self.commit_rank,
+            "commit_author": self.commit_author,
+            "commit_message": self.commit_message,
+            # "commit_date": self.commit_date.isoformat() if self.commit_date else None,
+        }
+
+    @classmethod
+    def from_dict(cls, data: dict) -> "Commit":
+        """Create a Commit instance from a dictionary."""
+        return cls(
+            commit_hash=str(data.get("commit_hash", "")),
+            vcs_url=data.get("vcs_url", ""),
+            commit_rank=data.get("commit_rank", 0),
+            commit_author=data.get("commit_author"),
+            commit_message=data.get("commit_message"),
+            # commit_date=data.get("commit_date"),
+        )
+
+
 class UnMergeablePackageError(Exception):
     """
     Raised when a package cannot be merged with another one.
@@ -218,6 +270,8 @@ class AffectedPackage:
     package: PackageURL
     affected_version_range: Optional[VersionRange] = None
     fixed_version: Optional[Version] = None
+    fixed_by_commits: List[Commit] = dataclasses.field(default_factory=list)
+    affected_by_commits: List[Commit] = dataclasses.field(default_factory=list)
 
     def __post_init__(self):
         if self.package.version:
@@ -248,6 +302,8 @@ def _cmp_key(self):
             str(self.package),
             str(self.affected_version_range or ""),
             str(self.fixed_version or ""),
+            str(self.affected_by_commits or []),
+            str(self.fixed_by_commits or []),
         )
 
     @classmethod
@@ -294,6 +350,12 @@ def to_dict(self):
             "package": purl_to_dict(self.package),
             "affected_version_range": affected_version_range,
             "fixed_version": str(self.fixed_version) if self.fixed_version else None,
+            "affected_by_commits": [
+                affected_by_commit.to_dict() for affected_by_commit in self.affected_by_commits
+            ],
+            "fixed_by_commits": [
+                fixed_by_commit.to_dict() for fixed_by_commit in self.fixed_by_commits
+            ],
         }
 
     @classmethod
@@ -304,6 +366,8 @@ def from_dict(cls, affected_pkg: dict):
         package = PackageURL(**affected_pkg["package"])
         affected_version_range = None
         affected_range = affected_pkg["affected_version_range"]
+        affected_by_commits = affected_pkg.get("affected_by_commits") or []
+        fixed_by_commits = affected_pkg.get("fixed_by_commits") or []
 
         # TODO: "None" is a likely bug
         if affected_range and affected_range != "None":
@@ -335,6 +399,12 @@ def from_dict(cls, affected_pkg: dict):
             package=package,
             affected_version_range=affected_version_range,
             fixed_version=fixed_version,
+            affected_by_commits=[
+                Commit.from_dict(affected_by_commit) for affected_by_commit in affected_by_commits
+            ],
+            fixed_by_commits=[
+                Commit.from_dict(fixed_by_commit) for fixed_by_commit in fixed_by_commits
+            ],
         )
 
 
@@ -350,6 +420,8 @@ class AffectedPackageV2:
     package: PackageURL
     affected_version_range: Optional[VersionRange] = None
     fixed_version_range: Optional[VersionRange] = None
+    fixed_by_commits: List[Commit] = dataclasses.field(default_factory=list)
+    affected_by_commits: List[Commit] = dataclasses.field(default_factory=list)
 
     def __post_init__(self):
         if self.package.version:
@@ -372,6 +444,8 @@ def _cmp_key(self):
             str(self.package),
             str(self.affected_version_range or ""),
             str(self.fixed_version_range or ""),
+            str(self.affected_by_commits or []),
+            str(self.fixed_by_commits or []),
         )
 
     def to_dict(self):
@@ -385,6 +459,12 @@ def to_dict(self):
             "package": purl_to_dict(self.package),
             "affected_version_range": affected_version_range,
             "fixed_version_range": fixed_version_range,
+            "affected_by_commits": [
+                affected_by_commit.to_dict() for affected_by_commit in self.affected_by_commits
+            ],
+            "fixed_by_commits": [
+                fixed_by_commit.to_dict() for fixed_by_commit in self.fixed_by_commits
+            ],
         }
 
     @classmethod
@@ -396,6 +476,8 @@ def from_dict(cls, affected_pkg: dict):
         fixed_version_range = None
         affected_range = affected_pkg["affected_version_range"]
         fixed_range = affected_pkg["fixed_version_range"]
+        affected_by_commits = affected_pkg.get("affected_by_commits") or []
+        fixed_by_commits = affected_pkg.get("fixed_by_commits") or []
 
         try:
             affected_version_range = VersionRange.from_string(affected_range)
@@ -413,10 +495,27 @@ def from_dict(cls, affected_pkg: dict):
             )
             return
 
+        invalid_fix_commits = [c for c in fixed_by_commits if not is_commit(c.commit_hash)]
+        invalid_affected_commits = [c for c in affected_by_commits if not is_commit(c.commit_hash)]
+
+        if invalid_fix_commits or invalid_affected_commits:
+            logger.error(
+                f"Invalid commit hash(es) found. "
+                f"Invalid fixed_by_commits: {invalid_fix_commits}, "
+                f"Invalid affected_by_commits: {invalid_affected_commits}"
+            )
+            return
+
         return cls(
             package=package,
             affected_version_range=affected_version_range,
             fixed_version_range=fixed_version_range,
+            affected_by_commits=[
+                Commit.from_dict(affected_by_commit) for affected_by_commit in affected_by_commits
+            ],
+            fixed_by_commits=[
+                Commit.from_dict(fixed_by_commit) for fixed_by_commit in fixed_by_commits
+            ],
         )
 
 

vulnerabilities/importers/curl.py

@@ -97,7 +97,7 @@ def parse_advisory_data(raw_data) -> AdvisoryData:
         ...     ]
         ... }
         >>> parse_advisory_data(raw_data)
-        AdvisoryData(advisory_id='', aliases=['CVE-2024-2379'], summary='QUIC certificate check bypass with wolfSSL', affected_packages=[AffectedPackage(package=PackageURL(type='generic', namespace='curl.se', name='curl', version=None, qualifiers={}, subpath=None), affected_version_range=GenericVersionRange(constraints=(VersionConstraint(comparator='=', version=SemverVersion(string='8.6.0')),)), fixed_version=SemverVersion(string='8.7.0'))], references=[Reference(reference_id='', reference_type='', url='https://curl.se/docs/CVE-2024-2379.html', severities=[VulnerabilitySeverity(system=Cvssv3ScoringSystem(identifier='cvssv3.1', name='CVSSv3.1 Base Score', url='https://www.first.org/cvss/v3-1/', notes='CVSSv3.1 base score and vector'), value='Low', scoring_elements='', published_at=None, url=None)]), Reference(reference_id='', reference_type='', url='https://hackerone.com/reports/2410774', severities=[])], references_v2=[], date_published=datetime.datetime(2024, 3, 27, 8, 0, tzinfo=datetime.timezone.utc), weaknesses=[297], severities=[], url='https://curl.se/docs/CVE-2024-2379.json', original_advisory_text=None)
+        AdvisoryData(advisory_id='', aliases=['CVE-2024-2379'], summary='QUIC certificate check bypass with wolfSSL', affected_packages=[AffectedPackage(package=PackageURL(type='generic', namespace='curl.se', name='curl', version=None, qualifiers={}, subpath=None), affected_version_range=GenericVersionRange(constraints=(VersionConstraint(comparator='=', version=SemverVersion(string='8.6.0')),)), fixed_version=SemverVersion(string='8.7.0')),"affected_by_commits": [],"fixed_by_commits": []], references=[Reference(reference_id='', reference_type='', url='https://curl.se/docs/CVE-2024-2379.html', severities=[VulnerabilitySeverity(system=Cvssv3ScoringSystem(identifier='cvssv3.1', name='CVSSv3.1 Base Score', url='https://www.first.org/cvss/v3-1/', notes='CVSSv3.1 base score and vector'), value='Low', scoring_elements='', published_at=None, url=None)]), Reference(reference_id='', reference_type='', url='https://hackerone.com/reports/2410774', severities=[])], references_v2=[], date_published=datetime.datetime(2024, 3, 27, 8, 0, tzinfo=datetime.timezone.utc), weaknesses=[297], severities=[], url='https://curl.se/docs/CVE-2024-2379.json', original_advisory_text=None)
     """
 
     affected = get_item(raw_data, "affected")[0] if len(get_item(raw_data, "affected")) > 0 else []

vulnerabilities/models.py

@@ -2982,6 +2982,8 @@ def to_dict(self):
             "package": purl_to_dict(self.base_purl),
             "affected_version_range": self.affecting_vers,
             "fixed_version_range": self.fixed_vers,
+            "affected_by_commits": [commit.to_dict() for commit in self.affecting_commits.all()],
+            "fixed_by_commits": [commit.to_dict() for commit in self.fixed_by_commits.all()],
         }
 
     def to_affected_package_data(self):

vulnerabilities/pipes/advisory.py

@@ -29,6 +29,7 @@
 from vulnerabilities.models import AdvisoryWeakness
 from vulnerabilities.models import AffectedByPackageRelatedVulnerability
 from vulnerabilities.models import Alias
+from vulnerabilities.models import CodeCommit
 from vulnerabilities.models import FixingPackageRelatedVulnerability
 from vulnerabilities.models import Package
 from vulnerabilities.models import VulnerabilityReference
@@ -96,6 +97,37 @@ def get_or_create_advisory_weaknesses(weaknesses: List[str]) -> List[AdvisoryWea
     return list(AdvisoryWeakness.objects.filter(cwe_id__in=weaknesses))
 
 
+def get_or_create_advisory_code_commits(code_commits: List) -> List["CodeCommit"]:
+    """
+    Given a list of affected commit objects (each with commit_hash and vcs_url),
+    create any missing CodeCommit entries and return the full list of CodeCommit objects.
+    """
+
+    potential_matches = CodeCommit.objects.filter(
+        vcs_url__in=[v for v, _ in code_commits],
+        commit_hash__in=[h for _, h in code_commits],
+    )
+
+    existing_commits = {c for c in potential_matches if (c.vcs_url, c.commit_hash) in code_commits}
+
+    to_create = [
+        CodeCommit(
+            commit_hash=commit.commit_hash,
+            vcs_url=commit.vcs_url,
+            commit_author=getattr(commit, "commit_author", None),
+            commit_message=getattr(commit, "commit_message", None),
+            commit_date=getattr(commit, "commit_date", None),
+        )
+        for commit in code_commits
+        if commit not in existing_commits
+    ]
+
+    if to_create:
+        CodeCommit.objects.bulk_create(to_create, ignore_conflicts=True)
+
+    return list(existing_commits)
+
+
 def insert_advisory(advisory: AdvisoryData, pipeline_id: str, logger: Callable = None):
     from vulnerabilities.utils import compute_content_id
 
@@ -216,6 +248,13 @@ def insert_advisory_v2(
             impact.affecting_packages.add(*affected_packages_v2)
             impact.fixed_by_packages.add(*fixed_packages_v2)
 
+            affected_commit_v2 = get_or_create_advisory_code_commits(
+                affected_pkg.affected_by_commits
+            )
+            fixed_commit_v2 = get_or_create_advisory_code_commits(affected_pkg.fixed_by_commits)
+            impact.affecting_packages.add(*affected_commit_v2)
+            impact.fixed_by_packages.add(*fixed_commit_v2)
+
     return advisory_obj
 
 

vulnerabilities/tests/pipes/test_advisory.py

@@ -19,12 +19,14 @@
 from vulnerabilities import models
 from vulnerabilities.importer import AdvisoryData
 from vulnerabilities.importer import AffectedPackage
+from vulnerabilities.importer import Commit
 from vulnerabilities.importer import Reference
 from vulnerabilities.models import AdvisoryAlias
 from vulnerabilities.models import AdvisoryReference
 from vulnerabilities.models import AdvisorySeverity
 from vulnerabilities.models import AdvisoryWeakness
 from vulnerabilities.pipes.advisory import get_or_create_advisory_aliases
+from vulnerabilities.pipes.advisory import get_or_create_advisory_code_commits
 from vulnerabilities.pipes.advisory import get_or_create_advisory_references
 from vulnerabilities.pipes.advisory import get_or_create_advisory_severities
 from vulnerabilities.pipes.advisory import get_or_create_advisory_weaknesses
@@ -41,6 +43,8 @@ def setUp(self):
                 AffectedPackage(
                     package=PackageURL(type="pypi", name="dummy"),
                     affected_version_range=VersionRange.from_string("vers:pypi/>=1.0.0|<=2.0.0"),
+                    affected_by_commits=[Commit(commit_hash="ab45wa4", vcs_url="http://test")],
+                    fixed_by_commits=[Commit(commit_hash="78wte7", vcs_url="http://test")],
                 )
             ],
             references=[Reference(url="https://example.com/with/more/info/CVE-2020-13371337")],
@@ -160,6 +164,20 @@ def advisory_references():
     ]
 
 
+@pytest.fixture
+def advisory_commit():
+    return [
+        Commit(
+            commit_hash="ef1659c01708b2111d6f06e2aa32f0f9d8768e10",
+            vcs_url="https://github.com/aboutcode-org/vulnerablecode",
+        ),
+        Commit(
+            commit_hash="eccbb45ac2d9c0eb7e22ea82d1fc49f9f4cda818",
+            vcs_url="https://github.com/aboutcode-org/vulnerablecode",
+        ),
+    ]
+
+
 @pytest.fixture
 def advisory_severities():
     class Severity:
@@ -225,3 +243,13 @@ def test_get_or_create_advisory_weaknesses(advisory_weaknesses):
     for w in weaknesses:
         assert isinstance(w, AdvisoryWeakness)
         assert w.cwe_id in advisory_weaknesses
+
+
+@pytest.mark.django_db
+def test_get_or_create_advisory_commit(advisory_commit):
+    commits = get_or_create_advisory_code_commits(advisory_commit)
+    assert len(commits) == len(advisory_commit)
+    for commit in commits:
+        assert isinstance(commit, Commit)
+        assert commit.commit_hash in [c.commit_hash for c in advisory_commit]
+        assert commit.vcs_url in [c.vcs_url for c in advisory_commit]

vulnerabilities/tests/pipes/test_vulnerablecode_importer_pipeline_v2.py

@@ -16,9 +16,9 @@
 from packageurl import PackageURL
 from univers.version_range import VersionRange
 
-from vulnerabilities.importer import AdvisoryData
+from vulnerabilities.importer import AdvisoryData, Commit
 from vulnerabilities.importer import AffectedPackageV2
-from vulnerabilities.models import AdvisoryV2
+from vulnerabilities.models import AdvisoryV2, CodeCommit
 from vulnerabilities.models import ImpactedPackage
 from vulnerabilities.models import PackageV2
 from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2
@@ -51,11 +51,15 @@ def dummy_advisory():
                 package=PackageURL.from_string("pkg:npm/foobar"),
                 affected_version_range=VersionRange.from_string("vers:npm/<=1.2.3"),
                 fixed_version_range=VersionRange.from_string("vers:npm/1.2.4"),
+                affected_by_commits=[Commit(commit_hash="ab45wa4", vcs_url="http://test")],
+                fixed_by_commits=[Commit(commit_hash="78wte7", vcs_url="http://test")],
             ),
             AffectedPackageV2(
                 package=PackageURL.from_string("pkg:npm/foobar"),
                 affected_version_range=VersionRange.from_string("vers:npm/<=3.2.3"),
                 fixed_version_range=VersionRange.from_string("vers:npm/3.2.4"),
+                affected_by_commits=[Commit(commit_hash="ab45wa4", vcs_url="http://test")],
+                fixed_by_commits=[Commit(commit_hash="78wte7", vcs_url="http://test")],
             ),
         ],
         advisory_id="ADV-123",
@@ -92,4 +96,5 @@ def test_advisory_import_atomicity(dummy_importer):
     dummy_importer.collect_and_store_advisories()
     assert AdvisoryV2.objects.count() == 1
     assert ImpactedPackage.objects.count() == 2
+    assert CodeCommit.objects.count() == 2
     assert PackageV2.objects.count() == 4

vulnerabilities/tests/test_data/apache_httpd/CVE-1999-1199-apache-httpd-expected.json

@@ -14,7 +14,9 @@
         "subpath": ""
       },
       "affected_version_range": "vers:apache/1.3.0|1.3.1|!=1.3.2",
-      "fixed_version": null
+      "fixed_version": null,
+      "affected_by_commits": [],
+      "fixed_by_commits": []
     }
   ],
   "references": [

vulnerabilities/tests/test_data/apache_httpd/CVE-2017-9798-apache-httpd-expected.json

@@ -14,7 +14,9 @@
         "subpath": ""
       },
       "affected_version_range": "vers:apache/2.2.0|2.2.2|2.2.3|2.2.4|2.2.5|2.2.6|2.2.8|2.2.9|2.2.10|2.2.11|2.2.12|2.2.13|2.2.14|2.2.15|2.2.16|2.2.17|2.2.18|2.2.19|2.2.20|2.2.21|2.2.22|2.2.23|2.2.24|2.2.25|2.2.26|2.2.27|2.2.29|2.2.31|2.2.32|2.2.34|2.4.1|2.4.2|2.4.3|2.4.4|2.4.6|2.4.7|2.4.9|2.4.10|2.4.12|2.4.16|2.4.17|2.4.18|2.4.20|2.4.23|2.4.25|2.4.26|2.4.27|!=2.4.28",
-      "fixed_version": null
+      "fixed_version": null,
+      "affected_by_commits": [],
+      "fixed_by_commits": []
     }
   ],
   "references": [

vulnerabilities/tests/test_data/apache_httpd/CVE-2021-44224-apache-httpd-expected.json

@@ -14,7 +14,9 @@
         "subpath": ""
       },
       "affected_version_range": "vers:apache/>=2.4.7|<=2.4.51|!=2.4.52",
-      "fixed_version": null
+      "fixed_version": null,
+      "affected_by_commits": [],
+      "fixed_by_commits": []
     }
   ],
   "references": [