Add code commit for VCS URLs that we support (github, bitbucket anf gitlab) for unsupported store them as references.

Add a test for OSV

Signed-off-by: ziad hany <[email protected]>

Author: ziadhany

vulnerabilities/importer.py

@@ -210,7 +210,7 @@ def __post_init__(self):
             raise ValueError("Commit must have a non-empty commit_hash.")
 
         if not is_commit(self.commit_hash):
-            raise ValueError("Commit must be a valid a commit_hash.")
+            raise ValueError(f"Commit must be a valid a commit_hash: {self.commit_hash}.")
 
         if not self.vcs_url:
             raise ValueError("Commit must have a non-empty vcs_url.")
@@ -426,8 +426,8 @@ def __post_init__(self):
             or self.fixed_by_commits
         ):
             raise ValueError(
-                f"Affected Package {self.package!r} should have either fixed version range or an "
-                "affected version range."
+                f"Affected package {self.package!r} must have either a fixed version range, "
+                "an affected version range, affected commits, or fixed commits."
             )
 
     def __lt__(self, other):
@@ -456,12 +456,8 @@ def to_dict(self):
             "package": purl_to_dict(self.package),
             "affected_version_range": affected_version_range,
             "fixed_version_range": fixed_version_range,
-            "affected_by_commits": [
-                affected_by_commit.to_dict() for affected_by_commit in self.affected_by_commits
-            ],
-            "fixed_by_commits": [
-                fixed_by_commit.to_dict() for fixed_by_commit in self.fixed_by_commits
-            ],
+            "affected_by_commits": [commit.to_dict() for commit in self.affected_by_commits],
+            "fixed_by_commits": [commit.to_dict() for commit in self.fixed_by_commits],
         }
 
     @classmethod
@@ -497,12 +493,9 @@ def from_dict(cls, affected_pkg: dict):
             affected_version_range=affected_version_range,
             fixed_version_range=fixed_version_range,
             affected_by_commits=[
-                CodeCommitData.from_dict(affected_by_commit)
-                for affected_by_commit in affected_by_commits
-            ],
-            fixed_by_commits=[
-                CodeCommitData.from_dict(fixed_by_commit) for fixed_by_commit in fixed_by_commits
+                CodeCommitData.from_dict(commit) for commit in affected_by_commits
             ],
+            fixed_by_commits=[CodeCommitData.from_dict(commit) for commit in fixed_by_commits],
         )
 
 

vulnerabilities/importers/osv.py

@@ -10,6 +10,7 @@
 import json
 import logging
 from typing import Iterable
+from typing import Iterator
 from typing import List
 from typing import Optional
 from typing import Tuple
@@ -18,10 +19,12 @@
 from cvss.exceptions import CVSS3MalformedError
 from cvss.exceptions import CVSS4MalformedError
 from packageurl import PackageURL
+from packageurl.contrib.url2purl import url2purl
 from univers.version_range import RANGE_CLASS_BY_SCHEMES
 from univers.versions import InvalidVersion
 from univers.versions import SemverVersion
 from univers.versions import Version
+from websockets.version import commit
 
 from vulnerabilities.importer import AdvisoryData
 from vulnerabilities.importer import AffectedPackage
@@ -49,6 +52,8 @@
     "cargo": "cargo",
 }
 
+SUPPORTED_VCS_TYPES = {"github", "bitbucket", "gitlab"}
+
 
 def parse_advisory_data(
     raw_data: dict, supported_ecosystems, advisory_url: str
@@ -164,11 +169,30 @@ def parse_advisory_data_v2(
             get_fixed_version_range(fixed_versions, purl.type) if fixed_versions else None
         )
 
+        if not purl and (affected_by_commits or fixed_by_commits):
+            for code_commit in affected_by_commits + fixed_by_commits:
+                purl = url2purl(code_commit.vcs_url)
+                if purl.type not in SUPPORTED_VCS_TYPES:
+                    logger.error(
+                        f"Storing commit_hash:{code_commit.commit_hash}, code_commit:{code_commit.vcs_url} as reference, not creating CodeCommits."
+                    )
+                    ref = ReferenceV2(
+                        reference_id=code_commit.commit_hash,
+                        reference_type="commit",
+                        url=code_commit.vcs_url,
+                    )
+                    references.append(ref)
+
+                    affected_by_commits, fixed_by_commits = [], []
+
         if not purl or purl.type not in supported_ecosystems:
             logger.error(f"Unsupported package type: {purl!r} in OSV: {advisory_id!r}")
-            continue
+            # Ignore version ranges not associated with the same supported_ecosystems.
+            fixed_version_range, affected_version_range = None, None
 
-        if fixed_version_range or affected_version_range or fixed_by_commits or affected_by_commits:
+        if purl and (
+            fixed_version_range or affected_version_range or fixed_by_commits or affected_by_commits
+        ):
             affected_packages.append(
                 AffectedPackageV2(
                     package=purl,
@@ -200,7 +224,7 @@ def parse_advisory_data_v2(
     )
 
 
-def extract_introduced_and_fixed(ranges) -> Tuple[List[str], List[str]]:
+def extract_introduced_and_fixed(ranges) -> Iterator[Tuple[Optional[str], Optional[str]]]:
     """
     Return pairs of introduced and fixed versions or commit hashes given a ``ranges``
     mapping of OSV data.

vulnerabilities/tests/pipelines/v2_importers/test_oss_fuzz_v2.py

@@ -6,13 +6,19 @@
 # See https://github.com/aboutcode-org/vulnerablecode for support or download.
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
+import os
 from unittest import mock
 
 import pytest
+import saneyaml
 import yaml
 
-from vulnerabilities.importer import AdvisoryData
+from vulnerabilities.importers.osv import parse_advisory_data_v2
 from vulnerabilities.pipelines.v2_importers.oss_fuzz import OSSFuzzImporterPipeline
+from vulnerabilities.tests import util_tests
+
+BASE_DIR = os.path.dirname(os.path.abspath(__file__))
+TEST_DATA = os.path.join(BASE_DIR, "../../test_data/oss_fuzz/v2")
 
 
 @pytest.mark.django_db
@@ -49,3 +55,24 @@ def test_advisories_count(tmp_path):
     pipeline.vcs_response.dest_dir = tmp_path
 
     assert pipeline.advisories_count() == 2
+
+
+class TestOSSFuzzImporter:
+    @pytest.mark.parametrize(
+        "input_file,expected_file",
+        [
+            ("oss-fuzz-data1.yaml", "oss-fuzz-data-1.yaml-expected.json"),
+            ("oss-fuzz-data2.yaml", "oss-fuzz-data-2.yaml-expected.json"),
+            ("oss-fuzz-data3.yaml", "oss-fuzz-data-3.yaml-expected.json"),
+            ("oss-fuzz-data4.yaml", "oss-fuzz-data-4.yaml-expected.json"),
+        ],
+    )
+    def test_to_advisories1(self, input_file, expected_file):
+        with open(os.path.join(TEST_DATA, input_file)) as f:
+            mock_response = saneyaml.load(f)
+        expected_file = os.path.join(TEST_DATA, expected_file)
+        imported_data = parse_advisory_data_v2(
+            mock_response, "oss-fuzz", advisory_url="https://test.com", advisory_text=mock_response
+        )
+        result = imported_data.to_dict()
+        util_tests.check_results_against_json(result, expected_file)

vulnerabilities/tests/pipelines/v2_importers/test_pypa_importer_pipeline_v2.py

@@ -6,14 +6,16 @@
 # See https://github.com/aboutcode-org/vulnerablecode for support or download.
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
-
+import os
 from unittest.mock import MagicMock
 from unittest.mock import patch
 
 import pytest
 import saneyaml
 
 from vulnerabilities.importer import AdvisoryData
+from vulnerabilities.importers.osv import parse_advisory_data_v2
+from vulnerabilities.tests import util_tests
 
 
 @pytest.fixture
@@ -171,3 +173,26 @@ def test_advisories_count_empty(mock_vcs_response, mock_fetch_via_vcs):
     # Test that advisories_count returns 0 for an empty directory
     count = pipeline.advisories_count()
     assert count == 0
+
+
+BASE_DIR = os.path.dirname(os.path.abspath(__file__))
+TEST_DATA = os.path.join(BASE_DIR, "../../test_data/pypa/v2")
+
+
+class TestOSSFuzzImporter:
+    @pytest.mark.parametrize(
+        "input_file,expected_file",
+        [
+            ("pypa-1.yaml", "pypa-expected-1.json"),
+            ("pypa-2.yaml", "pypa-expected-2.json"),
+        ],
+    )
+    def test_to_advisories1(self, input_file, expected_file):
+        with open(os.path.join(TEST_DATA, input_file)) as f:
+            mock_response = saneyaml.load(f)
+        expected_file = os.path.join(TEST_DATA, expected_file)
+        imported_data = parse_advisory_data_v2(
+            mock_response, "pypi", advisory_url="https://test.com", advisory_text=mock_response
+        )
+        result = imported_data.to_dict()
+        util_tests.check_results_against_json(result, expected_file)

vulnerabilities/tests/test_data/oss_fuzz/v2/oss-fuzz-data-1.yaml-expected.json

@@ -0,0 +1,27 @@
+{
+  "advisory_id": "OSV-2021-933",
+  "aliases": [],
+  "summary": "Heap-buffer-overflow in print_mac\nOSS-Fuzz report: https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=35887\n\n```\nCrash type: Heap-buffer-overflow WRITE 4\nCrash state:\nprint_mac\nlog_packet\ndhcp_reply\n```",
+  "affected_packages": [],
+  "references_v2": [
+    {
+      "reference_id": "",
+      "reference_type": "",
+      "url": "https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=35887"
+    },
+    {
+      "reference_id": "96f6444958c29a670f4254722d787f328153605c",
+      "reference_type": "commit",
+      "url": "git://thekelleys.org.uk/dnsmasq.git"
+    },
+    {
+      "reference_id": "d242cbffa4f20c9f7472f79b3a9e47008b6fe77c",
+      "reference_type": "commit",
+      "url": "git://thekelleys.org.uk/dnsmasq.git"
+    }
+  ],
+  "severities": [],
+  "date_published": "2021-07-08T00:01:26.369555+00:00",
+  "weaknesses": [],
+  "url": "https://test.com"
+}

vulnerabilities/tests/test_data/oss_fuzz/v2/oss-fuzz-data-2.yaml-expected.json

@@ -0,0 +1,50 @@
+{
+  "advisory_id": "OSV-2022-145",
+  "aliases": [],
+  "summary": "Heap-buffer-overflow in print_mac\nOSS-Fuzz report: https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=44581\n\n```\nCrash type: Heap-buffer-overflow WRITE 4\nCrash state:\nprint_mac\nlog_packet\ndhcp_reply\n```",
+  "affected_packages": [
+    {
+      "package": {
+        "type": "generic",
+        "namespace": "",
+        "name": "dnsmasq",
+        "version": "",
+        "qualifiers": "",
+        "subpath": ""
+      },
+      "affected_version_range": null,
+      "fixed_version_range": null,
+      "affected_by_commits": [
+        {
+          "commit_hash": "e426c2d3bc182d790f83039b77a09d55230ca71f",
+          "vcs_url": "git://thekelleys.org.uk/dnsmasq.git",
+          "commit_author": null,
+          "commit_message": null,
+          "commit_diff": null,
+          "commit_date": null
+        }
+      ],
+      "fixed_by_commits": [
+        {
+          "commit_hash": "03345ecefeb0d82e3c3a4c28f27c3554f0611b39",
+          "vcs_url": "git://thekelleys.org.uk/dnsmasq.git",
+          "commit_author": null,
+          "commit_message": null,
+          "commit_diff": null,
+          "commit_date": null
+        }
+      ]
+    }
+  ],
+  "references_v2": [
+    {
+      "reference_id": "",
+      "reference_type": "",
+      "url": "https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=44581"
+    }
+  ],
+  "severities": [],
+  "date_published": "2022-02-13T00:01:27.883603+00:00",
+  "weaknesses": [],
+  "url": "https://test.com"
+}

vulnerabilities/tests/test_data/oss_fuzz/v2/oss-fuzz-data-3.yaml-expected.json

@@ -0,0 +1,50 @@
+{
+  "advisory_id": "OSV-2018-194",
+  "aliases": [],
+  "summary": "Heap-use-after-free in r_core_task_decref\nOSS-Fuzz report: https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=11359\n\n```\nCrash type: Heap-use-after-free READ 8\nCrash state:\nr_core_task_decref\nr_list_delete\nr_list_purge\n```",
+  "affected_packages": [
+    {
+      "package": {
+        "type": "github",
+        "namespace": "radare",
+        "name": "radare2",
+        "version": "",
+        "qualifiers": "",
+        "subpath": ""
+      },
+      "affected_version_range": null,
+      "fixed_version_range": null,
+      "affected_by_commits": [
+        {
+          "commit_hash": "77d80106e65ed4ff3ba5faf568b078648faed94f",
+          "vcs_url": "https://github.com/radare/radare2",
+          "commit_author": null,
+          "commit_message": null,
+          "commit_diff": null,
+          "commit_date": null
+        }
+      ],
+      "fixed_by_commits": [
+        {
+          "commit_hash": "5783cf42c40aaed9b9180ae7069c7a60ea86dc45",
+          "vcs_url": "https://github.com/radare/radare2",
+          "commit_author": null,
+          "commit_message": null,
+          "commit_diff": null,
+          "commit_date": null
+        }
+      ]
+    }
+  ],
+  "references_v2": [
+    {
+      "reference_id": "",
+      "reference_type": "",
+      "url": "https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=11359"
+    }
+  ],
+  "severities": [],
+  "date_published": "2021-01-13T00:01:20.948805+00:00",
+  "weaknesses": [],
+  "url": "https://test.com"
+}

vulnerabilities/tests/test_data/oss_fuzz/v2/oss-fuzz-data-4.yaml-expected.json

@@ -0,0 +1,50 @@
+{
+  "advisory_id": "OSV-2018-194",
+  "aliases": [],
+  "summary": "Heap-use-after-free in r_core_task_decref\nOSS-Fuzz report: https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=11359\n\n```\nCrash type: Heap-use-after-free READ 8\nCrash state:\nr_core_task_decref\nr_list_delete\nr_list_purge\n```",
+  "affected_packages": [
+    {
+      "package": {
+        "type": "github",
+        "namespace": "radare",
+        "name": "radare2",
+        "version": "",
+        "qualifiers": "",
+        "subpath": ""
+      },
+      "affected_version_range": null,
+      "fixed_version_range": null,
+      "affected_by_commits": [
+        {
+          "commit_hash": "77d80106e65ed4ff3ba5faf568b078648faed94f",
+          "vcs_url": "https://github.com/radare/radare2",
+          "commit_author": null,
+          "commit_message": null,
+          "commit_diff": null,
+          "commit_date": null
+        }
+      ],
+      "fixed_by_commits": [
+        {
+          "commit_hash": "5783cf42c40aaed9b9180ae7069c7a60ea86dc45",
+          "vcs_url": "https://github.com/radare/radare2",
+          "commit_author": null,
+          "commit_message": null,
+          "commit_diff": null,
+          "commit_date": null
+        }
+      ]
+    }
+  ],
+  "references_v2": [
+    {
+      "reference_id": "",
+      "reference_type": "",
+      "url": "https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=11359"
+    }
+  ],
+  "severities": [],
+  "date_published": "2021-01-13T00:01:20.948805+00:00",
+  "weaknesses": [],
+  "url": "https://test.com"
+}