diff --git a/vulnerabilities/importers/__init__.py b/vulnerabilities/importers/__init__.py index f508790c7..65c8a79ef 100644 --- a/vulnerabilities/importers/__init__.py +++ b/vulnerabilities/importers/__init__.py @@ -48,6 +48,7 @@ from vulnerabilities.pipelines.v2_importers import ( elixir_security_importer as elixir_security_importer_v2, ) +from vulnerabilities.pipelines.v2_importers import epss_importer_v2 from vulnerabilities.pipelines.v2_importers import github_osv_importer as github_osv_importer_v2 from vulnerabilities.pipelines.v2_importers import gitlab_importer as gitlab_importer_v2 from vulnerabilities.pipelines.v2_importers import istio_importer as istio_importer_v2 @@ -83,6 +84,7 @@ github_osv_importer_v2.GithubOSVImporterPipeline, redhat_importer_v2.RedHatImporterPipeline, aosp_importer_v2.AospImporterPipeline, + epss_importer_v2.EPSSImporterPipeline, nvd_importer.NVDImporterPipeline, github_importer.GitHubAPIImporterPipeline, gitlab_importer.GitLabImporterPipeline, diff --git a/vulnerabilities/pipelines/v2_importers/epss_importer_v2.py b/vulnerabilities/pipelines/v2_importers/epss_importer_v2.py new file mode 100644 index 000000000..1b3dbaeff --- /dev/null +++ b/vulnerabilities/pipelines/v2_importers/epss_importer_v2.py @@ -0,0 +1,84 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# +import csv +import gzip +import logging +import urllib.request +from datetime import datetime +from typing import Iterable + +from vulnerabilities import severity_systems +from vulnerabilities.importer import AdvisoryData +from vulnerabilities.importer import ReferenceV2 +from vulnerabilities.importer import VulnerabilitySeverity +from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2 + +logger = logging.getLogger(__name__) + + +class EPSSImporterPipeline(VulnerableCodeBaseImporterPipelineV2): + """Exploit Prediction Scoring System (EPSS) Importer""" + + advisory_url = "https://epss.cyentia.com/epss_scores-current.csv.gz" + pipeline_id = "epss_importer_v2" + spdx_license_expression = "unknown" + importer_name = "EPSS Importer" + + def advisories_count(self): + return len(self.lines) + + @classmethod + def steps(cls): + return ( + cls.fetch_db, + cls.collect_and_store_advisories, + ) + + def fetch_db(self): + logger.info(f"Fetching EPSS database from {self.advisory_url}") + response = urllib.request.urlopen(self.advisory_url) + with gzip.open(response, "rb") as f: + self.lines = [l.decode("utf-8") for l in f.readlines()] + + def collect_advisories(self) -> Iterable[AdvisoryData]: + if not self.lines: + logger.error("No EPSS data loaded") + raise ValueError("EPSS data is empty") + + epss_reader = csv.reader(self.lines) + model_version, score_date = next( + epss_reader + ) # score_date='score_date:2024-05-19T00:00:00+0000' + published_at = datetime.strptime(score_date[11::], "%Y-%m-%dT%H:%M:%S%z") + + next(epss_reader) # skip the header row + for epss_row in epss_reader: + cve, score, percentile = epss_row + + if not cve or not score or not percentile: + logger.error(f"Invalid epss row: {epss_row}") + continue + + severity = VulnerabilitySeverity( + system=severity_systems.EPSS, + value=score, + scoring_elements=percentile, + published_at=published_at, + ) + + references = ReferenceV2( + url=f"https://api.first.org/data/v1/epss?cve={cve}", + ) + + yield AdvisoryData( + advisory_id=cve, + severities=[severity], + references_v2=[references], + url=self.advisory_url, + ) diff --git a/vulnerabilities/tests/pipelines/v2_importers/test_epss_importer_v2.py b/vulnerabilities/tests/pipelines/v2_importers/test_epss_importer_v2.py new file mode 100644 index 000000000..bd463edf1 --- /dev/null +++ b/vulnerabilities/tests/pipelines/v2_importers/test_epss_importer_v2.py @@ -0,0 +1,32 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +from pathlib import Path + +import pytest + +from vulnerabilities.pipelines.v2_importers.epss_importer_v2 import EPSSImporterPipeline +from vulnerabilities.tests import util_tests + +TEST_DATA = Path(__file__).parent.parent.parent / "test_data" / "epss" + +TEST_CVE_FILES = (TEST_DATA / "epss_scores-2025-x-x.csv",) + + +@pytest.mark.django_db +@pytest.mark.parametrize("csv_file", TEST_CVE_FILES) +def test_epss_advisories_per_file(csv_file): + pipeline = EPSSImporterPipeline() + + with open(csv_file, "r") as f: + pipeline.lines = f.readlines() + + result = [adv.to_dict() for adv in pipeline.collect_advisories()] + expected_file = Path(TEST_DATA / "epss-expected.json") + util_tests.check_results_against_json(result, expected_file) diff --git a/vulnerabilities/tests/test_data/epss/epss-expected.json b/vulnerabilities/tests/test_data/epss/epss-expected.json new file mode 100644 index 000000000..1cc008c01 --- /dev/null +++ b/vulnerabilities/tests/test_data/epss/epss-expected.json @@ -0,0 +1,677 @@ +[ + { + "advisory_id": "CVE-1999-0001", + "aliases": [], + "summary": "", + "affected_packages": [], + "references_v2": [ + { + "reference_id": "", + "reference_type": "", + "url": "https://api.first.org/data/v1/epss?cve=CVE-1999-0001" + } + ], + "patches": [], + "severities": [ + { + "system": "epss", + "value": "0.01151", + "scoring_elements": "0.7802", + "published_at": "2025-12-26T12:55:00+00:00" + } + ], + "date_published": null, + "weaknesses": [], + "url": "https://epss.cyentia.com/epss_scores-current.csv.gz" + }, + { + "advisory_id": "CVE-1999-0002", + "aliases": [], + "summary": "", + "affected_packages": [], + "references_v2": [ + { + "reference_id": "", + "reference_type": "", + "url": "https://api.first.org/data/v1/epss?cve=CVE-1999-0002" + } + ], + "patches": [], + "severities": [ + { + "system": "epss", + "value": "0.10546", + "scoring_elements": "0.93015", + "published_at": "2025-12-26T12:55:00+00:00" + } + ], + "date_published": null, + "weaknesses": [], + "url": "https://epss.cyentia.com/epss_scores-current.csv.gz" + }, + { + "advisory_id": "CVE-1999-0003", + "aliases": [], + "summary": "", + "affected_packages": [], + "references_v2": [ + { + "reference_id": "", + "reference_type": "", + "url": "https://api.first.org/data/v1/epss?cve=CVE-1999-0003" + } + ], + "patches": [], + "severities": [ + { + "system": "epss", + "value": "0.89352", + "scoring_elements": "0.99521", + "published_at": "2025-12-26T12:55:00+00:00" + } + ], + "date_published": null, + "weaknesses": [], + "url": "https://epss.cyentia.com/epss_scores-current.csv.gz" + }, + { + "advisory_id": "CVE-1999-0004", + "aliases": [], + "summary": "", + "affected_packages": [], + "references_v2": [ + { + "reference_id": "", + "reference_type": "", + "url": "https://api.first.org/data/v1/epss?cve=CVE-1999-0004" + } + ], + "patches": [], + "severities": [ + { + "system": "epss", + "value": "0.03037", + "scoring_elements": "0.86249", + "published_at": "2025-12-26T12:55:00+00:00" + } + ], + "date_published": null, + "weaknesses": [], + "url": "https://epss.cyentia.com/epss_scores-current.csv.gz" + }, + { + "advisory_id": "CVE-2025-9970", + "aliases": [], + "summary": "", + "affected_packages": [], + "references_v2": [ + { + "reference_id": "", + "reference_type": "", + "url": "https://api.first.org/data/v1/epss?cve=CVE-2025-9970" + } + ], + "patches": [], + "severities": [ + { + "system": "epss", + "value": "7e-05", + "scoring_elements": "0.00428", + "published_at": "2025-12-26T12:55:00+00:00" + } + ], + "date_published": null, + "weaknesses": [], + "url": "https://epss.cyentia.com/epss_scores-current.csv.gz" + }, + { + "advisory_id": "CVE-2025-9971", + "aliases": [], + "summary": "", + "affected_packages": [], + "references_v2": [ + { + "reference_id": "", + "reference_type": "", + "url": "https://api.first.org/data/v1/epss?cve=CVE-2025-9971" + } + ], + "patches": [], + "severities": [ + { + "system": "epss", + "value": "0.0024", + "scoring_elements": "0.47036", + "published_at": "2025-12-26T12:55:00+00:00" + } + ], + "date_published": null, + "weaknesses": [], + "url": "https://epss.cyentia.com/epss_scores-current.csv.gz" + }, + { + "advisory_id": "CVE-2025-9972", + "aliases": [], + "summary": "", + "affected_packages": [], + "references_v2": [ + { + "reference_id": "", + "reference_type": "", + "url": "https://api.first.org/data/v1/epss?cve=CVE-2025-9972" + } + ], + "patches": [], + "severities": [ + { + "system": "epss", + "value": "0.00543", + "scoring_elements": "0.67059", + "published_at": "2025-12-26T12:55:00+00:00" + } + ], + "date_published": null, + "weaknesses": [], + "url": "https://epss.cyentia.com/epss_scores-current.csv.gz" + }, + { + "advisory_id": "CVE-2025-9975", + "aliases": [], + "summary": "", + "affected_packages": [], + "references_v2": [ + { + "reference_id": "", + "reference_type": "", + "url": "https://api.first.org/data/v1/epss?cve=CVE-2025-9975" + } + ], + "patches": [], + "severities": [ + { + "system": "epss", + "value": "0.00039", + "scoring_elements": "0.11657", + "published_at": "2025-12-26T12:55:00+00:00" + } + ], + "date_published": null, + "weaknesses": [], + "url": "https://epss.cyentia.com/epss_scores-current.csv.gz" + }, + { + "advisory_id": "CVE-2025-9976", + "aliases": [], + "summary": "", + "affected_packages": [], + "references_v2": [ + { + "reference_id": "", + "reference_type": "", + "url": "https://api.first.org/data/v1/epss?cve=CVE-2025-9976" + } + ], + "patches": [], + "severities": [ + { + "system": "epss", + "value": "0.00085", + "scoring_elements": "0.25184", + "published_at": "2025-12-26T12:55:00+00:00" + } + ], + "date_published": null, + "weaknesses": [], + "url": "https://epss.cyentia.com/epss_scores-current.csv.gz" + }, + { + "advisory_id": "CVE-2025-9977", + "aliases": [], + "summary": "", + "affected_packages": [], + "references_v2": [ + { + "reference_id": "", + "reference_type": "", + "url": "https://api.first.org/data/v1/epss?cve=CVE-2025-9977" + } + ], + "patches": [], + "severities": [ + { + "system": "epss", + "value": "0.06195", + "scoring_elements": "0.90539", + "published_at": "2025-12-26T12:55:00+00:00" + } + ], + "date_published": null, + "weaknesses": [], + "url": "https://epss.cyentia.com/epss_scores-current.csv.gz" + }, + { + "advisory_id": "CVE-2025-9978", + "aliases": [], + "summary": "", + "affected_packages": [], + "references_v2": [ + { + "reference_id": "", + "reference_type": "", + "url": "https://api.first.org/data/v1/epss?cve=CVE-2025-9978" + } + ], + "patches": [], + "severities": [ + { + "system": "epss", + "value": "0.00045", + "scoring_elements": "0.13792", + "published_at": "2025-12-26T12:55:00+00:00" + } + ], + "date_published": null, + "weaknesses": [], + "url": "https://epss.cyentia.com/epss_scores-current.csv.gz" + }, + { + "advisory_id": "CVE-2025-9979", + "aliases": [], + "summary": "", + "affected_packages": [], + "references_v2": [ + { + "reference_id": "", + "reference_type": "", + "url": "https://api.first.org/data/v1/epss?cve=CVE-2025-9979" + } + ], + "patches": [], + "severities": [ + { + "system": "epss", + "value": "0.00037", + "scoring_elements": "0.10925", + "published_at": "2025-12-26T12:55:00+00:00" + } + ], + "date_published": null, + "weaknesses": [], + "url": "https://epss.cyentia.com/epss_scores-current.csv.gz" + }, + { + "advisory_id": "CVE-2025-9980", + "aliases": [], + "summary": "", + "affected_packages": [], + "references_v2": [ + { + "reference_id": "", + "reference_type": "", + "url": "https://api.first.org/data/v1/epss?cve=CVE-2025-9980" + } + ], + "patches": [], + "severities": [ + { + "system": "epss", + "value": "0.00031", + "scoring_elements": "0.08656", + "published_at": "2025-12-26T12:55:00+00:00" + } + ], + "date_published": null, + "weaknesses": [], + "url": "https://epss.cyentia.com/epss_scores-current.csv.gz" + }, + { + "advisory_id": "CVE-2025-9981", + "aliases": [], + "summary": "", + "affected_packages": [], + "references_v2": [ + { + "reference_id": "", + "reference_type": "", + "url": "https://api.first.org/data/v1/epss?cve=CVE-2025-9981" + } + ], + "patches": [], + "severities": [ + { + "system": "epss", + "value": "0.00031", + "scoring_elements": "0.08656", + "published_at": "2025-12-26T12:55:00+00:00" + } + ], + "date_published": null, + "weaknesses": [], + "url": "https://epss.cyentia.com/epss_scores-current.csv.gz" + }, + { + "advisory_id": "CVE-2025-9982", + "aliases": [], + "summary": "", + "affected_packages": [], + "references_v2": [ + { + "reference_id": "", + "reference_type": "", + "url": "https://api.first.org/data/v1/epss?cve=CVE-2025-9982" + } + ], + "patches": [], + "severities": [ + { + "system": "epss", + "value": "0.00053", + "scoring_elements": "0.16926", + "published_at": "2025-12-26T12:55:00+00:00" + } + ], + "date_published": null, + "weaknesses": [], + "url": "https://epss.cyentia.com/epss_scores-current.csv.gz" + }, + { + "advisory_id": "CVE-2025-9983", + "aliases": [], + "summary": "", + "affected_packages": [], + "references_v2": [ + { + "reference_id": "", + "reference_type": "", + "url": "https://api.first.org/data/v1/epss?cve=CVE-2025-9983" + } + ], + "patches": [], + "severities": [ + { + "system": "epss", + "value": "0.00023", + "scoring_elements": "0.05543", + "published_at": "2025-12-26T12:55:00+00:00" + } + ], + "date_published": null, + "weaknesses": [], + "url": "https://epss.cyentia.com/epss_scores-current.csv.gz" + }, + { + "advisory_id": "CVE-2025-9984", + "aliases": [], + "summary": "", + "affected_packages": [], + "references_v2": [ + { + "reference_id": "", + "reference_type": "", + "url": "https://api.first.org/data/v1/epss?cve=CVE-2025-9984" + } + ], + "patches": [], + "severities": [ + { + "system": "epss", + "value": "0.00055", + "scoring_elements": "0.17352", + "published_at": "2025-12-26T12:55:00+00:00" + } + ], + "date_published": null, + "weaknesses": [], + "url": "https://epss.cyentia.com/epss_scores-current.csv.gz" + }, + { + "advisory_id": "CVE-2025-9985", + "aliases": [], + "summary": "", + "affected_packages": [], + "references_v2": [ + { + "reference_id": "", + "reference_type": "", + "url": "https://api.first.org/data/v1/epss?cve=CVE-2025-9985" + } + ], + "patches": [], + "severities": [ + { + "system": "epss", + "value": "0.01936", + "scoring_elements": "0.82968", + "published_at": "2025-12-26T12:55:00+00:00" + } + ], + "date_published": null, + "weaknesses": [], + "url": "https://epss.cyentia.com/epss_scores-current.csv.gz" + }, + { + "advisory_id": "CVE-2025-9990", + "aliases": [], + "summary": "", + "affected_packages": [], + "references_v2": [ + { + "reference_id": "", + "reference_type": "", + "url": "https://api.first.org/data/v1/epss?cve=CVE-2025-9990" + } + ], + "patches": [], + "severities": [ + { + "system": "epss", + "value": "0.00343", + "scoring_elements": "0.56418", + "published_at": "2025-12-26T12:55:00+00:00" + } + ], + "date_published": null, + "weaknesses": [], + "url": "https://epss.cyentia.com/epss_scores-current.csv.gz" + }, + { + "advisory_id": "CVE-2025-9991", + "aliases": [], + "summary": "", + "affected_packages": [], + "references_v2": [ + { + "reference_id": "", + "reference_type": "", + "url": "https://api.first.org/data/v1/epss?cve=CVE-2025-9991" + } + ], + "patches": [], + "severities": [ + { + "system": "epss", + "value": "0.00343", + "scoring_elements": "0.56418", + "published_at": "2025-12-26T12:55:00+00:00" + } + ], + "date_published": null, + "weaknesses": [], + "url": "https://epss.cyentia.com/epss_scores-current.csv.gz" + }, + { + "advisory_id": "CVE-2025-9992", + "aliases": [], + "summary": "", + "affected_packages": [], + "references_v2": [ + { + "reference_id": "", + "reference_type": "", + "url": "https://api.first.org/data/v1/epss?cve=CVE-2025-9992" + } + ], + "patches": [], + "severities": [ + { + "system": "epss", + "value": "0.00043", + "scoring_elements": "0.13241", + "published_at": "2025-12-26T12:55:00+00:00" + } + ], + "date_published": null, + "weaknesses": [], + "url": "https://epss.cyentia.com/epss_scores-current.csv.gz" + }, + { + "advisory_id": "CVE-2025-9993", + "aliases": [], + "summary": "", + "affected_packages": [], + "references_v2": [ + { + "reference_id": "", + "reference_type": "", + "url": "https://api.first.org/data/v1/epss?cve=CVE-2025-9993" + } + ], + "patches": [], + "severities": [ + { + "system": "epss", + "value": "0.00129", + "scoring_elements": "0.33233", + "published_at": "2025-12-26T12:55:00+00:00" + } + ], + "date_published": null, + "weaknesses": [], + "url": "https://epss.cyentia.com/epss_scores-current.csv.gz" + }, + { + "advisory_id": "CVE-2025-9994", + "aliases": [], + "summary": "", + "affected_packages": [], + "references_v2": [ + { + "reference_id": "", + "reference_type": "", + "url": "https://api.first.org/data/v1/epss?cve=CVE-2025-9994" + } + ], + "patches": [], + "severities": [ + { + "system": "epss", + "value": "0.00114", + "scoring_elements": "0.30855", + "published_at": "2025-12-26T12:55:00+00:00" + } + ], + "date_published": null, + "weaknesses": [], + "url": "https://epss.cyentia.com/epss_scores-current.csv.gz" + }, + { + "advisory_id": "CVE-2025-9996", + "aliases": [], + "summary": "", + "affected_packages": [], + "references_v2": [ + { + "reference_id": "", + "reference_type": "", + "url": "https://api.first.org/data/v1/epss?cve=CVE-2025-9996" + } + ], + "patches": [], + "severities": [ + { + "system": "epss", + "value": "0.00241", + "scoring_elements": "0.47334", + "published_at": "2025-12-26T12:55:00+00:00" + } + ], + "date_published": null, + "weaknesses": [], + "url": "https://epss.cyentia.com/epss_scores-current.csv.gz" + }, + { + "advisory_id": "CVE-2025-9997", + "aliases": [], + "summary": "", + "affected_packages": [], + "references_v2": [ + { + "reference_id": "", + "reference_type": "", + "url": "https://api.first.org/data/v1/epss?cve=CVE-2025-9997" + } + ], + "patches": [], + "severities": [ + { + "system": "epss", + "value": "0.00245", + "scoring_elements": "0.47699", + "published_at": "2025-12-26T12:55:00+00:00" + } + ], + "date_published": null, + "weaknesses": [], + "url": "https://epss.cyentia.com/epss_scores-current.csv.gz" + }, + { + "advisory_id": "CVE-2025-9998", + "aliases": [], + "summary": "", + "affected_packages": [], + "references_v2": [ + { + "reference_id": "", + "reference_type": "", + "url": "https://api.first.org/data/v1/epss?cve=CVE-2025-9998" + } + ], + "patches": [], + "severities": [ + { + "system": "epss", + "value": "0.00022", + "scoring_elements": "0.05357", + "published_at": "2025-12-26T12:55:00+00:00" + } + ], + "date_published": null, + "weaknesses": [], + "url": "https://epss.cyentia.com/epss_scores-current.csv.gz" + }, + { + "advisory_id": "CVE-2025-9999", + "aliases": [], + "summary": "", + "affected_packages": [], + "references_v2": [ + { + "reference_id": "", + "reference_type": "", + "url": "https://api.first.org/data/v1/epss?cve=CVE-2025-9999" + } + ], + "patches": [], + "severities": [ + { + "system": "epss", + "value": "0.00044", + "scoring_elements": "0.13444", + "published_at": "2025-12-26T12:55:00+00:00" + } + ], + "date_published": null, + "weaknesses": [], + "url": "https://epss.cyentia.com/epss_scores-current.csv.gz" + } +] \ No newline at end of file diff --git a/vulnerabilities/tests/test_data/epss/epss_scores-2025-x-x.csv b/vulnerabilities/tests/test_data/epss/epss_scores-2025-x-x.csv new file mode 100644 index 000000000..fd2a04010 --- /dev/null +++ b/vulnerabilities/tests/test_data/epss/epss_scores-2025-x-x.csv @@ -0,0 +1,29 @@ +#model_version:v2025.03.14,score_date:2025-12-26T12:55:00Z +cve,epss,percentile +CVE-1999-0001,0.01151,0.7802 +CVE-1999-0002,0.10546,0.93015 +CVE-1999-0003,0.89352,0.99521 +CVE-1999-0004,0.03037,0.86249 +CVE-2025-9970,7e-05,0.00428 +CVE-2025-9971,0.0024,0.47036 +CVE-2025-9972,0.00543,0.67059 +CVE-2025-9975,0.00039,0.11657 +CVE-2025-9976,0.00085,0.25184 +CVE-2025-9977,0.06195,0.90539 +CVE-2025-9978,0.00045,0.13792 +CVE-2025-9979,0.00037,0.10925 +CVE-2025-9980,0.00031,0.08656 +CVE-2025-9981,0.00031,0.08656 +CVE-2025-9982,0.00053,0.16926 +CVE-2025-9983,0.00023,0.05543 +CVE-2025-9984,0.00055,0.17352 +CVE-2025-9985,0.01936,0.82968 +CVE-2025-9990,0.00343,0.56418 +CVE-2025-9991,0.00343,0.56418 +CVE-2025-9992,0.00043,0.13241 +CVE-2025-9993,0.00129,0.33233 +CVE-2025-9994,0.00114,0.30855 +CVE-2025-9996,0.00241,0.47334 +CVE-2025-9997,0.00245,0.47699 +CVE-2025-9998,0.00022,0.05357 +CVE-2025-9999,0.00044,0.13444 \ No newline at end of file