diff --git a/docs/source/conf.py b/docs/source/conf.py index a6247eafa..fd078c97d 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -42,6 +42,7 @@ "http://ftp.suse.com/pub/projects/security/yaml/", r"https://nixos\.wiki/", # NixOS wiki blocks CI bots with 403 "https://usn.ubuntu.com/usn-db/database-all.json.bz2", + "https://public.vulnerablecode.io/vulnerabilities/search/", ] # Add any Sphinx extension module names here, as strings. They can be diff --git a/vulnerabilities/importer.py b/vulnerabilities/importer.py index abd6f90c5..ab5a6f9a2 100644 --- a/vulnerabilities/importer.py +++ b/vulnerabilities/importer.py @@ -19,7 +19,6 @@ from typing import Optional from typing import Set from typing import Tuple -from typing import Union import pytz from dateutil import parser as dateparser @@ -40,7 +39,6 @@ from vulnerabilities.utils import get_reference_id from vulnerabilities.utils import is_commit from vulnerabilities.utils import is_cve -from vulnerabilities.utils import nearest_patched_package from vulnerabilities.utils import purl_to_dict from vulnerabilities.utils import update_purl_version @@ -65,7 +63,7 @@ def __post_init__(self): raise TypeError(f"system must be a ScoringSystem, got {type(self.system)!r}") if not isinstance(self.value, str): - self.value = str(self.value) + self.value = str(self.value) if self.value else "" def to_dict(self): data = { diff --git a/vulnerabilities/migrations/0139_cleanup_none_string_in_severity.py b/vulnerabilities/migrations/0139_cleanup_none_string_in_severity.py new file mode 100644 index 000000000..124e907a4 --- /dev/null +++ b/vulnerabilities/migrations/0139_cleanup_none_string_in_severity.py @@ -0,0 +1,132 @@ +# Generated by Django 5.2.11 on 2026-07-02 09:50 + +from django.db import migrations + +from vulnerabilities.importer import AdvisoryDataV2 +from vulnerabilities.importer import AffectedPackageV2 +from vulnerabilities.importer import PatchData +from vulnerabilities.importer import ReferenceV2 +from vulnerabilities.importer import VulnerabilitySeverity +from vulnerabilities.utils import compute_content_id_v2 +from vulnerabilities.utils import normalize_list +from vulnerabilities.utils import purl_to_dict + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0138_fix_malformed_cvss_vector"), + ] + + def cleanup_none_severity_string(apps, schema_editor): + AdvisorySeverity = apps.get_model("vulnerabilities", "AdvisorySeverity") + AdvisoryV2 = apps.get_model("vulnerabilities", "AdvisoryV2") + batch = [] + batch_size = 5000 + + advisory_ids = list( + AdvisoryV2.objects.filter(severities__value="None") + .distinct() + .values_list("id", flat=True) + ) + + AdvisorySeverity.objects.filter(value="None").update(value="") + + for advisory in AdvisoryV2.objects.filter(id__in=advisory_ids).iterator(chunk_size=2000): + advisory.unique_content_id = compute_content_id_v2(to_advisory_data(advisory)) + batch.append(advisory) + + if len(batch) >= batch_size: + AdvisoryV2.objects.bulk_update(batch, ["unique_content_id"]) + batch.clear() + + if batch: + AdvisoryV2.objects.bulk_update(batch, ["unique_content_id"]) + + operations = [ + migrations.RunPython( + cleanup_none_severity_string, + reverse_code=migrations.RunPython.noop, + ), + ] + + +def commit_patch_to_dict(patch): + return { + "vcs_url": patch.vcs_url, + "commit_hash": patch.commit_hash, + "patch_text": patch.patch_text, + "patch_checksum": patch.patch_checksum, + } + + +def to_affected_package_data(impact): + """Return `AffectedPackageV2` data from the impact.""" + return AffectedPackageV2.from_dict( + { + "package": purl_to_dict(impact.base_purl), + "affected_version_range": impact.affecting_vers, + "fixed_version_range": impact.fixed_vers, + "introduced_by_commit_patches": [ + commit_patch_to_dict(commit) + for commit in impact.introduced_by_package_commit_patches.all() + ], + "fixed_by_commit_patches": [ + commit_patch_to_dict(commit) + for commit in impact.fixed_by_package_commit_patches.all() + ], + } + ) + + +def to_patch_data(patch): + """Return `PatchData` from the Patch.""" + + return PatchData.from_dict( + { + "patch_url": patch.patch_url, + "patch_text": patch.patch_text, + "patch_checksum": patch.patch_checksum, + } + ) + + +def to_reference_v2_data(ref): + return ReferenceV2.from_dict( + { + "reference_id": ref.reference_id, + "reference_type": ref.reference_type, + "url": ref.url, + } + ) + + +def to_vulnerability_severity_data(severity): + return VulnerabilitySeverity.from_dict( + { + "system": severity.scoring_system, + "value": severity.value, + "scoring_elements": severity.scoring_elements, + "published_at": severity.published_at, + "url": severity.url, + } + ) + + +def to_advisory_data(advisory): + return AdvisoryDataV2( + advisory_id=advisory.advisory_id, + aliases=normalize_list([item.alias for item in advisory.aliases.all()]), + summary=advisory.summary, + affected_packages=normalize_list( + [to_affected_package_data(impacted) for impacted in advisory.impacted_packages.all()] + ), + references=normalize_list([to_reference_v2_data(ref) for ref in advisory.references.all()]), + patches=normalize_list([to_patch_data(patch) for patch in advisory.patches.all()]), + date_published=advisory.date_published, + weaknesses=normalize_list([weak.cwe_id for weak in advisory.weaknesses.all()]), + severities=normalize_list( + [to_vulnerability_severity_data(sev) for sev in advisory.severities.all()] + ), + url=advisory.url, + ) diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index 38a8ef8d5..46670dadb 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -9,8 +9,6 @@ import csv import datetime -import hashlib -import json import logging import uuid import xml.etree.ElementTree as ET diff --git a/vulnerabilities/tests/pipelines/v2_improvers/test_compute_advisory_todo_v2.py b/vulnerabilities/tests/pipelines/v2_improvers/test_compute_advisory_todo_v2.py index 255c87bfd..7408d8aed 100644 --- a/vulnerabilities/tests/pipelines/v2_improvers/test_compute_advisory_todo_v2.py +++ b/vulnerabilities/tests/pipelines/v2_improvers/test_compute_advisory_todo_v2.py @@ -26,6 +26,7 @@ from vulnerabilities.pipelines.v2_improvers.compute_advisory_todo import ComputeToDo from vulnerabilities.pipes.advisory import insert_advisory_v2 from vulnerabilities.tests.pipelines import TestLogger +from vulnerabilities.utils import canonical_value class TestComputeToDo(TestCase): @@ -690,7 +691,10 @@ def test_todo_conflict_details_partial_curation_unpaired_purl_and_conflicting_af result_partial_curation = issue_details["partial_curation_advisory"] self.assertEqual(1, AdvisoryToDoV2.objects.count()) self.assertEqual("CONFLICTING_AFFECTED_PACKAGES", todo.issue_type) - self.assertDictEqual(expected_partial_curation_advisory, result_partial_curation) + self.assertCountEqual( + expected_partial_curation_advisory["affected_packages"], + result_partial_curation["affected_packages"], + ) def test_todo_conflicting_severity(self): insert_advisory_v2( diff --git a/vulnerabilities/tests/test_data_migrations.py b/vulnerabilities/tests/test_data_migrations.py index c75468bab..6d22fdfbc 100644 --- a/vulnerabilities/tests/test_data_migrations.py +++ b/vulnerabilities/tests/test_data_migrations.py @@ -23,6 +23,10 @@ from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import AffectedPackage from vulnerabilities.importer import Reference +from vulnerabilities.models import AdvisorySeverity +from vulnerabilities.models import AdvisoryV2 +from vulnerabilities.models import ImpactedPackage +from vulnerabilities.utils import compute_content_id_v2 from vulnerabilities.utils import purl_to_dict @@ -1378,3 +1382,50 @@ def test_scheme_migration_correctness(self): self.assertEqual(self.impact2.affecting_vers, "vers:apk/3.4.5") self.assertEqual(self.impact2.fixed_vers, None) + + +class TestCleanAdvisorySeverityMigration(TestMigrations): + app_name = "vulnerabilities" + migrate_from = "0138_fix_malformed_cvss_vector" + migrate_to = "0139_cleanup_none_string_in_severity" + + def setUpBeforeMigration(self, apps): + # AdvisoryV2 = apps.get_model("vulnerabilities", "AdvisoryV2") + # ImpactedPackage = apps.get_model("vulnerabilities", "ImpactedPackage") + # AdvisorySeverity = apps.get_model("vulnerabilities", "AdvisorySeverity") + + self.advisory1 = AdvisoryV2.objects.create( + unique_content_id="b001d1a8952bc056d0161f1dd45dd8f90b25f62c56a887ea21d09fafd78a0f61", + url="https://old.example.com", + summary="Old advisory", + advisory_id="test_adv1", + avid="test_pipeline/test_adv", + datasource_id="test_pipeline", + pipeline_id="test_pipeline_v2", + ) + + ImpactedPackage.objects.create( + advisory=self.advisory1, + base_purl="pkg:npm/foobar0", + affecting_vers="vers:npm/4.3.2", + fixed_vers="vers:npm/5.0.0", + ) + + self.severity = AdvisorySeverity.objects.create( + scoring_system=severity_systems.CVSSV4, + scoring_elements="CVSS:4.0/AV:N/AC:L/AT:P/PR:H/UI:P/VC:N/VI:N/VA:N", + value="None", + ) + + self.advisory1.severities.add(self.severity) + + def test_severity_value_cleaned(self): + self.severity.refresh_from_db() + self.assertEqual(self.severity.value, "") + + def test_advisory_content_id_recomputed(self): + self.advisory1.refresh_from_db() + self.assertEqual( + self.advisory1.unique_content_id, + "a15d4651cb05e3513c12263a11e34bd9103f68833cac8f7ffdbbd71b9cb4cf16", + ) diff --git a/vulnerabilities/tests/test_importer.py b/vulnerabilities/tests/test_importer.py index cb7c005fa..952625be8 100644 --- a/vulnerabilities/tests/test_importer.py +++ b/vulnerabilities/tests/test_importer.py @@ -13,6 +13,7 @@ from vulnerabilities.importer import PackageCommitPatchData from vulnerabilities.importer import PatchData from vulnerabilities.importer import ReferenceV2 +from vulnerabilities.importer import VulnerabilitySeverity from vulnerabilities.pipes.advisory import classify_patch_source @@ -201,3 +202,15 @@ def test_classify_patch_source_integration(url, commit_hash, patch_text, results assert actual_data_obj.reference_id == expected_data_obj.reference_id assert actual_data_obj.reference_type == expected_data_obj.reference_type assert actual_data_obj.url == expected_data_obj.url + + +def test_vulnerability_severity_value_string_conversion(): + severity = VulnerabilitySeverity.from_dict( + { + "system": "cvssv3", + "value": None, + "scoring_elements": "CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:N/A:N", + } + ) + + assert severity.value == "" diff --git a/vulnerablecode/static/css/package_curation.css b/vulnerablecode/static/css/package_curation.css index 00af968bf..bbdb5aed7 100644 --- a/vulnerablecode/static/css/package_curation.css +++ b/vulnerablecode/static/css/package_curation.css @@ -1,126 +1,126 @@ - .state-affected { - background-color: #f14668 !important; - color: #fff !important; - } - - .state-fixed { - background-color: #48c78e !important; - color: #fff !important; - } - - .state-empty { - background-color: #ffdd57 !important; - color: #000 !important; - box-shadow: inset 0 0 0 1px #dbdbdb; - } - - .table { - table-layout: fixed !important; - width: 100% !important; - } - - .table th { - border-width: 0 1px 1px 1px !important; - border-color: #dbdbdb !important; - background-color: #fff !important; - } - - #table-header th { - height: 140px; - vertical-align: top !important; - padding: 0.75rem 0.5rem !important; - } - - #table-header th>div { - display: inline-flex !important; - flex-direction: column !important; - justify-content: space-between !important; - height: 100% !important; - width: 100%; - } - - #table-header th:first-child { - width: 150px !important; - } - - #table-header th:nth-child(2) { - width: 150px !important; - } - - #table-header th:nth-child(n+3) { - width: 120px !important; - } - - .curation-cell { - cursor: pointer; - text-align: center !important; - user-select: none; - transition: background-color 0.1s ease; - border: 1px solid #dbdbdb !important; - } - - .curation-cell:hover { - filter: brightness(0.95); - box-shadow: inset 0 0 0 1px #3273dc; - } - - .advisory-cell { - cursor: not-allowed; - word-break: break-all; - } - - .folded-row-marker, - .range-row-marker { - cursor: pointer; - text-align: center; - color: #4a4a4a; - font-weight: 600; - background-color: #f5f5f5; - } - - .folded-row-marker.is-expanded .icon { - transform: rotate(180deg); - } - - .range-data-cell { - background-color: #fafafa; - padding: 6px !important; - vertical-align: top !important; - border: 1px solid #dbdbdb !important; - - white-space: normal !important; - word-break: break-all !important; - overflow-wrap: break-word !important; - } - - #summaries-container p strong { - display: block; - word-break: break-all; - overflow-wrap: break-word; - line-height: 1.3; - margin-bottom: 4px; - } - - .summary-text { - line-height: 1.4; - white-space: pre-wrap; - word-break: break-word; - } - - div.summary-text a:link, - div.summary-text a { - color: #3273dc !important; - } - - div.summary-text a:visited { - color: #8a4de8 !important; - } - - div.summary-text a:hover { - color: #0a0a0a !important; - } - - .advisory-wrapper { +.state-affected { + background-color: #f14668 !important; + color: #fff !important; +} + +.state-fixed { + background-color: #48c78e !important; + color: #fff !important; +} + +.state-empty { + background-color: #ffdd57 !important; + color: #000 !important; + box-shadow: inset 0 0 0 1px #dbdbdb; +} + +.table { + table-layout: fixed !important; + width: 100% !important; +} + +.table th { + border-width: 0 1px 1px 1px !important; + border-color: #dbdbdb !important; + background-color: #fff !important; +} + +#table-header th { + height: 140px; + vertical-align: top !important; + padding: 0.75rem 0.5rem !important; +} + +#table-header th>div { + display: inline-flex !important; + flex-direction: column !important; + justify-content: space-between !important; + height: 100% !important; + width: 100%; +} + +#table-header th:first-child { + width: 150px !important; +} + +#table-header th:nth-child(2) { + width: 150px !important; +} + +#table-header th:nth-child(n+3) { + width: 120px !important; +} + +.curation-cell { + cursor: pointer; + text-align: center !important; + user-select: none; + transition: background-color 0.1s ease; + border: 1px solid #dbdbdb !important; +} + +.curation-cell:hover { + filter: brightness(0.95); + box-shadow: inset 0 0 0 1px #3273dc; +} + +.advisory-cell { + cursor: not-allowed; + word-break: break-all; +} + +.folded-row-marker, +.range-row-marker { + cursor: pointer; + text-align: center; + color: #4a4a4a; + font-weight: 600; + background-color: #f5f5f5; +} + +.folded-row-marker.is-expanded .icon { + transform: rotate(180deg); +} + +.range-data-cell { + background-color: #fafafa; + padding: 6px !important; + vertical-align: top !important; + border: 1px solid #dbdbdb !important; + + white-space: normal !important; + word-break: break-all !important; + overflow-wrap: break-word !important; +} + +#summaries-container p strong { + display: block; + word-break: break-all; + overflow-wrap: break-word; + line-height: 1.3; + margin-bottom: 4px; +} + +.summary-text { + line-height: 1.4; + white-space: pre-wrap; + word-break: break-word; +} + +div.summary-text a:link, +div.summary-text a { + color: #3273dc !important; +} + +div.summary-text a:visited { + color: #8a4de8 !important; +} + +div.summary-text a:hover { + color: #0a0a0a !important; +} + +.advisory-wrapper { width: 100%; word-break: break-all; white-space: normal; diff --git a/vulnerablecode/static/js/package_curation.js b/vulnerablecode/static/js/package_curation.js index 3b8db149d..e0083614b 100644 --- a/vulnerablecode/static/js/package_curation.js +++ b/vulnerablecode/static/js/package_curation.js @@ -238,7 +238,7 @@ const app = { createRow(v, item) { const tr = document.createElement('tr'); const state = this.userStates[this.currentIndex][v]; - tr.innerHTML = `${v}`; + tr.innerHTML = `${v}`; const userTd = document.createElement('td'); userTd.className = `curation-cell state-${state}`; userTd.innerText = state === "empty"? "Select value": state.toUpperCase();