Merge pull request #2366 from aboutcode-org/validate-cvss-score

Avoid converting None to "None" in AdvisorySeverity

Author: keshav-space

docs/source/conf.py

@@ -42,6 +42,7 @@
     "http://ftp.suse.com/pub/projects/security/yaml/",
     r"https://nixos\.wiki/",  # NixOS wiki blocks CI bots with 403
     "https://usn.ubuntu.com/usn-db/database-all.json.bz2",
+    "https://public.vulnerablecode.io/vulnerabilities/search/",
 ]
 
 # Add any Sphinx extension module names here, as strings. They can be

vulnerabilities/importer.py

@@ -19,7 +19,6 @@
 from typing import Optional
 from typing import Set
 from typing import Tuple
-from typing import Union
 
 import pytz
 from dateutil import parser as dateparser
@@ -40,7 +39,6 @@
 from vulnerabilities.utils import get_reference_id
 from vulnerabilities.utils import is_commit
 from vulnerabilities.utils import is_cve
-from vulnerabilities.utils import nearest_patched_package
 from vulnerabilities.utils import purl_to_dict
 from vulnerabilities.utils import update_purl_version
 
@@ -65,7 +63,7 @@ def __post_init__(self):
             raise TypeError(f"system must be a ScoringSystem, got {type(self.system)!r}")
 
         if not isinstance(self.value, str):
-            self.value = str(self.value)
+            self.value = str(self.value) if self.value else ""
 
     def to_dict(self):
         data = {

vulnerabilities/migrations/0139_cleanup_none_string_in_severity.py

@@ -0,0 +1,132 @@
+# Generated by Django 5.2.11 on 2026-07-02 09:50
+
+from django.db import migrations
+
+from vulnerabilities.importer import AdvisoryDataV2
+from vulnerabilities.importer import AffectedPackageV2
+from vulnerabilities.importer import PatchData
+from vulnerabilities.importer import ReferenceV2
+from vulnerabilities.importer import VulnerabilitySeverity
+from vulnerabilities.utils import compute_content_id_v2
+from vulnerabilities.utils import normalize_list
+from vulnerabilities.utils import purl_to_dict
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("vulnerabilities", "0138_fix_malformed_cvss_vector"),
+    ]
+
+    def cleanup_none_severity_string(apps, schema_editor):
+        AdvisorySeverity = apps.get_model("vulnerabilities", "AdvisorySeverity")
+        AdvisoryV2 = apps.get_model("vulnerabilities", "AdvisoryV2")
+        batch = []
+        batch_size = 5000
+
+        advisory_ids = list(
+            AdvisoryV2.objects.filter(severities__value="None")
+            .distinct()
+            .values_list("id", flat=True)
+        )
+
+        AdvisorySeverity.objects.filter(value="None").update(value="")
+
+        for advisory in AdvisoryV2.objects.filter(id__in=advisory_ids).iterator(chunk_size=2000):
+            advisory.unique_content_id = compute_content_id_v2(to_advisory_data(advisory))
+            batch.append(advisory)
+
+            if len(batch) >= batch_size:
+                AdvisoryV2.objects.bulk_update(batch, ["unique_content_id"])
+                batch.clear()
+
+        if batch:
+            AdvisoryV2.objects.bulk_update(batch, ["unique_content_id"])
+
+    operations = [
+        migrations.RunPython(
+            cleanup_none_severity_string,
+            reverse_code=migrations.RunPython.noop,
+        ),
+    ]
+
+
+def commit_patch_to_dict(patch):
+    return {
+        "vcs_url": patch.vcs_url,
+        "commit_hash": patch.commit_hash,
+        "patch_text": patch.patch_text,
+        "patch_checksum": patch.patch_checksum,
+    }
+
+
+def to_affected_package_data(impact):
+    """Return `AffectedPackageV2` data from the impact."""
+    return AffectedPackageV2.from_dict(
+        {
+            "package": purl_to_dict(impact.base_purl),
+            "affected_version_range": impact.affecting_vers,
+            "fixed_version_range": impact.fixed_vers,
+            "introduced_by_commit_patches": [
+                commit_patch_to_dict(commit)
+                for commit in impact.introduced_by_package_commit_patches.all()
+            ],
+            "fixed_by_commit_patches": [
+                commit_patch_to_dict(commit)
+                for commit in impact.fixed_by_package_commit_patches.all()
+            ],
+        }
+    )
+
+
+def to_patch_data(patch):
+    """Return `PatchData` from the Patch."""
+
+    return PatchData.from_dict(
+        {
+            "patch_url": patch.patch_url,
+            "patch_text": patch.patch_text,
+            "patch_checksum": patch.patch_checksum,
+        }
+    )
+
+
+def to_reference_v2_data(ref):
+    return ReferenceV2.from_dict(
+        {
+            "reference_id": ref.reference_id,
+            "reference_type": ref.reference_type,
+            "url": ref.url,
+        }
+    )
+
+
+def to_vulnerability_severity_data(severity):
+    return VulnerabilitySeverity.from_dict(
+        {
+            "system": severity.scoring_system,
+            "value": severity.value,
+            "scoring_elements": severity.scoring_elements,
+            "published_at": severity.published_at,
+            "url": severity.url,
+        }
+    )
+
+
+def to_advisory_data(advisory):
+    return AdvisoryDataV2(
+        advisory_id=advisory.advisory_id,
+        aliases=normalize_list([item.alias for item in advisory.aliases.all()]),
+        summary=advisory.summary,
+        affected_packages=normalize_list(
+            [to_affected_package_data(impacted) for impacted in advisory.impacted_packages.all()]
+        ),
+        references=normalize_list([to_reference_v2_data(ref) for ref in advisory.references.all()]),
+        patches=normalize_list([to_patch_data(patch) for patch in advisory.patches.all()]),
+        date_published=advisory.date_published,
+        weaknesses=normalize_list([weak.cwe_id for weak in advisory.weaknesses.all()]),
+        severities=normalize_list(
+            [to_vulnerability_severity_data(sev) for sev in advisory.severities.all()]
+        ),
+        url=advisory.url,
+    )

vulnerabilities/models.py

@@ -9,8 +9,6 @@
 
 import csv
 import datetime
-import hashlib
-import json
 import logging
 import uuid
 import xml.etree.ElementTree as ET

vulnerabilities/tests/pipelines/v2_improvers/test_compute_advisory_todo_v2.py

@@ -26,6 +26,7 @@
 from vulnerabilities.pipelines.v2_improvers.compute_advisory_todo import ComputeToDo
 from vulnerabilities.pipes.advisory import insert_advisory_v2
 from vulnerabilities.tests.pipelines import TestLogger
+from vulnerabilities.utils import canonical_value
 
 
 class TestComputeToDo(TestCase):
@@ -690,7 +691,10 @@ def test_todo_conflict_details_partial_curation_unpaired_purl_and_conflicting_af
         result_partial_curation = issue_details["partial_curation_advisory"]
         self.assertEqual(1, AdvisoryToDoV2.objects.count())
         self.assertEqual("CONFLICTING_AFFECTED_PACKAGES", todo.issue_type)
-        self.assertDictEqual(expected_partial_curation_advisory, result_partial_curation)
+        self.assertCountEqual(
+            expected_partial_curation_advisory["affected_packages"],
+            result_partial_curation["affected_packages"],
+        )
 
     def test_todo_conflicting_severity(self):
         insert_advisory_v2(

vulnerabilities/tests/test_data_migrations.py

@@ -23,6 +23,10 @@
 from vulnerabilities.importer import AdvisoryData
 from vulnerabilities.importer import AffectedPackage
 from vulnerabilities.importer import Reference
+from vulnerabilities.models import AdvisorySeverity
+from vulnerabilities.models import AdvisoryV2
+from vulnerabilities.models import ImpactedPackage
+from vulnerabilities.utils import compute_content_id_v2
 from vulnerabilities.utils import purl_to_dict
 
 
@@ -1378,3 +1382,50 @@ def test_scheme_migration_correctness(self):
 
         self.assertEqual(self.impact2.affecting_vers, "vers:apk/3.4.5")
         self.assertEqual(self.impact2.fixed_vers, None)
+
+
+class TestCleanAdvisorySeverityMigration(TestMigrations):
+    app_name = "vulnerabilities"
+    migrate_from = "0138_fix_malformed_cvss_vector"
+    migrate_to = "0139_cleanup_none_string_in_severity"
+
+    def setUpBeforeMigration(self, apps):
+        # AdvisoryV2 = apps.get_model("vulnerabilities", "AdvisoryV2")
+        # ImpactedPackage = apps.get_model("vulnerabilities", "ImpactedPackage")
+        # AdvisorySeverity = apps.get_model("vulnerabilities", "AdvisorySeverity")
+
+        self.advisory1 = AdvisoryV2.objects.create(
+            unique_content_id="b001d1a8952bc056d0161f1dd45dd8f90b25f62c56a887ea21d09fafd78a0f61",
+            url="https://old.example.com",
+            summary="Old advisory",
+            advisory_id="test_adv1",
+            avid="test_pipeline/test_adv",
+            datasource_id="test_pipeline",
+            pipeline_id="test_pipeline_v2",
+        )
+
+        ImpactedPackage.objects.create(
+            advisory=self.advisory1,
+            base_purl="pkg:npm/foobar0",
+            affecting_vers="vers:npm/4.3.2",
+            fixed_vers="vers:npm/5.0.0",
+        )
+
+        self.severity = AdvisorySeverity.objects.create(
+            scoring_system=severity_systems.CVSSV4,
+            scoring_elements="CVSS:4.0/AV:N/AC:L/AT:P/PR:H/UI:P/VC:N/VI:N/VA:N",
+            value="None",
+        )
+
+        self.advisory1.severities.add(self.severity)
+
+    def test_severity_value_cleaned(self):
+        self.severity.refresh_from_db()
+        self.assertEqual(self.severity.value, "")
+
+    def test_advisory_content_id_recomputed(self):
+        self.advisory1.refresh_from_db()
+        self.assertEqual(
+            self.advisory1.unique_content_id,
+            "a15d4651cb05e3513c12263a11e34bd9103f68833cac8f7ffdbbd71b9cb4cf16",
+        )

vulnerabilities/tests/test_importer.py

@@ -13,6 +13,7 @@
 from vulnerabilities.importer import PackageCommitPatchData
 from vulnerabilities.importer import PatchData
 from vulnerabilities.importer import ReferenceV2
+from vulnerabilities.importer import VulnerabilitySeverity
 from vulnerabilities.pipes.advisory import classify_patch_source
 
 
@@ -201,3 +202,15 @@ def test_classify_patch_source_integration(url, commit_hash, patch_text, results
             assert actual_data_obj.reference_id == expected_data_obj.reference_id
             assert actual_data_obj.reference_type == expected_data_obj.reference_type
             assert actual_data_obj.url == expected_data_obj.url
+
+
+def test_vulnerability_severity_value_string_conversion():
+    severity = VulnerabilitySeverity.from_dict(
+        {
+            "system": "cvssv3",
+            "value": None,
+            "scoring_elements": "CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:N/A:N",
+        }
+    )
+
+    assert severity.value == ""