wip

Signed-off-by: Keshav Priyadarshi <git@keshav.space>

Author: keshav-space

vulnerabilities/importers/__init__.py

@@ -32,6 +32,7 @@
 from vulnerabilities.importers import ubuntu_usn
 from vulnerabilities.importers import vulnrichment
 from vulnerabilities.importers import xen
+from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2
 from vulnerabilities.pipelines import alpine_linux_importer
 from vulnerabilities.pipelines import github_importer
 from vulnerabilities.pipelines import gitlab_importer
@@ -189,3 +190,9 @@
         collect_fix_commits_v2.CollectGitlabFixCommitsPipeline,
     ]
 )
+
+TODO_EXCLUDED_PIPELINES = [
+    key
+    for key, value in IMPORTERS_REGISTRY.items()
+    if issubclass(value, VulnerableCodeBaseImporterPipelineV2) and value.exclude_from_package_todo
+]

vulnerabilities/models.py

@@ -2877,11 +2877,7 @@ def latest_for_avid(self, avid: str):
         )
 
     def latest_per_avid(self):
-        return self.order_by(
-            "avid",
-            F("date_collected").desc(nulls_last=True),
-            "-id",
-        ).distinct("avid")
+        return self.filter(is_latest=True)
 
     def latest_for_avids(self, avids):
         return self.filter(avid__in=avids).latest_per_avid()
@@ -2953,6 +2949,12 @@ def latest_advisories_for_purl(self, purl):
         qs = self.filter(id__in=Subquery(adv_ids))
         return qs.latest_per_avid()
 
+    def todo_excluded(self):
+        """Exclude advisory ineligible for ToDo computation."""
+        from vulnerabilities.importers import TODO_EXCLUDED_PIPELINES
+
+        return self.exclude(datasource_id__in=TODO_EXCLUDED_PIPELINES)
+
 
 class AdvisorySet(models.Model):
 
@@ -2998,6 +3000,7 @@ class AdvisoryV2(models.Model):
         max_length=200,
         blank=False,
         null=False,
+        db_index=True,
         help_text="Unique ID for the datasource used for this advisory ." "e.g.: nginx_importer_v2",
     )
 
@@ -3081,6 +3084,14 @@ class AdvisoryV2(models.Model):
         help_text="UTC Date on which the advisory was collected",
     )
 
+    is_latest = models.BooleanField(
+        default=False,
+        blank=False,
+        null=False,
+        db_index=True,
+        help_text="Indicates whether this is the latest version of the advisory identified by its AVID.",
+    )
+
     original_advisory_text = models.TextField(
         blank=True,
         null=True,
@@ -3133,6 +3144,11 @@ class AdvisoryV2(models.Model):
     class Meta:
         unique_together = ["datasource_id", "advisory_id", "unique_content_id"]
         ordering = ["datasource_id", "advisory_id", "date_published", "unique_content_id"]
+        constraints = [
+            models.UniqueConstraint(
+                fields=["avid"], condition=Q(is_latest=True), name="unique_latest_per_avid"
+            )
+        ]
         indexes = [
             models.Index(
                 fields=["avid", "-date_collected", "-id"],

vulnerabilities/pipelines/__init__.py

@@ -278,6 +278,11 @@ class VulnerableCodeBaseImporterPipelineV2(VulnerableCodePipeline):
     ignorable_versions = []
     precedence = 0
 
+    # Set this to True if computing fixed/affected package ToDo is not fruitful for this source.
+    # An example of such advisory would be pipeline dedicated to collecting issues,
+    # pull requests, commit messages, EPSS, exploits, etc.
+    exclude_from_package_todo = False
+
     # Control how often progress log is shown (range: 1–100, higher value = less frequent log)
     progress_step = 10
 

vulnerabilities/pipelines/v2_importers/__init__.py

@@ -0,0 +1,8 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# VulnerableCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/aboutcode-org/vulnerablecode for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#

vulnerabilities/pipelines/v2_importers/aosp_importer.py

@@ -32,6 +32,7 @@ class AospImporterPipeline(VulnerableCodeBaseImporterPipelineV2):
     license_url = "https://github.com/quarkslab/aosp_dataset/blob/master/LICENSE"
 
     precedence = 200
+    exclude_from_package_todo = True
 
     @classmethod
     def steps(cls):

vulnerabilities/pipelines/v2_importers/epss_importer_v2.py

@@ -30,6 +30,8 @@ class EPSSImporterPipeline(VulnerableCodeBaseImporterPipelineV2):
     spdx_license_expression = "unknown"
     importer_name = "EPSS Importer"
 
+    exclude_from_package_todo = True
+
     precedence = 200
 
     def advisories_count(self):

vulnerabilities/pipelines/v2_importers/nvd_importer.py

@@ -71,6 +71,8 @@ class NVDImporterPipeline(VulnerableCodeBaseImporterPipelineV2):
         MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE.
     """
 
+    exclude_from_package_todo = True
+
     precedence = 100
 
     @classmethod

vulnerabilities/pipelines/v2_importers/project_kb_msr2019_importer.py

@@ -30,6 +30,8 @@ class ProjectKBMSR2019Pipeline(VulnerableCodeBaseImporterPipelineV2):
     license_url = "https://github.com/SAP/project-kb/blob/main/LICENSE.txt"
     repo_url = "git+https://github.com/SAP/project-kb"
 
+    exclude_from_package_todo = True
+
     precedence = 200
 
     @classmethod

vulnerabilities/pipelines/v2_importers/project_kb_statements_importer.py

@@ -37,6 +37,8 @@ class ProjectKBStatementsPipeline(VulnerableCodeBaseImporterPipelineV2):
     license_url = "https://github.com/SAP/project-kb/blob/main/LICENSE.txt"
     repo_url = "git+https://github.com/SAP/project-kb@vulnerability-data"
 
+    exclude_from_package_todo = True
+
     precedence = 200
 
     @classmethod

vulnerabilities/pipelines/v2_importers/suse_score_importer.py

@@ -23,6 +23,8 @@ class SUSESeverityScoreImporterPipeline(VulnerableCodeBaseImporterPipelineV2):
     pipeline_id = "suse_importer_v2"
     url = "https://ftp.suse.com/pub/projects/security/yaml/suse-cvss-scores.yaml"
 
+    exclude_from_package_todo = True
+
     @classmethod
     def steps(cls):
         return (