Add ruby importer and improver

Fix style test

Fix test

Rewrite affected_packages

Ruby initial config

Reference: #796

Signed-off-by: ziadhany <[email protected]>

Author: ziadhany

vulnerabilities/importers/__init__.py

@@ -30,6 +30,7 @@
 from vulnerabilities.importers import pysec
 from vulnerabilities.importers import redhat
 from vulnerabilities.importers import retiredotnet
+from vulnerabilities.importers import ruby
 from vulnerabilities.importers import suse_scores
 from vulnerabilities.importers import ubuntu
 from vulnerabilities.importers import ubuntu_usn
@@ -63,6 +64,7 @@
     xen.XenImporter,
     ubuntu_usn.UbuntuUSNImporter,
     fireeye.FireyeImporter,
+    ruby.RubyImporter,
 ]
 
 IMPORTERS_REGISTRY = {x.qualified_name: x for x in IMPORTERS_REGISTRY}

vulnerabilities/importers/ruby.py

@@ -7,132 +7,191 @@
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
 
-import asyncio
-from typing import List
-from typing import Set
+import logging
+from pathlib import Path
+from typing import Iterable
 
 from dateutil.parser import parse
+from django.db.models import QuerySet
 from packageurl import PackageURL
 from pytz import UTC
-from univers.version_range import VersionRange
-from univers.versions import SemverVersion
+from univers.version_range import GemVersionRange
+from univers.versions import RubygemsVersion
 
 from vulnerabilities.importer import AdvisoryData
+from vulnerabilities.importer import AffectedPackage
 from vulnerabilities.importer import GitImporter
 from vulnerabilities.importer import Reference
+from vulnerabilities.importer import VulnerabilitySeverity
+from vulnerabilities.improver import Improver
+from vulnerabilities.improver import Inference
+from vulnerabilities.models import Advisory
 from vulnerabilities.package_managers import RubyVersionAPI
+from vulnerabilities.severity_systems import SCORING_SYSTEMS
+from vulnerabilities.utils import build_description
+from vulnerabilities.utils import evolve_purl
 from vulnerabilities.utils import load_yaml
-from vulnerabilities.utils import nearest_patched_package
 
+logger = logging.getLogger(__name__)
 
-class RubyImporter(GitImporter):
-    def __enter__(self):
-        super(RubyImporter, self).__enter__()
-
-        if not getattr(self, "_added_files", None):
-            self._added_files, self._updated_files = self.file_changes(
-                recursive=True, file_ext="yml", subdir="./gems"
-            )
 
-        self.pkg_manager_api = RubyVersionAPI()
-        self.set_api(self.collect_packages())
-
-    def set_api(self, packages):
-        asyncio.run(self.pkg_manager_api.load_api(packages))
-
-    def updated_advisories(self) -> Set[AdvisoryData]:
-        files = self._updated_files.union(self._added_files)
-        advisories = []
-        for f in files:
-            processed_data = self.process_file(f)
-            if processed_data:
-                advisories.append(processed_data)
-        return self.batch_advisories(advisories)
-
-    def collect_packages(self):
-        packages = set()
-        files = self._updated_files.union(self._added_files)
-        for f in files:
-            data = load_yaml(f)
-            if data.get("gem"):
-                packages.add(data["gem"])
-
-        return packages
-
-    def process_file(self, path) -> List[AdvisoryData]:
-        record = load_yaml(path)
+class RubyImporter(GitImporter):
+    license_url = "https://github.com/rubysec/ruby-advisory-db/blob/master/LICENSE.txt"
+    spdx_license_expression = "unknown"
+
+    def __init__(self):
+        super().__init__(repo_url="git+https://github.com/rubysec/ruby-advisory-db")
+
+    def advisory_data(self) -> Iterable[AdvisoryData]:
+        self.clone()
+        base_path = Path(self.vcs_response.dest_dir)
+        supported_subdir = ["rubies", "gems"]
+        for subdir in supported_subdir:
+            for file_path in base_path.glob(f"{subdir}/**/*.yml"):
+                if file_path.name.startswith("OSVDB-"):
+                    continue
+                raw_data = load_yaml(file_path)
+                yield parse_ruby_advisory(raw_data, subdir)
+
+
+def parse_ruby_advisory(record, schema_type):
+    """
+    Parse a ruby advisory file and return an AdvisoryData or None.
+    Each advisory file contains the advisory information in YAML format.
+    Schema: https://github.com/rubysec/ruby-advisory-db/tree/master/spec/schemas
+    """
+    if schema_type == "gems":
         package_name = record.get("gem")
-        if not package_name:
-            return
-
-        if "cve" in record:
-            cve_id = "CVE-{}".format(record["cve"])
-        else:
-            return
-
-        publish_time = parse(record["date"]).replace(tzinfo=UTC)
-        safe_version_ranges = record.get("patched_versions", [])
-        # this case happens when the advisory contain only 'patched_versions' field
-        # and it has value None(i.e it is empty :( ).
-        if not safe_version_ranges:
-            safe_version_ranges = []
-        safe_version_ranges += record.get("unaffected_versions", [])
-        safe_version_ranges = [i for i in safe_version_ranges if i]
-
-        if not getattr(self, "pkg_manager_api", None):
-            self.pkg_manager_api = RubyVersionAPI()
-        all_vers = self.pkg_manager_api.get(package_name, until=publish_time).valid_versions
-        safe_versions, affected_versions = self.categorize_versions(all_vers, safe_version_ranges)
-
-        impacted_purls = [
-            PackageURL(
-                name=package_name,
-                type="gem",
-                version=version,
-            )
-            for version in affected_versions
-        ]
-
-        resolved_purls = [
-            PackageURL(
-                name=package_name,
-                type="gem",
-                version=version,
-            )
-            for version in safe_versions
-        ]
+        library = record.get("library")
+        framework = record.get("framework")
+        platform = record.get("platform")
+        purl = PackageURL(type="gem", name=package_name)
 
-        references = []
-        if record.get("url"):
-            references.append(Reference(url=record.get("url")))
+        return AdvisoryData(
+            aliases=get_aliases(record),
+            summary=get_summary(record),
+            affected_packages=get_affected_packages(record, purl),
+            references=get_references(record),
+            date_published=get_publish_time(record),
+        )
 
+    elif schema_type == "rubies":
+        engine = record.get("engine")  # engine enum: [jruby, rbx, ruby]
+        purl = PackageURL(type="ruby", name=engine)
         return AdvisoryData(
-            summary=record.get("description", ""),
-            affected_packages=nearest_patched_package(impacted_purls, resolved_purls),
-            references=references,
-            vulnerability_id=cve_id,
+            aliases=get_aliases(record),
+            summary=get_summary(record),
+            affected_packages=get_affected_packages(record, purl),
+            references=get_references(record),
+            date_published=get_publish_time(record),
         )
 
-    @staticmethod
-    def categorize_versions(all_versions, unaffected_version_ranges):
 
-        for id, elem in enumerate(unaffected_version_ranges):
-            unaffected_version_ranges[id] = VersionRange.from_scheme_version_spec_string(
-                "semver", elem
+def get_affected_packages(record, purl):
+    safe_version_ranges = record.get("patched_versions", [])
+    # this case happens when the advisory contain only 'patched_versions' field
+    # and it has value None(i.e it is empty :( ).
+    if not safe_version_ranges:
+        safe_version_ranges = []
+    safe_version_ranges += record.get("unaffected_versions", [])
+    safe_version_ranges = [i for i in safe_version_ranges if i]
+
+    affected_packages = []
+    affected_version_ranges = [
+        GemVersionRange.from_native(elem).invert() for elem in safe_version_ranges
+    ]
+
+    for affected_version_range in affected_version_ranges:
+        affected_packages.append(
+            AffectedPackage(
+                package=purl,
+                affected_version_range=affected_version_range,
             )
+        )
+    return affected_packages
+
+
+def get_aliases(record) -> [str]:
+    aliases = []
+    if record.get("cve"):
+        aliases.append("CVE-{}".format(record.get("cve")))
+    if record.get("osvdb"):
+        aliases.append("OSV-{}".format(record.get("osvdb")))
+    if record.get("ghsa"):
+        aliases.append("GHSA-{}".format(record.get("ghsa")))
+    return aliases
+
 
-        safe_versions = []
-        vulnerable_versions = []
-        for i in all_versions:
-            vobj = SemverVersion(i)
-            is_vulnerable = False
-            for ver_rng in unaffected_version_ranges:
-                if vobj in ver_rng:
-                    safe_versions.append(i)
-                    is_vulnerable = True
-                    break
-
-            if not is_vulnerable:
-                vulnerable_versions.append(i)
-
-        return safe_versions, vulnerable_versions
+def get_references(record) -> [Reference]:
+    references = []
+    cvss_v2 = record.get("cvss_v2")
+    cvss_v3 = record.get("cvss_v3")
+
+    if record.get("url"):
+        if not (cvss_v2 or cvss_v3):
+            references.append(Reference(url=record.get("url")))
+        if cvss_v2:
+            references.append(
+                Reference(
+                    url=record.get("url"),
+                    severities=[
+                        VulnerabilitySeverity(system=SCORING_SYSTEMS["cvssv2"], value=cvss_v2)
+                    ],
+                )
+            )
+        if cvss_v3:
+            references.append(
+                Reference(
+                    url=record.get("url"),
+                    severities=[
+                        VulnerabilitySeverity(system=SCORING_SYSTEMS["cvssv3"], value=cvss_v3)
+                    ],
+                )
+            )
+    return references
+
+
+def get_publish_time(record):
+    return parse(record["date"]).replace(tzinfo=UTC)
+
+
+def get_summary(record):
+    title = record.get("title")
+    description = record.get("description", "")
+    return build_description(summary=title, description=description)
+
+
+class RubyImprover(Improver):
+    pkg_manager_api = RubyVersionAPI()
+
+    @property
+    def interesting_advisories(self) -> QuerySet:
+        return Advisory.objects.filter(created_by=RubyImporter.qualified_name)
+
+    def get_inferences(self, advisory_data) -> Iterable[Inference]:
+        for affected_package in advisory_data.affected_packages:
+            purl = affected_package.package
+            pkg_name = purl.name
+            all_vers_pkgs = self.pkg_manager_api.fetch(pkg_name)
+
+            safe_versions = []
+            affected_purls = []
+            for pkg_version in all_vers_pkgs:
+                vobj = RubygemsVersion(pkg_version.value)
+                try:
+                    if vobj in affected_package.affected_version_range:
+                        new_purl = evolve_purl(purl=purl, version=str(pkg_version.value))
+                        affected_purls.append(new_purl)
+                    else:
+                        safe_versions.append(pkg_version.value)
+                except Exception as e:
+                    logger.error(f"{e}")
+
+            for fixed_version in safe_versions:
+                fixed_purl = evolve_purl(purl=purl, version=str(fixed_version))
+                yield Inference.from_advisory_data(
+                    advisory_data,
+                    confidence=90,
+                    affected_purls=affected_purls,
+                    fixed_purl=fixed_purl,
+                )

vulnerabilities/improvers/__init__.py

@@ -21,6 +21,7 @@
     oval.DebianOvalBasicImprover,
     oval.UbuntuOvalBasicImprover,
     importers.apache_httpd.ApacheHTTPDImprover,
+    importers.ruby.RubyImprover,
 ]
 
 IMPROVERS_REGISTRY = {x.qualified_name: x for x in IMPROVERS_REGISTRY}

vulnerabilities/tests/conftest.py

@@ -28,7 +28,6 @@ def no_rmtree(monkeypatch):
     "test_apache_kafka.py",
     "test_models.py",
     "test_package_managers.py",
-    "test_ruby.py",
     "test_rust.py",
     "test_suse_backports.py",
     "test_suse.py",

vulnerabilities/tests/test_data/ruby/CVE-2007-5770-expected.json

@@ -0,0 +1,47 @@
+{
+  "aliases": [
+    "CVE-2007-5770"
+  ],
+  "summary": "Ruby Net::HTTPS library does not validate server certificate CN\nThe (1) Net::ftptls, (2) Net::telnets, (3) Net::imap, (4) Net::pop, and (5)\nNet::smtp libraries in Ruby 1.8.5 and 1.8.6 do not verify that the\ncommonName (CN) field in a server certificate matches the domain name in a\nrequest sent over SSL, which makes it easier for remote attackers to\nintercept SSL transmissions via a man-in-the-middle attack or spoofed web\nsite, different components than CVE-2007-5162.",
+  "affected_packages": [
+    {
+      "package": {
+        "type": "ruby",
+        "namespace": null,
+        "name": "ruby",
+        "version": null,
+        "qualifiers": null,
+        "subpath": null
+      },
+      "affected_version_range": "vers:gem/<1.8.6.230|>=1.8.7",
+      "fixed_version": null
+    },
+    {
+      "package": {
+        "type": "ruby",
+        "namespace": null,
+        "name": "ruby",
+        "version": null,
+        "qualifiers": null,
+        "subpath": null
+      },
+      "affected_version_range": "vers:gem/<1.8.7",
+      "fixed_version": null
+    }
+  ],
+  "references": [
+    {
+      "reference_id": "",
+      "url": "http://www.cvedetails.com/cve/CVE-2007-5770/",
+      "severities": [
+        {
+          "system": "cvssv2",
+          "value": "4.3",
+          "scoring_elements": ""
+        }
+      ]
+    }
+  ],
+  "date_published": "2007-10-08T00:00:00+00:00",
+  "weaknesses": []
+}

vulnerabilities/tests/test_data/ruby/CVE-2007-5770.yml

@@ -0,0 +1,17 @@
+---
+engine: ruby
+cve: 2007-5770
+url: http://www.cvedetails.com/cve/CVE-2007-5770/
+title: Ruby Net::HTTPS library does not validate server certificate CN
+date: 2007-10-08
+description: |
+  The (1) Net::ftptls, (2) Net::telnets, (3) Net::imap, (4) Net::pop, and (5)
+  Net::smtp libraries in Ruby 1.8.5 and 1.8.6 do not verify that the
+  commonName (CN) field in a server certificate matches the domain name in a
+  request sent over SSL, which makes it easier for remote attackers to
+  intercept SSL transmissions via a man-in-the-middle attack or spoofed web
+  site, different components than CVE-2007-5162.
+cvss_v2: 4.3
+patched_versions:
+- ~> 1.8.6.230
+- '>= 1.8.7'