|
| 1 | +# |
| 2 | +# Copyright (c) nexB Inc. and others. All rights reserved. |
| 3 | +# VulnerableCode is a trademark of nexB Inc. |
| 4 | +# SPDX-License-Identifier: Apache-2.0 |
| 5 | +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. |
| 6 | +# See https://github.com/aboutcode-org/vulnerablecode for support or download. |
| 7 | +# See https://aboutcode.org for more information about nexB OSS projects. |
| 8 | +# |
| 9 | + |
| 10 | +import logging |
| 11 | +import re |
| 12 | +from datetime import datetime |
| 13 | +from datetime import timezone |
| 14 | +from typing import Iterable |
| 15 | +from typing import List |
| 16 | +from typing import Optional |
| 17 | + |
| 18 | +from bs4 import BeautifulSoup |
| 19 | + |
| 20 | +from vulnerabilities.importer import AdvisoryDataV2 |
| 21 | +from vulnerabilities.importer import ReferenceV2 |
| 22 | +from vulnerabilities.importer import VulnerabilitySeverity |
| 23 | +from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2 |
| 24 | +from vulnerabilities.severity_systems import SCORING_SYSTEMS |
| 25 | +from vulnerabilities.utils import fetch_response |
| 26 | + |
| 27 | +logger = logging.getLogger(__name__) |
| 28 | + |
| 29 | +KDE_SECURITY_URL = "https://kde.org/info/security/" |
| 30 | + |
| 31 | + |
| 32 | +class KDEImporterPipeline(VulnerableCodeBaseImporterPipelineV2): |
| 33 | + """Importer for KDE security advisories.""" |
| 34 | + |
| 35 | + pipeline_id = "kde_importer" |
| 36 | + spdx_license_expression = "CC-BY-SA-4.0" |
| 37 | + license_url = "https://kde.org/info/security/" |
| 38 | + repo_url = "https://kde.org/info/security/" |
| 39 | + |
| 40 | + precedence = 200 |
| 41 | + |
| 42 | + _cached_advisory_urls = None |
| 43 | + |
| 44 | + @classmethod |
| 45 | + def steps(cls): |
| 46 | + return (cls.collect_and_store_advisories,) |
| 47 | + |
| 48 | + def get_advisory_urls(self) -> List[str]: |
| 49 | + if self._cached_advisory_urls is not None: |
| 50 | + return self._cached_advisory_urls |
| 51 | + |
| 52 | + try: |
| 53 | + response = fetch_response(KDE_SECURITY_URL) |
| 54 | + html = response.text |
| 55 | + except Exception as e: |
| 56 | + logger.error(f"Failed to fetch KDE security page: {e}") |
| 57 | + return [] |
| 58 | + |
| 59 | + soup = BeautifulSoup(html, "lxml") |
| 60 | + advisory_urls = [] |
| 61 | + |
| 62 | + for link in soup.find_all("a", href=True): |
| 63 | + href = link["href"] |
| 64 | + if "advisory-" in href and href.endswith(".txt"): |
| 65 | + if href.startswith("http"): |
| 66 | + advisory_urls.append(href) |
| 67 | + else: |
| 68 | + advisory_urls.append(f"https://kde.org{href}") |
| 69 | + |
| 70 | + self._cached_advisory_urls = advisory_urls |
| 71 | + return advisory_urls |
| 72 | + |
| 73 | + def advisories_count(self) -> int: |
| 74 | + return len(self.get_advisory_urls()) |
| 75 | + |
| 76 | + def collect_advisories(self) -> Iterable[AdvisoryDataV2]: |
| 77 | + for url in self.get_advisory_urls(): |
| 78 | + try: |
| 79 | + response = fetch_response(url) |
| 80 | + advisory = parse_kde_advisory(response.text, url) |
| 81 | + if advisory: |
| 82 | + yield advisory |
| 83 | + except Exception as e: |
| 84 | + logger.error(f"Failed to parse advisory {url}: {e}") |
| 85 | + continue |
| 86 | + |
| 87 | + |
| 88 | +def parse_kde_advisory(advisory_text: str, url: str) -> Optional[AdvisoryDataV2]: |
| 89 | + """Parse a KDE security advisory and return an AdvisoryDataV2.""" |
| 90 | + if not advisory_text: |
| 91 | + return None |
| 92 | + |
| 93 | + title = extract_field(advisory_text, "Title") |
| 94 | + risk_rating = extract_field(advisory_text, "Risk Rating") |
| 95 | + cve = extract_field(advisory_text, "CVE") |
| 96 | + date_str = extract_field(advisory_text, "Date") |
| 97 | + |
| 98 | + summary = extract_section(advisory_text, "Overview") |
| 99 | + |
| 100 | + advisory_id = generate_advisory_id(url, cve) |
| 101 | + if not advisory_id: |
| 102 | + logger.warning(f"Could not generate advisory ID for {url}") |
| 103 | + return None |
| 104 | + |
| 105 | + aliases = parse_cve_ids(cve) |
| 106 | + if advisory_id in aliases: |
| 107 | + aliases.remove(advisory_id) |
| 108 | + |
| 109 | + date_published = parse_date(date_str) |
| 110 | + |
| 111 | + references = [ReferenceV2(url=url)] |
| 112 | + for cve_id in aliases: |
| 113 | + references.append( |
| 114 | + ReferenceV2( |
| 115 | + reference_id=cve_id, |
| 116 | + url=f"https://cve.mitre.org/cgi-bin/cvename.cgi?name={cve_id}", |
| 117 | + ) |
| 118 | + ) |
| 119 | + |
| 120 | + severities = [] |
| 121 | + if risk_rating: |
| 122 | + severities.append( |
| 123 | + VulnerabilitySeverity( |
| 124 | + system=SCORING_SYSTEMS["generic_textual"], |
| 125 | + value=risk_rating.strip(), |
| 126 | + url=url, |
| 127 | + ) |
| 128 | + ) |
| 129 | + |
| 130 | + return AdvisoryDataV2( |
| 131 | + advisory_id=advisory_id, |
| 132 | + aliases=aliases, |
| 133 | + summary=summary or "", |
| 134 | + references=references, |
| 135 | + date_published=date_published, |
| 136 | + severities=severities, |
| 137 | + url=url, |
| 138 | + original_advisory_text=advisory_text, |
| 139 | + ) |
| 140 | + |
| 141 | + |
| 142 | +def extract_field(text: str, field_name: str) -> Optional[str]: |
| 143 | + """Extract a field value like 'Title: value' from advisory text.""" |
| 144 | + pattern = rf"^{re.escape(field_name)}:\s*(.+?)$" |
| 145 | + match = re.search(pattern, text, re.MULTILINE | re.IGNORECASE) |
| 146 | + if match: |
| 147 | + return match.group(1).strip() |
| 148 | + return None |
| 149 | + |
| 150 | + |
| 151 | +def extract_section(text: str, section_name: str) -> Optional[str]: |
| 152 | + """Extract a section like 'Overview\n========\ncontent' from advisory text.""" |
| 153 | + pattern = rf"{re.escape(section_name)}\s*\n=+\s*\n(.*?)(?=\n\w+[^\n]*\n=+|\Z)" |
| 154 | + match = re.search(pattern, text, re.DOTALL | re.IGNORECASE) |
| 155 | + if match: |
| 156 | + content = match.group(1).strip() |
| 157 | + content = re.sub(r"\n\s*\n", "\n\n", content) |
| 158 | + return content |
| 159 | + return None |
| 160 | + |
| 161 | + |
| 162 | +def generate_advisory_id(url: str, cve: Optional[str]) -> Optional[str]: |
| 163 | + """Generate advisory ID from URL (advisory-YYYYMMDD-N.txt) or fall back to CVE.""" |
| 164 | + match = re.search(r"advisory-(\d{8})-(\d+)\.txt", url) |
| 165 | + if match: |
| 166 | + return f"KDE-SA-{match.group(1)}-{match.group(2)}" |
| 167 | + |
| 168 | + cve_ids = parse_cve_ids(cve) |
| 169 | + if cve_ids: |
| 170 | + return cve_ids[0] |
| 171 | + |
| 172 | + return None |
| 173 | + |
| 174 | + |
| 175 | +def parse_cve_ids(cve_field: Optional[str]) -> List[str]: |
| 176 | + """Parse CVE IDs from field, including old CAN- format.""" |
| 177 | + if not cve_field: |
| 178 | + return [] |
| 179 | + |
| 180 | + cve_pattern = r"(CVE-\d{4}-\d+|CAN-\d{4}-\d+)" |
| 181 | + matches = re.findall(cve_pattern, cve_field, re.IGNORECASE) |
| 182 | + return [m.upper().replace("CAN-", "CVE-") for m in matches] |
| 183 | + |
| 184 | + |
| 185 | +def parse_date(date_str: Optional[str]) -> Optional[datetime]: |
| 186 | + """Parse date from various formats used in KDE advisories.""" |
| 187 | + if not date_str: |
| 188 | + return None |
| 189 | + |
| 190 | + date_str = date_str.strip() |
| 191 | + |
| 192 | + formats = [ |
| 193 | + "%Y-%m-%d", |
| 194 | + "%d %B %Y", |
| 195 | + "%d/%m/%Y", |
| 196 | + "%B %d, %Y", |
| 197 | + "%d %b %Y", |
| 198 | + ] |
| 199 | + |
| 200 | + for fmt in formats: |
| 201 | + try: |
| 202 | + dt = datetime.strptime(date_str, fmt) |
| 203 | + return dt.replace(tzinfo=timezone.utc) |
| 204 | + except ValueError: |
| 205 | + continue |
| 206 | + |
| 207 | + try: |
| 208 | + from dateutil import parser as dateparser |
| 209 | + |
| 210 | + dt = dateparser.parse(date_str) |
| 211 | + if dt: |
| 212 | + return dt.replace(tzinfo=timezone.utc) |
| 213 | + except Exception: |
| 214 | + pass |
| 215 | + |
| 216 | + logger.warning(f"Could not parse date: {date_str}") |
| 217 | + return None |
0 commit comments