Merge branch 'main' into feature/mula/rate-limit

* main:
  Fix robot test (#1420)
  Use the correct clearance level variable in organization member list template (#1427)
  Fix translation in Debian package (#1432)
  Reschedule tasks when no results in bytes are found after grace period (#1410)
  Don't scan hostname nmap in nmap boefje (#1415)
  Add and use our own CVE API (#1383)
  Add `task_id` as a query parameter to the `GET /origins` endpoint (#1414)
  Remove member group checks and check for permission instead (#1275)
  Bump cryptography from 41.0.0 to 41.0.2 in /boefjes/boefjes/plugins/kat_ssl_certificates (#1396)
  Bump cryptography from 41.0.1 to 41.0.2 in /bytes (#1397)
  Build the Debian build image on the main branch (#1387)
  Add explicit `black` config to all modules (#1395)
  Fix <no title> in the user guide docs (#1391)
  Add configurable octpoes request timeout (#1382)
  Remove hardcoded clearance level in member list for superusers (#1390)
  Add Debian build depends for CVE API package (#1384)
  Add buttons to manual rerun tasks, both boefjes or normalizers (#1339)
  Use fix multiprocessing bug on macOS where `qsize()` is not implemented (#1374)

.github/workflows/build-debian-docker-image.yml

@@ -2,10 +2,16 @@ name: Create and publish Docker image for building Debian packages
 
 on:
   workflow_dispatch: {}
-  pull_request:
+  push:
+    branches:
+      - 'main'
     paths:
       - "packaging"
       - ".github/workflows/build-debian-docker-image.yml"
+  pull_request:
+    paths:
+      - "packaging/**"
+      - ".github/workflows/build-debian-docker-image.yml"
 
 env:
   REGISTRY: ghcr.io

.github/workflows/debian_package.yml

@@ -27,6 +27,9 @@ jobs:
           bytes:
             - 'bytes/**'
             - '.github/workflows/debian_package.yml'
+          cveapi:
+            - 'cveapi/**'
+            - '.github/workflows/debian_package.yml'
           keiko:
             - 'keiko/**'
             - '.github/workflows/debian_package.yml'
@@ -49,7 +52,12 @@ jobs:
       matrix:
         dist: [debian11, debian12, ubuntu22.04]
         # On main, release branches and tags we always want to build all the packages
-        package: ${{ github.event_name == 'push' && fromJSON('["boefjes", "bytes", "keiko", "mula", "octopoes", "rocky"]') || fromJSON(needs.changes.outputs.packages) }}
+        package: ${{ github.event_name == 'push' && fromJSON('["boefjes", "bytes", "cveapi", "keiko", "mula", "octopoes", "rocky"]') || fromJSON(needs.changes.outputs.packages) }}
+        exclude:
+          - package: cveapi
+            dist: debian11
+          - package: cveapi
+            dist: ubuntu22.04
     runs-on: ubuntu-22.04
     env:
       PKG_NAME: kat-${{ matrix.package }}
@@ -89,4 +97,4 @@ jobs:
         uses: actions/upload-artifact@v3
         with:
           name: ${{env.PKG_NAME}}_${{ env.RELEASE_VERSION }}_${{ matrix.dist }}.deb
-          path: ${{matrix.package}}/build/${{env.PKG_NAME}}_${{ env.RELEASE_VERSION }}_amd64.deb
+          path: ${{matrix.package}}/build/${{env.PKG_NAME}}_${{ env.RELEASE_VERSION }}_${{ matrix.package == 'cveapi' && 'all' || 'amd64' }}.deb

.pre-commit-config.yaml

@@ -81,6 +81,7 @@ repos:
           requirements-.*.txt$ |
           retirejs.json$ |
           ^boefjes/boefjes/plugins/kat_fierce/lists |
+          ^boefjes/tests/examples/inputs/cve-result-without-cvss.json |
           ^keiko/glossaries |
           ^keiko/templates/.*/template.tex$ |
           ^rocky/assets/js/vendor |

boefjes/boefjes/app.py

@@ -3,7 +3,7 @@
 import os
 import signal
 import time
-from typing import Callable, Dict, List
+from typing import Dict, List, Tuple
 
 from pydantic import ValidationError
 from requests import HTTPError
@@ -27,26 +27,27 @@ class SchedulerWorkerManager(WorkerManager):
     def __init__(
         self,
         item_handler: Handler,
-        client_factory: Callable[[], SchedulerClientInterface],
+        scheduler_client: SchedulerClientInterface,
         settings: Settings,
         log_level: str,  # TODO: (re)move?
     ):
         self.item_handler = item_handler
-        self.client_factory = client_factory
-        self.scheduler_client = client_factory()
+        self.scheduler_client = scheduler_client
         self.settings = settings
 
-        self.task_queue = mp.Queue()
-        self.handling_tasks = mp.Manager().dict()
+        manager = mp.Manager()
+
+        self.task_queue = manager.Queue()  # multiprocessing.Queue() will not work on macOS, see mp.Queue.qsize()
+        self.handling_tasks = manager.dict()
+        self.workers = []
 
         logger.setLevel(log_level)
 
     def run(self, queue_type: WorkerManager.Queue) -> None:
         logger.info("Created worker pool for queue '%s'", queue_type.value)
 
-        self.worker_args = (self.task_queue, self.item_handler, self.client_factory, self.handling_tasks)
         self.workers = [
-            mp.Process(target=_start_working, args=self.worker_args) for _ in range(self.settings.pool_size)
+            mp.Process(target=_start_working, args=self._worker_args()) for _ in range(self.settings.pool_size)
         ]
         for worker in self.workers:
             worker.start()
@@ -132,18 +133,19 @@ def _check_workers(self) -> None:
         new_workers = []
 
         for worker in self.workers:
-            if worker.is_alive():
+            if not worker._closed and worker.is_alive():
                 new_workers.append(worker)
                 continue
 
             logger.warning(
                 "Worker[pid=%s, %s] not alive, creating new worker...", worker.pid, _format_exit_code(worker.exitcode)
             )
 
-            self._cleanup_pending_worker_task(worker)
-            worker.close()
+            if not worker._closed:  # Closed workers do not have a pid, so cleaning up would fail
+                self._cleanup_pending_worker_task(worker)
+                worker.close()
 
-            new_worker = mp.Process(target=_start_working, args=self.worker_args)
+            new_worker = mp.Process(target=_start_working, args=self._worker_args())
             new_worker.start()
             new_workers.append(new_worker)
 
@@ -170,6 +172,9 @@ def _cleanup_pending_worker_task(self, worker: mp.Process) -> None:
         except HTTPError:
             logger.exception("Could not get scheduler task[id=%s]", handling_task_id)
 
+    def _worker_args(self) -> Tuple:
+        return self.task_queue, self.item_handler, self.scheduler_client, self.handling_tasks
+
     def exit(self, queue_type: WorkerManager.Queue):
         if not self.task_queue.empty():
             items: List[QueuePrioritizedItem] = [self.task_queue.get() for _ in range(self.task_queue.qsize())]
@@ -200,10 +205,9 @@ def _format_exit_code(exitcode: int) -> str:
 def _start_working(
     task_queue: mp.Queue,
     handler: Handler,
-    client_factory: Callable[[], SchedulerClientInterface],
+    scheduler_client: SchedulerClientInterface,
     handling_tasks: Dict[int, str],
 ):
-    scheduler_client = client_factory()
     logger.info("Started listening for tasks from worker[pid=%s]", os.getpid())
 
     while True:
@@ -228,18 +232,14 @@ def _start_working(
 
 
 def get_runtime_manager(settings: Settings, queue: WorkerManager.Queue, log_level: str) -> WorkerManager:
-    # Not a lambda since multiprocessing tries and fails to pickle lambda's
-    def client_factory():
-        return SchedulerAPIClient(settings.scheduler_api)
-
     if queue is WorkerManager.Queue.BOEFJES:
         item_handler = BoefjeHandler(LocalBoefjeJobRunner(get_local_repository()), get_local_repository())
     else:
         item_handler = NormalizerHandler(LocalNormalizerJobRunner(get_local_repository()))
 
     return SchedulerWorkerManager(
         item_handler,
-        client_factory,  # Do not share a session between workers
+        SchedulerAPIClient(settings.scheduler_api),  # Do not share a session between workers
         settings,
         log_level,
     )

boefjes/boefjes/plugins/kat_cve_finding_types/boefje.json

@@ -8,7 +8,9 @@
     "produces": [
         "CVEFindingType"
     ],
-    "environment_keys": [],
+    "environment_keys": [
+        "CVEAPI_URL"
+    ],
     "scan_level": 0,
     "enabled": true
 }

boefjes/boefjes/plugins/kat_cve_finding_types/main.py

@@ -1,3 +1,4 @@
+from os import getenv
 from typing import List, Tuple, Union
 
 import requests
@@ -7,6 +8,7 @@
 
 def run(boefje_meta: BoefjeMeta) -> List[Tuple[set, Union[bytes, str]]]:
     cve_id = boefje_meta.arguments["input"]["id"]
-    response = requests.get(f"https://v1.cveapi.com/{cve_id}.json")
+    cveapi_url = getenv("CVEAPI_URL", "https://cve.openkat.dev/v1")
+    response = requests.get(f"{cveapi_url}/{cve_id}.json")
 
     return [(set(), response.content)]

boefjes/boefjes/plugins/kat_cve_finding_types/normalize.py

@@ -29,17 +29,27 @@ def run(normalizer_meta: NormalizerMeta, raw: Union[bytes, str]) -> Iterable[OOI
     cve_finding_type_id = normalizer_meta.raw_data.boefje_meta.arguments["input"]["id"]
     data = json.loads(raw)
 
-    descriptions = data["cve"]["description"]["description_data"]
+    descriptions = data["cve"]["descriptions"]
     english_description = [description for description in descriptions if description["lang"] == "en"][0]
 
-    if data["impact"] == {}:
+    if not data["cve"]["metrics"]:
         risk_severity = RiskLevelSeverity.UNKNOWN
         risk_score = None
     else:
-        try:
-            risk_score = data["impact"]["baseMetricV3"]["cvssV3"]["baseScore"]
-        except KeyError:
-            risk_score = data["impact"]["baseMetricV2"]["cvssV2"]["baseScore"]
+        metrics = data["cve"]["metrics"]
+        if "cvssMetricV31" in metrics:
+            cvss = metrics["cvssMetricV31"]
+        elif "cvssMetricV30" in metrics:
+            cvss = metrics["cvssMetricV30"]
+        else:
+            cvss = metrics["cvssMetricV20"]
+
+        for item in cvss:
+            if item["type"] == "Primary":
+                risk_score = item["cvssData"]["baseScore"]
+                break
+        else:
+            risk_score = cvss[0]["cvssData"]["baseScore"]
         risk_severity = get_risk_level(risk_score)
 
     yield CVEFindingType(

boefjes/boefjes/plugins/kat_cve_finding_types/schema.json

@@ -0,0 +1,13 @@
+{
+    "title": "Arguments",
+    "type": "object",
+    "properties": {
+        "CVEAPI_URL": {
+            "title": "CVEAPI_URL",
+            "maxLength": 2048,
+            "type": "string",
+            "description": "URL of the CVE API, defaults to https://cve.openkat.dev/v1",
+            "default": "https://cve.openkat.dev/v1"
+        }
+    }
+}

boefjes/boefjes/plugins/kat_nmap/main.py

@@ -28,7 +28,6 @@ def build_nmap_arguments(host: str, protocol: Protocol, top_ports: Optional[int]
     """Returns Nmap arguments to use based on protocol and top_ports for host."""
     ip = ip_address(host)
     args = [
-        "nmap",
         "--open",
         "-T4",
         "-Pn",

boefjes/boefjes/plugins/kat_ssl_certificates/requirements.txt

@@ -1 +1 @@
-cryptography==41.0.0
+cryptography==41.0.2

boefjes/pyproject.toml

@@ -40,3 +40,7 @@ build-backend = "setuptools.build_meta:__legacy__"
 [tool.flynt]
 line-length = 120
 transform-concats = true
+
+[tool.black]
+target-version = ["py38", "py39", "py310", "py311"]
+line-length = 120

boefjes/tests/conftest.py

@@ -1,7 +1,7 @@
 import multiprocessing
 import time
 from datetime import datetime, timezone
-from multiprocessing import Queue as MultiprocessingQueue
+from multiprocessing import Manager
 from pathlib import Path
 from typing import Dict, List, Optional, Tuple, Union
 
@@ -15,10 +15,6 @@
 from boefjes.runtime_interfaces import Handler, WorkerManager
 from tests.stubs import get_dummy_data
 
-_tasks = multiprocessing.Manager().dict()
-_popped_items = multiprocessing.Manager().dict()
-_pushed_items = multiprocessing.Manager().dict()
-
 
 class MockSchedulerClient(SchedulerClientInterface):
     def __init__(
@@ -29,7 +25,7 @@ def __init__(
         log_path: Path,
         raise_on_empty_queue: Exception = KeyboardInterrupt,
         iterations_to_wait_for_exception: int = 0,
-        sleep_time: int = 0.05,
+        sleep_time: int = 0.1,
     ):
         self.queue_response = queue_response
         self.boefje_responses = boefje_responses
@@ -40,11 +36,12 @@ def __init__(
         self.sleep_time = sleep_time
 
         self._iterations = 0
-        self._tasks: Dict[str, Task] = _tasks
-        self._popped_items: Dict[str, QueuePrioritizedItem] = _popped_items
-        self._pushed_items: Dict[str, Tuple[str, QueuePrioritizedItem]] = _pushed_items
+        self._tasks: Dict[str, Task] = multiprocessing.Manager().dict()
+        self._popped_items: Dict[str, QueuePrioritizedItem] = multiprocessing.Manager().dict()
+        self._pushed_items: Dict[str, Tuple[str, QueuePrioritizedItem]] = multiprocessing.Manager().dict()
 
     def get_queues(self) -> List[Queue]:
+        time.sleep(self.sleep_time)
         return parse_raw_as(List[Queue], self.queue_response)
 
     def pop_item(self, queue: str) -> Optional[QueuePrioritizedItem]:
@@ -97,7 +94,7 @@ def push_item(self, queue_id: str, p_item: QueuePrioritizedItem) -> None:
 class MockHandler(Handler):
     def __init__(self, exception=Exception):
         self.sleep_time = 0
-        self.queue = MultiprocessingQueue()
+        self.queue = Manager().Queue()
         self.exception = exception
 
     def handle(self, item: Union[BoefjeMeta, NormalizerMeta]):
@@ -118,13 +115,11 @@ def item_handler(tmp_path: Path):
 
 @pytest.fixture
 def manager(item_handler: MockHandler, tmp_path: Path) -> SchedulerWorkerManager:
-    def client_factory():
-        return MockSchedulerClient(
-            get_dummy_data("scheduler/queues_response.json"),
-            2 * [get_dummy_data("scheduler/pop_response_boefje.json")]
-            + [get_dummy_data("scheduler/should_crash.json")],
-            [get_dummy_data("scheduler/pop_response_normalizer.json")],
-            tmp_path / "patch_task_log",
-        )
-
-    return SchedulerWorkerManager(item_handler, client_factory, Settings(pool_size=1, poll_interval=0.01), "DEBUG")
+    scheduler_client = MockSchedulerClient(
+        get_dummy_data("scheduler/queues_response.json"),
+        2 * [get_dummy_data("scheduler/pop_response_boefje.json")] + [get_dummy_data("scheduler/should_crash.json")],
+        [get_dummy_data("scheduler/pop_response_normalizer.json")],
+        tmp_path / "patch_task_log",
+    )
+
+    return SchedulerWorkerManager(item_handler, scheduler_client, Settings(pool_size=1, poll_interval=0.01), "DEBUG")