Skip to content

Commit

Permalink
Merge branch 'main' into feature/mula/rescheduling
Browse files Browse the repository at this point in the history
* main: (95 commits)
  Translations for release 1.11 - EN -> NL, PAP (#1439)
  Add Question ooi model and create the first bit that generates a question (#1407)
  make port classification configurable (#1418)
  KATalogus API filtering and pagination (#1405)
  Fix robot test (#1420)
  Use the correct clearance level variable in organization member list template (#1427)
  Fix translation in Debian package (#1432)
  Reschedule tasks when no results in bytes are found after grace period (#1410)
  Don't scan hostname nmap in nmap boefje (#1415)
  Add and use our own CVE API (#1383)
  Add `task_id` as a query parameter to the `GET /origins` endpoint (#1414)
  Remove member group checks and check for permission instead (#1275)
  Bump cryptography from 41.0.0 to 41.0.2 in /boefjes/boefjes/plugins/kat_ssl_certificates (#1396)
  Bump cryptography from 41.0.1 to 41.0.2 in /bytes (#1397)
  Build the Debian build image on the main branch (#1387)
  Add explicit `black` config to all modules (#1395)
  Fix <no title> in the user guide docs (#1391)
  Add configurable octpoes request timeout (#1382)
  Remove hardcoded clearance level in member list for superusers (#1390)
  Add Debian build depends for CVE API package (#1384)
  ...
  • Loading branch information
jpbruinsslot committed Jul 25, 2023
2 parents ed1a4a0 + e7646eb commit 3607738
Show file tree
Hide file tree
Showing 483 changed files with 235,195 additions and 12,043 deletions.
10 changes: 6 additions & 4 deletions .env-dist
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,14 @@ DATABASE_MIGRATION=true
# If you use docker-compose.release-example.yml as base for a docker compose
# based setup you can use this variable to define the version of OpenKAT you
# want to use.
#KAT_VERSION=
# KAT_VERSION=

# Rocky
SECRET_KEY={%SECRET_KEY}
DEBUG=True
TWOFACTOR_ENABLED=True
DJANGO_ALLOWED_HOSTS=127.0.0.1 localhost
DJANGO_CSRF_TRUSTED_ORIGINS=http://localhost http://127.0.0.1
DJANGO_ALLOWED_HOSTS=127.0.0.1,localhost
DJANGO_CSRF_TRUSTED_ORIGINS=http://localhost,http://127.0.0.1
DJANGO_SUPERUSER_EMAIL=superuser@localhost
DJANGO_SUPERUSER_FULL_NAME="KAT Superuser"
DJANGO_SUPERUSER_PASSWORD={%DJANGO_SUPERUSER_PASSWORD}
Expand All @@ -26,12 +26,14 @@ ROCKY_DB_PASSWORD={%ROCKY_DB_PASSWORD}
ROCKY_DB_HOST=postgres
ROCKY_DB_PORT=5432
ROCKY_DB_USER_CREATEDB=CREATEDB
# ROCKY_DB_DSN=

# Queueing
RABBITMQ_DEFAULT_VHOST=kat
RABBITMQ_DEFAULT_USER={%QUEUE_USERNAME}
RABBITMQ_DEFAULT_PASS={%QUEUE_PASSWORD}

# Boefjes and Normalizers
QUEUE_NAME_BOEFJES=boefjes
QUEUE_NAME_NORMALIZERS=normalizers
QUEUE_URI=amqp://${RABBITMQ_DEFAULT_USER}:${RABBITMQ_DEFAULT_PASS}@rabbitmq:5672/${RABBITMQ_DEFAULT_VHOST}
Expand Down Expand Up @@ -95,4 +97,4 @@ KEIKO_REPORTS_FOLDER=/reports
KEIKO_ASSETS_FOLDER=assets

# OpenTracing
#SPAN_EXPORT_GRPC_ENDPOINT=http://jaeger:4317
# SPAN_EXPORT_GRPC_ENDPOINT=http://jaeger:4317
16 changes: 16 additions & 0 deletions .github/scripts/commit_sign_push.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
#!/bin/bash

#GITHUB_TOKEN should be ${{ secrets.GITHUB_TOKEN }}
#DESTINATION_BRANCH should be ${{ github.ref }}

FILES=$(git diff --name-only)
for FILE in $FILES; do
CONTENT=$(base64 -i "$FILE")
SHA=$(git rev-parse "$DESTINATION_BRANCH":"$FILE")
gh api --method PUT /repos/:owner/:repo/contents/"$FILE" \
--field message="Update $FILE" \
--field content="$CONTENT" \
--field encoding="base64" \
--field branch="$DESTINATION_BRANCH" \
--field sha="$SHA"
done
54 changes: 0 additions & 54 deletions .github/workflows/boefjes_debian_package.yml

This file was deleted.

8 changes: 7 additions & 1 deletion .github/workflows/build-debian-docker-image.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,16 @@ name: Create and publish Docker image for building Debian packages

on:
workflow_dispatch: {}
pull_request:
push:
branches:
- 'main'
paths:
- "packaging"
- ".github/workflows/build-debian-docker-image.yml"
pull_request:
paths:
- "packaging/**"
- ".github/workflows/build-debian-docker-image.yml"

env:
REGISTRY: ghcr.io
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/build_docs_on_pr.yml
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ jobs:

- uses: actions/setup-python@v4
with:
python-version: '3.8'
python-version: 3.11
cache: 'pip' # caching pip dependencies

- name: Install pip dependencies
Expand Down
50 changes: 0 additions & 50 deletions .github/workflows/bytes_debian_package.yml

This file was deleted.

52 changes: 52 additions & 0 deletions .github/workflows/check_poetry_dependencies.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
name: Check Poetry dependencies

on:
push:
paths:
- '**/requirements.txt'
- '**/requirements-dev.txt'
- '**/pyproject.toml'
- '**/poetry.lock'
pull_request:
paths:
- '**/requirements.txt'
- '**/requirements-dev.txt'
- '**/pyproject.toml'
- '**/poetry.lock'


jobs:
poetry-dependencies:

permissions:
# Give the default GITHUB_TOKEN write permission to commit and push the
# added or changed files to the repository.
contents: write

runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3

- name: Set up Python
uses: actions/setup-python@v4
id: py311
with:
python-version: 3.11
cache: pip

- name: Install pip dependencies
run: pip install poetry

- name: Check, lock, and export Poetry dependencies
run: make poetry-dependencies

- name: Check if there are any changed files
if: ${{ github.actor != 'dependabot[bot]' }}
run: git diff --exit-code

- name: Commit, sign, and push changes
if: ${{ github.actor == 'dependabot[bot]' }}
run: sh .github/scripts/commit_sign_push.sh
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
DESTINATION_BRANCH: ${{ github.ref }}
100 changes: 100 additions & 0 deletions .github/workflows/debian_package.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
name: Debian packages

on:
pull_request:
push:
branches:
- 'main'
- 'release-*'
tags:
- '*'

jobs:
changes:
runs-on: ubuntu-22.04
outputs:
packages: ${{ steps.filter.outputs.changes }}
steps:
- uses: dorny/paths-filter@v2
if: github.event_name != 'push'
id: filter
with:
filters: |
boefjes:
- 'boefjes/**'
- 'octopoes/**'
- '.github/workflows/debian_package.yml'
bytes:
- 'bytes/**'
- '.github/workflows/debian_package.yml'
cveapi:
- 'cveapi/**'
- '.github/workflows/debian_package.yml'
keiko:
- 'keiko/**'
- '.github/workflows/debian_package.yml'
mula:
- 'mula/**'
- '.github/workflows/debian_package.yml'
octopoes:
- 'octopoes/**'
- '.github/workflows/debian_package.yml'
rocky:
- 'rocky/**'
- 'octopoes/**'
- '.github/workflows/debian_package.yml'
build:
needs: changes
if: ${{ github.event_name == 'push' || (needs.changes.outputs.packages != '[]' && needs.changes.outputs.packages != '') }}
strategy:
fail-fast: false
matrix:
dist: [debian11, debian12, ubuntu22.04]
# On main, release branches and tags we always want to build all the packages
package: ${{ github.event_name == 'push' && fromJSON('["boefjes", "bytes", "cveapi", "keiko", "mula", "octopoes", "rocky"]') || fromJSON(needs.changes.outputs.packages) }}
exclude:
- package: cveapi
dist: debian11
- package: cveapi
dist: ubuntu22.04
runs-on: ubuntu-22.04
env:
PKG_NAME: kat-${{ matrix.package }}

steps:
- uses: actions/checkout@v3

- name: Generate version.py and set RELEASE_VERSION
run: |
pip install setuptools-scm==7.1.0
echo "RELEASE_VERSION=$(python -m setuptools_scm | sed s/rc/~rc/)" >> $GITHUB_ENV
cp _version.py boefjes/boefjes/version.py
cp _version.py boefjes/boefjes/katalogus/version.py
cp _version.py bytes/bytes/version.py
cp _version.py keiko/keiko/version.py
cp _version.py mula/scheduler/version.py
cp _version.py octopoes/octopoes/version.py
cp _version.py rocky/rocky/version.py
- name: Run debian package build
uses: addnab/docker-run-action@v3
with:
run: packaging/scripts/build-debian-package.sh
registry: ghcr.io
image: ghcr.io/minvws/nl-kat-${{ matrix.dist }}-build-image:latest
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
options: -v ${{ github.workspace }}/${{ matrix.package }}:/app
-v ${{ github.workspace }}/octopoes:/octopoes
-e REPOSITORY=${{ github.repository }}
-e RELEASE_VERSION=${{ env.RELEASE_VERSION }}
-e RELEASE_TAG=${{ env.RELEASE_TAG }}
-e PKG_NAME=${{ env.PKG_NAME }}
--workdir /app

- name: Upload .deb to artifacts
uses: actions/upload-artifact@v3
with:
name: ${{env.PKG_NAME}}_${{ env.RELEASE_VERSION }}_${{ matrix.dist }}.deb
path: ${{matrix.package}}/build/${{env.PKG_NAME}}_${{ env.RELEASE_VERSION }}_${{ matrix.package == 'cveapi' && 'all' || 'amd64' }}.deb
2 changes: 1 addition & 1 deletion .github/workflows/deploy_docs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ jobs:

- uses: actions/setup-python@v4
with:
python-version: '3.8'
python-version: 3.11
cache: 'pip' # caching pip dependencies

- name: Install pip dependencies
Expand Down

0 comments on commit 3607738

Please sign in to comment.