Skip to content
Extra Tests

Add support for Ubuntu 24.04 and Fedora 40, and drop Fedora 38 #1797

Sign in to view logs

Add support for Ubuntu 24.04 and Fedora 40, and drop Fedora 38

Add support for Ubuntu 24.04 and Fedora 40, and drop Fedora 38 #1797

Workflow file for this run

.github/workflows/extra_tests.yml at 8a23b3d
# Syntax reference:
# https://help.github.com/en/actions/automating-your-workflow-with-github-actions/workflow-syntax-for-github-actions
name: Extra Tests
permissions: read-all
defaults:
run:
shell: bash
on:
pull_request:
types: [opened, synchronize]
env:
CARGO_TERM_COLOR: always
CC: 'clang'
# When changing, search-and-replace the old value with the new value below,
# particularly in `image` fields. (Excluding the "examples" job.)
#
# It'd be nice to instead use `${{ env.CONTAINER }}`, but unfortunately it
# doesn't work. The docs are a bit ambiguous - they show `env` as available
# from `container`, but doesn't explicitly say anything about
# `container.image`. We've verified experimentally that it *isn't* available
# from there.
# https://docs.github.com/en/actions/learn-github-actions/contexts#context-availability
CONTAINER: 'ubuntu:24.04'
BUILDTYPE: 'release'
DEBIAN_FRONTEND: noninteractive
jobs:
build_shadow:
env:
# used by cargo
RUSTC_WRAPPER: sccache
SCCACHE_CACHE_SIZE: 2G
SCCACHE_DIR: /home/runner/.cache/sccache
# sccache doesn't work on incrementally compiled crates, and incremental
# compilation doesn't help us here anyway.
CARGO_INCREMENTAL: false
## Increment to force eviction of previous caches, e.g. if the sccache configuration
## or usage has changed in a way to make the previous cache not useful.
## Keep in sync with same variable in `run_tests.yml`.
SCCACHE_CACHE_VERSION: 1
# runs in a container; just use the latest base image.
runs-on: ubuntu-latest
container:
# Should match env.CONTAINER.
image: ubuntu:24.04
steps:
- run: apt-get update
- name: Checkout shadow
uses: actions/checkout@v3
with:
path: shadow
# Run on PR head instead of merge result. Running on the merge
# result can give confusing results, and we require PR to be up to
# date with target branch before merging, anyway.
# See https://github.com/shadow/shadow/issues/2166
ref: ${{ github.event.pull_request.head.sha }}
persist-credentials: false
- name: Set Rust toolchain
run: ln -s shadow/ci/rust-toolchain-stable.toml rust-toolchain.toml
- name: Install dependencies
run: |
cd shadow
. ci/container_scripts/install_deps.sh
. ci/container_scripts/install_extra_deps.sh
echo "$HOME/.cargo/bin" >> $GITHUB_PATH
- name: Get month
id: get-month
run: |
echo "month=$(/bin/date -u "+%Y%m")" >> $GITHUB_OUTPUT
- name: Restore cargo registry cache
uses: actions/cache@v3
with:
path: |
~/.cargo/registry
~/.cargo/git
key: cargo-registry-${{ steps.get-month.outputs.month }}-${{ hashFiles('shadow/src/Cargo.lock') }}
restore-keys: |
cargo-registry-${{ steps.get-month.outputs.month }}
- name: Get rust version
id: get-rustv
run: |
echo rustv=\"$(rustc --version)\" >> $GITHUB_OUTPUT
- name: Restore sccache cache
uses: actions/cache@v3
continue-on-error: false
with:
path: ${{ env.SCCACHE_DIR }}
key: sccache-${{ steps.get-month.outputs.month }}-${{ steps.get-rustv.outputs.rustv }}-${{ env.CONTAINER }}-${{ env.CC }}-${{ env.BUILDTYPE }}-${{ env.SCCACHE_CACHE_VERSION}}-${{ hashFiles('shadow/src/Cargo.lock') }}
restore-keys: |
sccache-${{ steps.get-month.outputs.month }}-${{ steps.get-rustv.outputs.rustv }}-${{ env.CONTAINER }}-${{ env.CC }}-${{ env.BUILDTYPE }}-${{ env.SCCACHE_CACHE_VERSION}}-
- name: Install sccache
env:
LINK: https://github.com/mozilla/sccache/releases/download
SCCACHE_VERSION: v0.3.1
run: |
apt-get install -y curl
SCCACHE_FILE=sccache-$SCCACHE_VERSION-x86_64-unknown-linux-musl
mkdir -p $HOME/.local/bin
curl -v -L "$LINK/$SCCACHE_VERSION/$SCCACHE_FILE.tar.gz" | tar xz
mv -f $SCCACHE_FILE/sccache $HOME/.local/bin/sccache
chmod +x $HOME/.local/bin/sccache
echo "$HOME/.local/bin" >> $GITHUB_PATH
- name: Start sccache server
run: sccache --start-server
- name: Build shadow
run: |
cd shadow
. ci/container_scripts/build_and_install.sh
- name: Print sccache stats
run: sccache --show-stats
- name: Stop sccache server
run: sccache --stop-server || true
# We need to wrap in a tarball to preserve permissions.
# We're grabbing the source directory and a lot of the build
# directory here, since there are scripts and generated
# outputs we reference in other jobs.
- name: Archive shadow
run: |
TARFILE=shadow.tar
tar -cf $TARFILE --files-from /dev/null
find shadow \
-name target -prune \
-or -name '*.o' -prune \
-or -print | xargs tar --no-recursion --append -f $TARFILE
# Above command skips the whole target directories, but we
# need the shim library, which is in one.
# Add it explicitly:
tar --no-recursion --append -f $TARFILE \
shadow/build/src/shim/target/*/libshadow_shim.so
gzip $TARFILE
- uses: actions/upload-artifact@v3
with:
name: build-shadow
path: shadow.tar.gz
# This is meant primarily to be used by the other jobs in this
# workflow. We don't typically need it for debugging.
retention-days: 1
build_tgen:
# runs in a container; just use the latest base image.
runs-on: ubuntu-latest
container:
# Should match env.CONTAINER.
image: 'ubuntu:24.04'
steps:
- name: Restore cache
id: restore-tgen-build-cache
uses: actions/cache@v3
with:
path: opt/tgen
# Sync with env.CONTAINER, and with checkout ref below.
key: tgen-build-key-${{ env.CONTAINER }}-816d68cd3d0ff7d0ec71e8bbbae24ecd6a636117
- name: Checkout tgen
if: steps.restore-tgen-build-cache.outputs.cache-hit != 'true'
uses: actions/checkout@v3
with:
path: tgen
repository: shadow/tgen
# When updating, change cache key in "Restore cache" step above
ref: 816d68cd3d0ff7d0ec71e8bbbae24ecd6a636117
persist-credentials: false
- name: Install dependencies
if: steps.restore-tgen-build-cache.outputs.cache-hit != 'true'
run: |
apt-get update
apt-get install -y \
automake \
cmake \
$CC \
libevent-dev \
libssl-dev \
zlib1g-dev \
libglib2.0-dev \
libigraph-dev
- name: Build tgen
if: steps.restore-tgen-build-cache.outputs.cache-hit != 'true'
run: |
cd tgen
mkdir build && cd build
cmake .. -DCMAKE_INSTALL_PREFIX="$GITHUB_WORKSPACE/opt/tgen"
make -j$(nproc)
- name: Install tgen
if: steps.restore-tgen-build-cache.outputs.cache-hit != 'true'
run: |
cd tgen/build
make install
# We need to wrap in a tarball to preserve permissions.
- name: Archive tgen
run: tar -czf tgen.tar.gz opt/tgen
- uses: actions/upload-artifact@v3
with:
name: build-tgen
path: tgen.tar.gz
# This is meant primarily to be used by the other jobs in this
# workflow. We don't typically need it for debugging.
retention-days: 1
tor:
needs:
- build_shadow
- build_tgen
# runs in a container; just use the latest base image.
runs-on: ubuntu-latest
container:
# Should match env.CONTAINER.
image: 'ubuntu:24.04'
# the default shm-size for ubuntu:18.04, but with the size increased from
# 65536k. github's default docker seccomp policy seems to disallow
# process_vm_readv and process_vm_writev; disable it altogether. See
# https://docs.docker.com/engine/security/seccomp/
options: '--shm-size=1024g --security-opt seccomp=unconfined'
strategy:
matrix:
# when changing the tor versions, make sure to also update the
# required CI tests in the GitHub repository settings
tor: ['tor-0.4.8.10']
steps:
- run: apt-get update
- name: Download shadow
uses: actions/download-artifact@v3
with:
name: build-shadow
- name: Extract shadow
run: tar xzf shadow.tar.gz
- name: Install shadow runtime dependencies
run: apt-get install -y libglib2.0-0
- name: Download tgen
uses: actions/download-artifact@v3
with:
name: build-tgen
path: .
- name: Extract tgen
run: tar xzf tgen.tar.gz
- name: Put tgen in path
run: echo "$GITHUB_WORKSPACE/opt/tgen/bin" >> $GITHUB_PATH
- name: Install tgen runtime dependencies
run: |
apt-get install -y \
libglib2.0-dev \
libigraph-dev
- name: Restore tor build cache
id: restore-tor-build-cache
uses: actions/cache@v3
with:
path: opt/tor
# sync with env.CONTAINER
key: tor-build-keyv3-${{ env.CONTAINER }}-${{ matrix.tor }}
- name: Install tor build deps
if: steps.restore-tor-build-cache.outputs.cache-hit != 'true'
run: |
apt-get install -y \
autoconf \
automake \
$CC \
libevent-dev \
libssl-dev \
make \
zlib1g-dev \
git
- name: Checkout tor
if: steps.restore-tor-build-cache.outputs.cache-hit != 'true'
run: |
git clone https://gitlab.torproject.org/tpo/core/tor.git
cd tor
git checkout ${{ matrix.tor }}
- name: Configure, build, and install tor
if: steps.restore-tor-build-cache.outputs.cache-hit != 'true'
run: |
cd tor
./autogen.sh
./configure --disable-asciidoc --disable-unittests --prefix="$GITHUB_WORKSPACE/opt/tor"
make -j$(nproc)
make install
- name: Install tor runtime deps
run: |
apt-get install -y \
libevent-dev \
libssl-dev \
zlib1g
- name: Install obfs4proxy
run: apt-get install -y obfs4proxy
- name: Put tor in path
run: echo "$GITHUB_WORKSPACE/opt/tor/bin" >> $GITHUB_PATH
- name: Install setup script dependencies
run: apt-get install -y cmake
- name: Test
run: |
cd shadow
./setup test -- --build-config extra --label-regex tor
- name: Last 200 log lines
if: failure()
run: |
tail -n 200 shadow/build/Testing/Temporary/LastTest.log
- name: Compress logs
if: failure()
run: |
shopt -s globstar
tar -cJf shadow/build/Testing/Temporary{.tar.xz,/}
for f in shadow/build/src/test/tor/**/*.data; do tar -cJf "$f.tar.xz" "$f/"; done
- name: Upload shadow data directory
uses: actions/upload-artifact@v3
if: failure()
with:
name: ${{ matrix.tor }}-shadow-data-dir
path: shadow/build/src/test/tor/**/*.data.tar.xz
- name: Upload shadow log file
uses: actions/upload-artifact@v3
if: failure()
with:
name: ${{ matrix.tor }}-shadow-log-file
path: shadow/build/Testing/Temporary.tar.xz
tgen:
needs:
- build_shadow
- build_tgen
# runs in a container; just use the latest base image.
runs-on: ubuntu-latest
container:
# Should match env.CONTAINER.
image: 'ubuntu:24.04'
# the default shm-size for ubuntu:18.04, but with the size increased from
# 65536k. github's default docker seccomp policy seems to disallow
# process_vm_readv and process_vm_writev; disable it altogether. See
# https://docs.docker.com/engine/security/seccomp/
options: '--shm-size=1024g --security-opt seccomp=unconfined'
steps:
- run: apt-get update
- name: Download shadow
uses: actions/download-artifact@v3
with:
name: build-shadow
- name: Extract shadow
run: tar xzf shadow.tar.gz
- name: Install shadow runtime dependencies
run: apt-get install -y libglib2.0-0
- name: Download tgen
uses: actions/download-artifact@v3
with:
name: build-tgen
path: .
- name: Extract tgen
run: tar xzf tgen.tar.gz
- name: Put tgen in path
run: echo "$GITHUB_WORKSPACE/opt/tgen/bin" >> $GITHUB_PATH
- name: Install tgen runtime dependencies
run: |
apt-get install -y \
libglib2.0-dev \
libigraph-dev
- name: Install setup script dependencies
run: apt-get install -y cmake
- name: Test
run: |
cd shadow
./setup test -- --build-config extra --label-regex tgen
- name: Last 200 log lines
if: failure()
run: |
tail -n 200 shadow/build/Testing/Temporary/LastTest.log
- name: Compress logs
if: failure()
run: |
shopt -s globstar
tar -cJf shadow/build/Testing/Temporary{.tar.xz,/}
for f in shadow/build/src/test/tgen/**/*.data; do tar -cJf "$f.tar.xz" "$f/"; done
- name: Upload shadow data directory
uses: actions/upload-artifact@v3
if: failure()
with:
name: tgen-shadow-data-dir
path: shadow/build/src/test/tgen/**/*.data.tar.xz
- name: Upload shadow log file
uses: actions/upload-artifact@v3
if: failure()
with:
name: tgen-shadow-log-file
path: shadow/build/Testing/Temporary.tar.xz
examples:
needs:
- build_shadow
# runs in a container; just use the latest base image.
runs-on: ubuntu-latest
env:
# the examples only run under Ubuntu 22.04, so this job has its own
# version independent from the other jobs.
CONTAINER: 'ubuntu:22.04'
container:
# Should match env.CONTAINER above for *this* job (not the global env.CONTAINER).
image: 'ubuntu:22.04'
# the default shm-size for ubuntu:18.04, but with the size increased from
# 65536k. github's default docker seccomp policy seems to disallow
# process_vm_readv and process_vm_writev; disable it altogether. See
# https://docs.docker.com/engine/security/seccomp/
options: '--shm-size=1024g --security-opt seccomp=unconfined'
steps:
- run: apt-get update
- name: Download shadow
uses: actions/download-artifact@v3
with:
name: build-shadow
- name: Extract shadow
run: tar xzf shadow.tar.gz
- name: Install shadow runtime dependencies
run: apt-get install -y libglib2.0-0
- name: Install example dependencies
run: |
# if we need to build C code, we want to do so with the default compiler
unset CC
cd shadow
# We install some dependencies in ~/.local/bin.
# Put it on the path to faciliate checking if a dependency is already
# installed.
PATH="$PATH:$HOME/.local/bin"
# dependencies for examples
./examples/apps/curl/install_deps.sh
./examples/apps/etcd/install_deps.sh
./examples/apps/http-server/install_deps.sh
./examples/apps/iperf-2/install_deps.sh
./examples/apps/nginx/install_deps.sh
./examples/apps/wget2/install_deps.sh
./examples/docs/basic-file-transfer/install_deps.sh
./examples/docs/traffic-generation/install_deps.sh
./examples/docs/tor/install_deps.sh
- name: Install setup script dependencies
run: apt-get install -y cmake
- name: Test
run: |
cd shadow
PATH="$PATH:$HOME/.local/bin"
./setup test -- --build-config extra --label-regex example
- name: Last 200 log lines
if: failure()
run: |
tail -n 200 shadow/build/Testing/Temporary/LastTest.log
- name: Compress logs
if: failure()
run: |
shopt -s globstar
tar -cJf shadow/build/Testing/Temporary{.tar.xz,/}
for f in shadow/build/examples/**/*.data; do tar -cJf "$f.tar.xz" "$f/"; done
- name: Upload shadow data directory
uses: actions/upload-artifact@v3
if: failure()
with:
name: examples-shadow-data-dir
path: shadow/build/examples/**/*.data.tar.xz
- name: Upload shadow log file
uses: actions/upload-artifact@v3
if: failure()
with:
name: examples-shadow-log-file
path: shadow/build/Testing/Temporary.tar.xz