From f3d78c47e3316afdb49d5fb0f3fa61a70da51eb5 Mon Sep 17 00:00:00 2001 From: Colleen Murphy Date: Wed, 24 Apr 2024 11:32:36 -0700 Subject: [PATCH] Update backfill test for MySQL backend Signed-off-by: Colleen Murphy --- .github/workflows/main.yml | 8 +++- docker-compose.backfill-test.yml | 17 +++++++ tests/backfill-test.sh | 77 ++++++++++++++++++++++++-------- 3 files changed, 82 insertions(+), 20 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a66d5b7b2..5d990a3a4 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -135,11 +135,15 @@ jobs: - name: Install backfill test dependencies run: | go install ./cmd/rekor-cli - sudo apt install redis-tools -y - - name: Backfill test + sudo apt install redis-tools default-mysql-client -y + - name: Backfill test redis run: ./tests/backfill-test.sh env: INDEX_BACKEND: redis + - name: Backfill test mysql + run: ./tests/backfill-test.sh + env: + INDEX_BACKEND: mysql - name: Upload logs if they exist uses: actions/upload-artifact@1746f4ab65b179e0ea60a494b83293b640dd5bba # v4.3.2 if: failure() diff --git a/docker-compose.backfill-test.yml b/docker-compose.backfill-test.yml index fbc2301ef..f91cc1cb7 100644 --- a/docker-compose.backfill-test.yml +++ b/docker-compose.backfill-test.yml @@ -42,3 +42,20 @@ services: - "2112:2112" depends_on: - mysql + mysql: + platform: linux/amd64 + image: gcr.io/trillian-opensource-ci/db_server:v1.4.0 + environment: + - MYSQL_ROOT_PASSWORD=zaphod + - MYSQL_DATABASE=test + - MYSQL_USER=test + - MYSQL_PASSWORD=zaphod + restart: always # keep the MySQL server running + healthcheck: + test: ["CMD", "/etc/init.d/mysql", "status"] + interval: 30s + timeout: 3s + retries: 3 + start_period: 10s + ports: + - "3306:3306" diff --git a/tests/backfill-test.sh b/tests/backfill-test.sh index 627fcde44..ca85d936b 100755 --- a/tests/backfill-test.sh +++ b/tests/backfill-test.sh @@ -18,6 +18,11 @@ REKOR_ADDRESS=http://localhost:3000 REDIS_HOST=localhost REDIS_PORT=6379 REDIS_PASSWORD=test +MYSQL_HOST=127.0.0.1 +MYSQL_PORT=3306 +MYSQL_USER=test +MYSQL_PASSWORD=zaphod +MYSQL_DB=test testdir=$(mktemp -d) @@ -100,6 +105,12 @@ redis_cli() { set +e } +mysql_cli() { + set -e + mysql -h $MYSQL_HOST -P $MYSQL_PORT -u $MYSQL_USER -p${MYSQL_PASSWORD} -D $MYSQL_DB "$@" + set +e +} + remove_keys() { set -e for i in $@ ; do @@ -107,8 +118,12 @@ remove_keys() { local uuid=$(echo $rekord | jq -r .UUID) local sha=sha256:$(echo $rekord | jq -r .Body.RekordObj.data.hash.value) local key=$(echo $rekord | jq -r .Body.RekordObj.signature.publicKey.content | base64 -d | sha256sum | cut -d ' ' -f 1) - redis_cli LREM $sha 1 $uuid - redis_cli LREM $key 1 $uuid + if [ "$INDEX_BACKEND" == "redis" ] ; then + redis_cli LREM $sha 1 $uuid + redis_cli LREM $key 1 $uuid + else + mysql_cli -e "DELETE FROM EntryIndex WHERE EntryUUID = '$uuid'" + fi done set +e } @@ -152,10 +167,14 @@ check_all_entries() { done expected_uuids=($expected_uuids) local expected_length=${#expected_uuids[@]} - local actual_length=$(redis_cli LLEN sha256:${sha}) - if [ $expected_length -ne $actual_length ] ; then - echo "Possible dupicate keys for artifact $artifact." - exit 1 + # Check the values of each key for redis so we know there aren't duplicates. + # We don't need to do this for mysql, we'll just go by the total row count. + if [ "$INDEX_BACKEND" == "redis" ] ; then + local actual_length=$(redis_cli LLEN sha256:${sha}) + if [ $expected_length -ne $actual_length ] ; then + echo "Possible dupicate keys for artifact $artifact." + exit 1 + fi fi done for key in "${!expected_keys[@]}" ; do @@ -177,15 +196,27 @@ check_all_entries() { local keysha=$(echo -n $(tail -1 $key) | sha256sum | cut -d ' ' -f 1) expected_uuids=($expected_uuids) local expected_length=${#expected_uuids[@]} - local actual_length=$(redis_cli LLEN $keysha) - if [ $expected_length -ne $actual_length ] ; then - echo "Possible dupicate keys for artifact $artifact." - exit 1 + # Check the values of each key for redis so we know there aren't duplicates. + # We don't need to do this for mysql, we'll just go by the total row count. + if [ "$INDEX_BACKEND" = "redis" ] ; then + local actual_length=$(redis_cli LLEN $keysha) + if [ $expected_length -ne $actual_length ] ; then + echo "Possible dupicate keys for artifact $artifact." + exit 1 + fi fi done - local dbsize=$(redis_cli DBSIZE) - if [ $dbsize -ne 20 ] ; then - echo "Found unexpected number of index entries: $dbsize." + local expected_size + local actual_size + if [ "${INDEX_BACKEND}" == "redis" ] ; then + expected_size=20 + actual_size=$(redis_cli DBSIZE) + else + expected_size=26 + actual_size=$(mysql_cli -NB -e "SELECT COUNT(*) FROM EntryIndex;") + fi + if [ $expected_size -ne $actual_size ] ; then + echo "Found unexpected number of index entries: $actual_size." exit 1 fi set +e @@ -194,9 +225,15 @@ check_all_entries() { run_backfill() { set -e local end_index=$1 - go run cmd/backfill-index/main.go --rekor-address $REKOR_ADDRESS \ - --redis-hostname $REDIS_HOST --redis-port $REDIS_PORT --redis-password $REDIS_PASSWORD \ - --concurrency 5 --start 0 --end $end_index + if [ "$INDEX_BACKEND" == "redis" ] ; then + go run cmd/backfill-index/main.go --rekor-address $REKOR_ADDRESS \ + --redis-hostname $REDIS_HOST --redis-port $REDIS_PORT --redis-password $REDIS_PASSWORD \ + --concurrency 5 --start 0 --end $end_index + else + go run cmd/backfill-index/main.go --rekor-address $REKOR_ADDRESS \ + --mysql-dsn "${MYSQL_USER}:${MYSQL_PASSWORD}@tcp(${MYSQL_HOST}:${MYSQL_PORT})/${MYSQL_DB}" \ + --concurrency 5 --start 0 --end $end_index + fi set +e } @@ -213,8 +250,12 @@ echo echo "##### Scenario 1: backfill from scratch #####" echo -# delete all keys (including the checkpoints, but those aren't needed here) -redis_cli FLUSHALL +# delete all keys (including the checkpoints on Redis, but those aren't needed here) +if [ "$INDEX_BACKEND" == "redis" ] ; then + redis_cli FLUSHALL +else + mysql_cli -e "DELETE FROM EntryIndex;" +fi # searching for any artifact should fail search_expect_fail $testdir/blob1