Update backfill test for MySQL backend

Signed-off-by: Colleen Murphy <colleenmurphy@google.com>

.github/workflows/main.yml

@@ -135,11 +135,15 @@ jobs:
       - name: Install backfill test dependencies
         run: |
           go install ./cmd/rekor-cli
-          sudo apt install redis-tools -y
-      - name: Backfill test
+          sudo apt install redis-tools default-mysql-client -y
+      - name: Backfill test redis
         run: ./tests/backfill-test.sh
         env:
           INDEX_BACKEND: redis
+      - name: Backfill test mysql
+        run: ./tests/backfill-test.sh
+        env:
+          INDEX_BACKEND: mysql
       - name: Upload logs if they exist
         uses: actions/upload-artifact@1746f4ab65b179e0ea60a494b83293b640dd5bba # v4.3.2
         if: failure()

docker-compose.backfill-test.yml

@@ -42,3 +42,20 @@ services:
       - "2112:2112"
     depends_on:
       - mysql
+  mysql:
+    platform: linux/amd64
+    image: gcr.io/trillian-opensource-ci/db_server:v1.4.0
+    environment:
+      - MYSQL_ROOT_PASSWORD=zaphod
+      - MYSQL_DATABASE=test
+      - MYSQL_USER=test
+      - MYSQL_PASSWORD=zaphod
+    restart: always # keep the MySQL server running
+    healthcheck:
+      test: ["CMD", "/etc/init.d/mysql", "status"]
+      interval: 30s
+      timeout: 3s
+      retries: 3
+      start_period: 10s
+    ports:
+      - "3306:3306"

tests/backfill-test.sh

@@ -18,6 +18,11 @@ REKOR_ADDRESS=http://localhost:3000
 REDIS_HOST=localhost
 REDIS_PORT=6379
 REDIS_PASSWORD=test
+MYSQL_HOST=127.0.0.1
+MYSQL_PORT=3306
+MYSQL_USER=test
+MYSQL_PASSWORD=zaphod
+MYSQL_DB=test
 
 testdir=$(mktemp -d)
 
@@ -100,15 +105,25 @@ redis_cli() {
     set +e
 }
 
+mysql_cli() {
+    set -e
+    mysql -h $MYSQL_HOST -P $MYSQL_PORT -u $MYSQL_USER -p${MYSQL_PASSWORD} -D $MYSQL_DB "$@"
+    set +e
+}
+
 remove_keys() {
     set -e
     for i in $@ ; do
         local rekord=$(rekor-cli --rekor_server $REKOR_ADDRESS get --log-index $i --format json)
         local uuid=$(echo $rekord | jq -r .UUID)
         local sha=sha256:$(echo $rekord | jq -r .Body.RekordObj.data.hash.value)
         local key=$(echo $rekord | jq -r .Body.RekordObj.signature.publicKey.content | base64 -d | sha256sum | cut -d ' ' -f 1)
-        redis_cli LREM $sha 1 $uuid
-        redis_cli LREM $key 1 $uuid
+        if [ "$INDEX_BACKEND" == "redis" ] ; then
+            redis_cli LREM $sha 1 $uuid
+            redis_cli LREM $key 1 $uuid
+        else
+            mysql_cli -e "DELETE FROM EntryIndex WHERE EntryUUID = '$uuid'"
+        fi
     done
     set +e
 }
@@ -152,10 +167,14 @@ check_all_entries() {
         done
         expected_uuids=($expected_uuids)
         local expected_length=${#expected_uuids[@]}
-        local actual_length=$(redis_cli LLEN sha256:${sha})
-        if [ $expected_length -ne $actual_length ] ; then
-            echo "Possible dupicate keys for artifact $artifact."
-            exit 1
+        # Check the values of each key for redis so we know there aren't duplicates.
+        # We don't need to do this for mysql, we'll just go by the total row count.
+        if [ "$INDEX_BACKEND" == "redis" ] ; then
+            local actual_length=$(redis_cli LLEN sha256:${sha})
+            if [ $expected_length -ne $actual_length ] ; then
+                echo "Possible dupicate keys for artifact $artifact."
+                exit 1
+            fi
         fi
     done
     for key in "${!expected_keys[@]}" ; do
@@ -177,15 +196,27 @@ check_all_entries() {
         local keysha=$(echo -n $(tail -1 $key) | sha256sum | cut -d ' ' -f 1)
         expected_uuids=($expected_uuids)
         local expected_length=${#expected_uuids[@]}
-        local actual_length=$(redis_cli LLEN $keysha)
-        if [ $expected_length -ne $actual_length ] ; then
-            echo "Possible dupicate keys for artifact $artifact."
-            exit 1
+        # Check the values of each key for redis so we know there aren't duplicates.
+        # We don't need to do this for mysql, we'll just go by the total row count.
+        if [ "$INDEX_BACKEND" = "redis" ] ; then
+            local actual_length=$(redis_cli LLEN $keysha)
+            if [ $expected_length -ne $actual_length ] ; then
+                echo "Possible dupicate keys for artifact $artifact."
+                exit 1
+            fi
         fi
     done
-    local dbsize=$(redis_cli DBSIZE)
-    if [ $dbsize -ne 20 ] ; then
-        echo "Found unexpected number of index entries: $dbsize."
+    local expected_size
+    local actual_size
+    if [ "${INDEX_BACKEND}" == "redis" ] ; then
+        expected_size=20
+        actual_size=$(redis_cli DBSIZE)
+    else
+        expected_size=26
+        actual_size=$(mysql_cli -NB -e "SELECT COUNT(*) FROM EntryIndex;")
+    fi
+    if [ $expected_size -ne $actual_size ] ; then
+        echo "Found unexpected number of index entries: $actual_size."
         exit 1
     fi
     set +e
@@ -194,9 +225,15 @@ check_all_entries() {
 run_backfill() {
     set -e
     local end_index=$1
-    go run cmd/backfill-index/main.go --rekor-address $REKOR_ADDRESS \
-        --redis-hostname $REDIS_HOST --redis-port $REDIS_PORT --redis-password $REDIS_PASSWORD \
-        --concurrency 5 --start 0 --end $end_index
+    if [ "$INDEX_BACKEND" == "redis" ] ; then
+        go run cmd/backfill-index/main.go --rekor-address $REKOR_ADDRESS \
+            --redis-hostname $REDIS_HOST --redis-port $REDIS_PORT --redis-password $REDIS_PASSWORD \
+            --concurrency 5 --start 0 --end $end_index
+    else
+        go run cmd/backfill-index/main.go --rekor-address $REKOR_ADDRESS \
+            --mysql-dsn "${MYSQL_USER}:${MYSQL_PASSWORD}@tcp(${MYSQL_HOST}:${MYSQL_PORT})/${MYSQL_DB}" \
+            --concurrency 5 --start 0 --end $end_index
+    fi
     set +e
 }
 
@@ -213,8 +250,12 @@ echo
 echo "##### Scenario 1: backfill from scratch #####"
 echo
 
-# delete all keys (including the checkpoints, but those aren't needed here)
-redis_cli FLUSHALL
+# delete all keys (including the checkpoints on Redis, but those aren't needed here)
+if [ "$INDEX_BACKEND" == "redis" ] ; then
+    redis_cli FLUSHALL
+else
+    mysql_cli -e "DELETE FROM EntryIndex;"
+fi
 
 # searching for any artifact should fail
 search_expect_fail $testdir/blob1