E2E testing for local images

Makefile

@@ -1,36 +1,43 @@
 # This Makefile contains common development / build commands for Package Analysis. For everything to work properly, it needs to be kept in the top-level project directory.
 
-REGISTRY := gcr.io/ossf-malware-analysis
-
+# Get absolute path to top-level package analysis project directory
 # outermost abspath removes the trailing slash from the directory path
 PREFIX := $(abspath $(dir $(abspath $(lastword $(MAKEFILE_LIST)))))
 SANDBOX_DIR := $(PREFIX)/sandboxes
 
-#
-# This is just the old 'build everything script'
-#
-.PHONY: legacy_build_docker
-legacy_build_docker:
-	bash build/build_docker.sh
+# Registry for Docker images built and used by package analysis
+REGISTRY := gcr.io/ossf-malware-analysis
+
+# If RELEASE_TAG environment variable is not specified, images will be tagged
+# as 'latest' which is equivalent to just tagging without specifying a version
+TAG := ${RELEASE_TAG}
+ifeq ($(TAG), )
+	TAG := latest
+	BUILD_ARG=
+else
+	# pass tag into analysis image build
+	BUILD_ARG=--build-arg=SANDBOX_IMAGE_TAG=$(TAG)
+endif
 
 .PHONY: all
 all: docker_build_all
 
 #
 # These recipes build all the top-level docker images
-# TODO grab release tag from env vars
 
 docker_build_%_image:
-	docker build -t ${REGISTRY}/$(IMAGE_NAME) -f $(DOCKERFILE) $(DIR)
+	@# if TAG is 'latest', the two -t arguments are equivalent and do the same thing
+	docker build $(BUILD_ARG) -t ${REGISTRY}/$(IMAGE_NAME) -t ${REGISTRY}/$(IMAGE_NAME):$(TAG) -f $(DOCKERFILE) $(DIR)
 
 #
 # These build the sandbox images and also update (sync) them locally
 # from Docker to podman. This is needed for local analyses; in order
 # to use these updated images, pass 'nopull' to run_analysis.sh
 #
 docker_build_%_sandbox:
-	docker build -t ${REGISTRY}/$(IMAGE_NAME) -f $(DOCKERFILE) $(DIR)
-	sudo buildah pull docker-daemon:${REGISTRY}/${IMAGE_NAME}:latest
+	@# if TAG is 'latest', the two -t arguments are equivalent and do the same thing
+	docker build -t ${REGISTRY}/$(IMAGE_NAME) -t ${REGISTRY}/$(IMAGE_NAME):$(TAG) -f $(DOCKERFILE) $(DIR)
+	sudo buildah pull docker-daemon:${REGISTRY}/${IMAGE_NAME}:$(TAG)
 
 docker_build_analysis_image: DIR=$(PREFIX)
 docker_build_analysis_image: DOCKERFILE=$(PREFIX)/cmd/analyze/Dockerfile
@@ -70,6 +77,37 @@ docker_build_all_sandboxes: docker_build_node_sandbox docker_build_python_sandbo
 .PHONY: docker_build_all
 docker_build_all: docker_build_all_sandboxes docker_build_analysis_image docker_build_scheduler_image
 
+#
+# Builds then pushes analysis and sandbox images
+#
+
+docker_push_%:
+	docker push --all-tags ${REGISTRY}/$(IMAGE_NAME)
+
+docker_push_analysis_image: docker_build_analysis_image
+docker_push_scheduler_image: docker_build_scheduler_image
+docker_push_node_sandbox: docker_build_node_sandbox
+docker_push_python_sandbox: docker_build_python_sandbox
+docker_push_ruby_sandbox: docker_build_ruby_sandbox
+docker_push_packagist_sandbox: docker_build_packagist_sandbox
+docker_push_crates_sandbox: docker_build_crates_sandbox
+docker_push_static_analysis_sandbox: docker_build_static_analysis_sandbox
+
+docker_push_analysis_image: IMAGE_NAME=analysis
+docker_push_scheduler_image: IMAGE_NAME=scheduler
+docker_push_node_sandbox: IMAGE_NAME=node
+docker_push_python_sandbox: IMAGE_NAME=python
+docker_push_ruby_sandbox: IMAGE_NAME=ruby
+docker_push_packagist_sandbox:	IMAGE_NAME=packagist
+docker_push_crates_sandbox: IMAGE_NAME=crates.io
+docker_push_static_analysis_sandbox: IMAGE_NAME=static-analysis
+
+.PHONY: docker_push_all_sandboxes
+docker_push_all_sandboxes: docker_push_node_sandbox docker_push_python_sandbox docker_push_ruby_sandbox docker_push_packagist_sandbox docker_push_crates_sandbox docker_push_static_analysis_sandbox
+
+.PHONY: docker_push_all
+docker_push_all: docker_push_all_sandboxes docker_push_analysis_image docker_push_scheduler_image
+
 
 #
 # This runs a lint check on all shell scripts in the repo
@@ -87,23 +125,39 @@ run:
 # These recipes control docker-compose, which is used for
 # end-to-end testing of the complete scheduler/worker system
 #
-.PHONY: docker_compose_start
-docker_compose_start:
-	cd ./examples/e2e && docker-compose up -d
-	sleep 3
-	curl localhost:8080
+
+E2E_TEST_COMPOSE_ARGS := -p pa-e2e-testing -f ./configs/e2e/docker-compose.yml -f ./test/e2e/docker-compose.test.yml
+
+.PHONY: e2e_test_start
+e2e_test_start:
+	docker-compose $(E2E_TEST_COMPOSE_ARGS) up -d
 	@echo
 	@echo "To see analysis results, go to http://localhost:9000/minio/package-analysis"
-	@echo "Remember to run `make docker_compose_stop` when done!"
+	@echo
+	@echo "Remember to run 'make e2e_test_stop' when done!"
+	@sleep 5
+	@echo
+	curl localhost:8080
+
+.PHONY: e2e_test_stop
+e2e_test_stop:
+	docker-compose $(E2E_TEST_COMPOSE_ARGS) down
+
+.PHONY: e2e_test_logs_all
+e2e_test_logs_all:
+	docker-compose $(E2E_TEST_COMPOSE_ARGS) logs
 
-.PHONY: docker_compose_logs
-docker_compose_logs:
-	cd ./examples/e2e && docker-compose logs
+.PHONY: e2e_test_logs_feeds
+e2e_test_logs_feeds:
+	docker-compose $(E2E_TEST_COMPOSE_ARGS) logs -f feeds
 
-.PHONY: docker_compose_stop
-docker_compose_stop:
-	cd ./examples/e2e && docker-compose down
+.PHONY: e2e_test_logs_scheduler
+e2e_test_logs_scheduler:
+	docker-compose $(E2E_TEST_COMPOSE_ARGS) logs -f scheduler
 
+.PHONY: e2e_test_logs_analysis
+e2e_test_logs_analysis:
+	docker-compose $(E2E_TEST_COMPOSE_ARGS) logs -f analysis
 
 .PHONY: test
 test:

examples/e2e/docker-compose.yml → configs/compose-e2e/docker-compose.yml

@@ -33,7 +33,7 @@ services:
       MINIO_ROOT_PASSWORD: minio123
       MINIO_REGION_NAME: dummy_region
     entrypoint: sh
-    command: -c 'mkdir -p /data/{package-analysis,package-analysis-file-writes} && /usr/bin/minio server /data'
+    command: -c 'mkdir -p /data/package-analysis{,-static,-file-writes} && /usr/bin/minio server /data'
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
       interval: 30s
@@ -48,6 +48,7 @@ services:
       OSSMALWARE_WORKER_SUBSCRIPTION: kafka://worker?topic=workers
       OSSF_MALWARE_NOTIFICATION_TOPIC: kafka://notifications
       OSSF_MALWARE_ANALYSIS_RESULTS: s3://package-analysis?endpoint=minio:9000&disableSSL=true&s3ForcePathStyle=true
+      OSSF_MALWARE_STATIC_ANALYSIS_RESULTS: s3://package-analysis-static?endpoint=minio:9000&disableSSL=true&s3ForcePathStyle=true
       OSSF_MALWARE_ANALYSIS_FILE_WRITE_RESULTS: s3://package-analysis-file-writes?endpoint=minio:9000&disableSSL=true&s3ForcePathStyle=true
       KAFKA_BROKERS: kafka:9092
       AWS_ACCESS_KEY_ID: minio
@@ -65,7 +66,7 @@ services:
       OSSMALWARE_WORKER_TOPIC: kafka://workers
       OSSMALWARE_SUBSCRIPTION_URL: kafka://worker?topic=package-feeds
       KAFKA_BROKERS: kafka:9092
-  
+
   feeds:
     restart: "on-failure"
     image: gcr.io/ossf-malware-analysis/scheduled-feeds:latest

examples/e2e/README.md

@@ -4,9 +4,10 @@ This example provides a simple way to spin up an end to end deployment of packag
 
 ## Running
 
+# Run the following commands from the top-level package analysis directory
+
 ```shell
-$ cd examples/e2e # must be run from the e2e folder
-$ docker-compose up -d
+$ docker-compose -f configs/e2e/docker-compose.yml up -d
 $ curl localhost:8080
 ```
 
@@ -24,11 +25,11 @@ using the following credentials for authentication:
 
 ## Logs Access
 
-`docker-compose logs -f feeds` will provide information on the packages which have been send downstream.
+`docker-compose -f configs/e2e/docker-compose.yml logs -f feeds` will provide information on the packages which have been send downstream.
 
-`docker-compose logs -f scheduler` will provide information on the packages which have been received and proxied onto the analysis workers.
+`docker-compose -f configs/e2e/docker-compose.yml logs -f scheduler` will provide information on the packages which have been received and proxied onto the analysis workers.
 
-`docker-compose logs -f analysis` provides too much stdout to be useful, outputs are sent to minio as mentioned above.
+`docker-compose -f configs/e2e/docker-compose.yml logs -f analysis` provides too much stdout to be useful, outputs are sent to minio as mentioned above.
 
 ## PubSub (Kafka) Inspection
 
@@ -45,17 +46,17 @@ Output from the Kafka PubSub topics can be inspected using
 
 ### Limitations
 
-- Locally built sandbox images are currently ignored.
+- Locally built sandbox images are ignored (see test/e2e directory for how to use local images)
 
 ### Feeds does not start (missing config)
 
-This can happen if `./config` is not world-readable. You will see the error message `open /config/feeds.yml: permission denied` in the feeds logs.
+This can happen if `configs/e2e/config` is not world-readable. You will see the error message `open /config/feeds.yml: permission denied` in the feeds logs.
 
-To fix simply run:
+To fix simply run the following commands from the top-level package analysis directory:
 
 ```shell
-$ chmod ugo+rx ./config
-$ chmod ugo+r ./config/feeds.yml
+$ chmod ugo+rx configs/e2e/config
+$ chmod ugo+r configs/e2e/config/feeds.yml
 ```
 
 ### Sandbox container is not starting (cgroups v2)
@@ -86,9 +87,8 @@ To build the necessary images yourself for the docker-compose, you can do the fo
 
 ```
 # In package-analysis
-cd build
-./build_docker.sh
+make docker_build_all
 
 # In package-feeds
 docker build . -t gcr.io/ossf-malware-analysis/scheduled-feeds:latest
-```
+```

test/e2e/README.md

@@ -0,0 +1,91 @@
+# End to End Testing with Package-Feeds integration
+
+This directory helps run end-to-end tests of the package analysis system
+to ensure everything is working properly.
+In particular, local changes to both the worker/analysis and sandbox images can be tested
+before they are pushed to the docker registry.
+
+The test is orchestrated using docker-compose, using a setup adapted from the end-to-end
+example under `examples/e2e`. However, all the commands can be run via the project Makefile.
+
+## Running
+
+### Starting the test
+
+In the top-level project directory, run
+
+```shell
+$ make RELEASE_TAG=test docker_build_all # rebuild images with 'test' tag
+$ make e2e_test_start
+
+```
+
+### Stopping the test
+
+In the top-level project directory, run
+
+```shell
+$ make e2e_test_stop
+```
+
+## Analysis Output
+
+Output can be found at http://localhost:9000/minio/package-analysis,
+using the following credentials for authentication:
+
+- username: `minio`
+- password: `minio123`
+
+## Logs Access
+
+In the top-level project directory, run
+
+`make e2e_test_logs_feeds` to see information on the packages which have been send downstream.
+
+`make e2e_test_logs_scheduler` to see information on the packages which have been received and proxied onto the analysis workers.
+
+`make e2e_tests_logs_analysis` to see analysis stdout (too much to be useful); better to check minio output as described above.
+
+## PubSub (Kafka) Inspection
+
+Output from the Kafka PubSub topics can be inspected using
+[KafkaCat](https://github.com/edenhill/kcat).
+
+1. Install `kafkacat` or `kcat` (e.g. `sudo apt install kafkacat`)
+2. Run `kafkacat` to observe the topics:
+    - package-feeds: `kafkacat -C -J -b localhost:9094 -t package-feeds`
+    - workers: `kafkacat -C -J -b localhost:9094 -t workers`
+    - notifications: `kafkacat -C -J -b localhost:9094 -t notifications`
+
+## Troubleshooting
+
+### Feeds does not start (missing config)
+
+This can happen if `./config` is not world-readable. You will see the error message `open /config/feeds.yml: permission denied` in the feeds logs.
+
+To fix simply run:
+
+```shell
+$ chmod ugo+rx ./config
+$ chmod ugo+r ./config/feeds.yml
+```
+
+### Sandbox container is not starting (cgroups v2)
+
+If the `analysis` logs show failures when trying to start the sandbox container, your machine may need to be configured to use cgroups v2.
+
+To work with cgroups v2 you will need to:
+
+1. add/edit `/etc/docker/daemon.json` and the following:
+
+```json
+{
+    "default-cgroupns-mode": "host"
+}
+```
+
+2. restart dockerd (if it is running). e.g.:
+
+```shell
+$ systemctl restart docker.service
+```

test/e2e/docker-compose.test.yml

@@ -0,0 +1,12 @@
+version: "3"
+services:
+  analysis:
+    image: gcr.io/ossf-malware-analysis/analysis:test
+    environment:
+      OSSF_SANDBOX_NOPULL: "true"
+    # for mounting local sandbox images inside container
+    volumes:
+      - "/var/lib/containers:/var/lib/containers"
+
+  scheduler:
+    image: gcr.io/ossf-malware-analysis/scheduler:test