Add registry-watcher image and separate docker-compose services

.dockerignore

@@ -1,4 +1,12 @@
+/target/
 /.rustwide
 /.rustwide-docker
+/Justfile
+/LICENSE
+/README.md
+/docker-compose.yml
+/docs/
 /ignored
-**/target
+/mcps
+/triagebot.toml
+/clippy.toml

.github/workflows/ci.yml

@@ -44,13 +44,8 @@ jobs:
 
       - name: Launch postgres
         run: |
-          cp .env.sample .env
-          mkdir -p ${DOCSRS_PREFIX}/public-html
-          docker-compose up -d db
-          # Give the database enough time to start up
-          sleep 5
-          # Make sure the database is actually working
-          psql "${DOCSRS_DATABASE_URL}"
+          touch .docker.env
+          docker compose up --wait --wait-timeout 30 db
 
       - name: run database migrations
         run: cargo run -- database migrate
@@ -68,9 +63,6 @@ jobs:
             --database-url $DOCSRS_DATABASE_URL \
             --target-version 0
 
-      - name: Clean up the database
-        run: docker-compose down --volumes
-
   test:
     env:
       SQLX_OFFLINE: 1
@@ -92,13 +84,8 @@ jobs:
 
       - name: Launch postgres and min.io
         run: |
-          cp .env.sample .env
-          mkdir -p ${DOCSRS_PREFIX}/public-html
-          docker-compose up -d db s3
-          # Give the database enough time to start up
-          sleep 5
-          # Make sure the database is actually working
-          psql "${DOCSRS_DATABASE_URL}"
+          touch .docker.env
+          docker compose up --wait --wait-timeout 30 db s3
 
       - name: run workspace tests
         run: |
@@ -110,9 +97,6 @@ jobs:
         run: |
           cargo test --locked -- --ignored --test-threads=1
 
-      - name: Clean up the database
-        run: docker-compose down --volumes
-
   GUI_test:
     runs-on: ubuntu-latest
     steps:
@@ -129,19 +113,12 @@ jobs:
 
       - name: Launch postgres and min.io
         run: |
-          cp .env.sample .env
-          mkdir -p ${DOCSRS_PREFIX}/public-html
-          docker-compose up -d db s3
-          # Give the database enough time to start up
-          sleep 5
-          # Make sure the database is actually working
-          psql "${DOCSRS_DATABASE_URL}"
+          touch .docker.env
+          docker compose up --wait --wait-timeout 30 db s3
 
       - name: Run GUI tests
         run: ./dockerfiles/run-gui-tests.sh
 
-      - name: Clean up the database
-        run: docker-compose down --volumes
 
   fmt:
     name: Rustfmt

.github/workflows/docker.yml

@@ -9,5 +9,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - name: Build the Docker image
-        run: docker build -t docs-rs -f dockerfiles/Dockerfile .
+      - run: docker build --target web-server -f dockerfiles/Dockerfile .
+      - run: docker build --target build-server -f dockerfiles/Dockerfile .
+      - run: docker build --target registry-watcher -f dockerfiles/Dockerfile .
+      - run: docker build --target cli -f dockerfiles/Dockerfile .

.gitignore

@@ -1,5 +1,6 @@
 /ignored
 /.env
+/.docker.env
 /src/web/badge/Cargo.lock
 target
 *.css

README.md

@@ -64,8 +64,7 @@ mkdir -p ignored/cratesfyi-prefix/crates.io-index
 # Builds the docs.rs binary
 cargo build
 # Start the external services.
-# It may be `docker compose` in newer versions
-docker-compose up -d db s3
+docker compose up --wait db s3
 # anything that doesn't run via docker-compose needs the settings defined in
 # .env. Either via `. ./.env` as below, or via any dotenv shell integration.
 . ./.env
@@ -111,19 +110,37 @@ can take a look at its documentation [here](https://github.com/GuillaumeGomez/br
 
 ### Pure docker-compose
 
-If you have trouble with the above commands, consider using `docker-compose up --build`,
+If you have trouble with the above commands, consider using `docker compose up --build`,
 which uses docker-compose for the web server as well.
 This will not cache dependencies - in particular, you'll have to rebuild all 400 whenever the lockfile changes -
 but makes sure that you're in a known environment so you should have fewer problems getting started.
 
-You can also use the `web` container to run builds on systems which don't support running builds directly (mostly on Mac OS or Windows):
+You'll need to `touch .docker.env` first, this file can have any environment
+variable overrides you want to use in docker containers.
+
+You can also use the `builder-a` container to run builds on systems which don't support running builds directly (mostly on Mac OS or Windows):
+
 ```sh
+# update the toolchain
+docker compose run --rm builder-a build update-toolchain
 # run a build for a single crate
-docker-compose run web build crate regex 1.3.1
-# or build essential files
-docker-compose run web build add-essential-files
-# rebuild the web container when you changed code.
-docker-compose build web
+docker compose run --rm builder-a build crate regex 1.3.1
+# rebuild containers when you changed code.
+docker compose up --wait --build
+```
+
+You can also run other non-build commands like the setup steps above, or queueing crates for the background builders from within the `cli` container:
+
+```sh
+docker compose run --rm cli database migrate
+docker compose run --rm cli queue add regex 1.3.1
+```
+
+If the command needs the crates.io-index clone then it must be run from within
+a `registry-watcher` container:
+
+```sh
+docker compose run --rm registry-watcher queue set-last-seen-reference --head
 ```
 
 Note that running tests is not supported when using pure docker-compose.
@@ -137,22 +154,22 @@ Three services are defined:
 
 | name | access                                          | credentials                | description                            |
 |------|-------------------------------------------------|----------------------------|----------------------------------------|
-| web  | http://localhost:3000                           | N/A                        | A container running the docs.rs binary |
-| db   | postgresql://cratesfyi:password@localhost:15432 | -                          | Postgres database used by web          |
-| s3   | http://localhost:9000                           | `cratesfyi` - `secret_key` | MinIO (simulates AWS S3) used by web   |
+| web  | http://0.0.0.0:3000                             | N/A                        | A container running the docs.rs binary |
+| db   | postgresql://cratesfyi:password@127.0.0.1:15432 | -                          | Postgres database used by web          |
+| s3   | http://127.0.0.1:9000                           | `cratesfyi` - `secret_key` | MinIO (simulates AWS S3) used by web   |
 
 [docker-compose.yml]: ./docker-compose.yml
 
 #### Rebuilding Containers
 
-To rebuild the site, run `docker-compose build`.
+To rebuild the site, run `docker compose --profile all build`.
 Note that docker-compose caches the build even if you change the source code,
 so this will be necessary anytime you make changes.
 
 If you want to completely clean up the database, don't forget to remove the volumes too:
 
 ```sh
-$ docker-compose down --volumes
+$ docker compose down --volumes
 ```
 
 #### FAQ
@@ -166,7 +183,7 @@ This is probably because you have `git.autocrlf` set to true,
 
 ##### I see the error `/opt/rustwide/cargo-home/bin/cargo: cannot execute binary file: Exec format error` when running builds.
 
-You are most likely not on a Linux platform. Running builds directly is only supported on `x86_64-unknown-linux-gnu`. On other platforms you can use the `docker-compose run web build [...]` workaround described above.
+You are most likely not on a Linux platform. Running builds directly is only supported on `x86_64-unknown-linux-gnu`. On other platforms you can use the `docker compose run --rm builder-a build [...]` workaround described above.
 
 See [rustwide#41](https://github.com/rust-lang/rustwide/issues/41) for more details about supporting more platforms directly.
 
@@ -194,11 +211,11 @@ cargo run -- start-web-server
 ```sh
 # Builds <CRATE_NAME> <CRATE_VERSION> and adds it into database
 # This is the main command to build and add a documentation into docs.rs.
-# For example, `docker-compose run web build crate regex 1.1.6`
+# For example, `docker compose run --rm builder-a build crate regex 1.1.6`
 cargo run -- build crate <CRATE_NAME> <CRATE_VERSION>
 
-# alternatively, via the web container
-docker-compose run web build crate <CRATE_NAME> <CRATE_VERSION>
+# alternatively, within docker-compose containers
+docker compose run --rm builder-a build crate <CRATE_NAME> <CRATE_VERSION>
 
 # Builds every crate on crates.io and adds them into database
 # (beware: this may take months to finish)