Revert build (2 of 2) (#6967)

.ci/compose.ci.yaml

@@ -0,0 +1,25 @@
+services:
+  redash:
+    build: ../
+    command: manage version
+    depends_on:
+      - postgres
+      - redis
+    ports:
+      - "5000:5000"
+    environment:
+      PYTHONUNBUFFERED: 0
+      REDASH_LOG_LEVEL: "INFO"
+      REDASH_REDIS_URL: "redis://redis:6379/0"
+      POSTGRES_PASSWORD: "FmTKs5vX52ufKR1rd8tn4MoSP7zvCJwb"
+      REDASH_DATABASE_URL: "postgresql://postgres:FmTKs5vX52ufKR1rd8tn4MoSP7zvCJwb@postgres/postgres"
+      REDASH_COOKIE_SECRET: "2H9gNG9obnAQ9qnR9BDTQUph6CbXKCzF"
+  redis:
+    image: redis:7-alpine
+    restart: unless-stopped
+  postgres:
+    image: pgautoupgrade/pgautoupgrade:latest
+    command: "postgres -c fsync=off -c full_page_writes=off -c synchronous_commit=OFF"
+    restart: unless-stopped
+    environment:
+      POSTGRES_HOST_AUTH_METHOD: "trust"

.ci/compose.cypress.yaml

@@ -0,0 +1,73 @@
+x-redash-service: &redash-service
+  build:
+    context: ../
+    args:
+      install_groups: "main"
+      code_coverage: ${CODE_COVERAGE}
+x-redash-environment: &redash-environment
+  REDASH_LOG_LEVEL: "INFO"
+  REDASH_REDIS_URL: "redis://redis:6379/0"
+  POSTGRES_PASSWORD: "FmTKs5vX52ufKR1rd8tn4MoSP7zvCJwb"
+  REDASH_DATABASE_URL: "postgresql://postgres:FmTKs5vX52ufKR1rd8tn4MoSP7zvCJwb@postgres/postgres"
+  REDASH_RATELIMIT_ENABLED: "false"
+  REDASH_ENFORCE_CSRF: "true"
+  REDASH_COOKIE_SECRET: "2H9gNG9obnAQ9qnR9BDTQUph6CbXKCzF"
+services:
+  server:
+    <<: *redash-service
+    command: server
+    depends_on:
+      - postgres
+      - redis
+    ports:
+      - "5000:5000"
+    environment:
+      <<: *redash-environment
+      PYTHONUNBUFFERED: 0
+  scheduler:
+    <<: *redash-service
+    command: scheduler
+    depends_on:
+      - server
+    environment:
+      <<: *redash-environment
+  worker:
+    <<: *redash-service
+    command: worker
+    depends_on:
+      - server
+    environment:
+      <<: *redash-environment
+      PYTHONUNBUFFERED: 0
+  cypress:
+    ipc: host
+    build:
+      context: ../
+      dockerfile: .ci/Dockerfile.cypress
+    depends_on:
+      - server
+      - worker
+      - scheduler
+    environment:
+      CYPRESS_baseUrl: "http://server:5000"
+      CYPRESS_coverage: ${CODE_COVERAGE}
+      PERCY_TOKEN: ${PERCY_TOKEN}
+      PERCY_BRANCH: ${CIRCLE_BRANCH}
+      PERCY_COMMIT: ${CIRCLE_SHA1}
+      PERCY_PULL_REQUEST: ${CIRCLE_PR_NUMBER}
+      COMMIT_INFO_BRANCH: ${CIRCLE_BRANCH}
+      COMMIT_INFO_MESSAGE: ${COMMIT_INFO_MESSAGE}
+      COMMIT_INFO_AUTHOR: ${CIRCLE_USERNAME}
+      COMMIT_INFO_SHA: ${CIRCLE_SHA1}
+      COMMIT_INFO_REMOTE: ${CIRCLE_REPOSITORY_URL}
+      CYPRESS_PROJECT_ID: ${CYPRESS_PROJECT_ID}
+      CYPRESS_RECORD_KEY: ${CYPRESS_RECORD_KEY}
+  redis:
+    image: redis:7-alpine
+    restart: unless-stopped
+  postgres:
+    image: pgautoupgrade/pgautoupgrade:latest
+    command: "postgres -c fsync=off -c full_page_writes=off -c synchronous_commit=OFF"
+    restart: unless-stopped
+    environment:
+      POSTGRES_HOST_AUTH_METHOD: "trust"

.ci/docker_build

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+# This script only needs to run on the main Redash repo
+
+if [ "${GITHUB_REPOSITORY}" != "getredash/redash" ]; then
+	echo "Skipping image build for Docker Hub, as this isn't the main Redash repository"
+	exit 0
+fi
+
+if [ "${GITHUB_REF_NAME}" != "master" ] && [ "${GITHUB_REF_NAME}" != "preview-image" ]; then
+	echo "Skipping image build for Docker Hub, as this isn't the 'master' nor 'preview-image' branch"
+	exit 0
+fi
+
+if [ "x${DOCKER_USER}" = "x" ] || [ "x${DOCKER_PASS}" = "x" ]; then
+	echo "Skipping image build for Docker Hub, as the login details aren't available"
+	exit 0
+fi
+
+set -e
+VERSION=$(jq -r .version package.json)
+VERSION_TAG="$VERSION.b${GITHUB_RUN_ID}.${GITHUB_RUN_NUMBER}"
+
+export DOCKER_BUILDKIT=1
+export COMPOSE_DOCKER_CLI_BUILD=1
+
+docker login -u "${DOCKER_USER}" -p "${DOCKER_PASS}"
+
+DOCKERHUB_REPO="redash/redash"
+DOCKER_TAGS="-t redash/redash:preview -t redash/preview:${VERSION_TAG}"
+
+# Build the docker container
+docker build --build-arg install_groups="main,all_ds,dev" ${DOCKER_TAGS} .
+
+# Push the container to the preview build locations
+docker push "${DOCKERHUB_REPO}:preview"
+docker push "redash/preview:${VERSION_TAG}"
+
+echo "Built: ${VERSION_TAG}"

.ci/pack

@@ -0,0 +1,9 @@
+#!/bin/bash
+NAME=redash
+VERSION=$(jq -r .version package.json)
+FULL_VERSION=$VERSION+b$CIRCLE_BUILD_NUM
+FILENAME=$NAME.$FULL_VERSION.tar.gz
+
+mkdir -p /tmp/artifacts/
+
+tar -zcv -f /tmp/artifacts/$FILENAME --exclude=".git" --exclude="optipng*" --exclude="cypress" --exclude="*.pyc" --exclude="*.pyo" --exclude="venv" *

.ci/update_version

@@ -0,0 +1,6 @@
+#!/bin/bash
+VERSION=$(jq -r .version package.json)
+FULL_VERSION=${VERSION}+b${GITHUB_RUN_ID}.${GITHUB_RUN_NUMBER}
+
+sed -ri "s/^__version__ = '([A-Za-z0-9.-]*)'/__version__ = '${FULL_VERSION}'/" redash/__init__.py
+sed -i "s/dev/${GITHUB_SHA}/" client/app/version.json

Dockerfile

@@ -1,35 +1,30 @@
-# Controls whether to build the frontend assets
-ARG FRONTEND_BUILD_MODE=0
+FROM node:18-bookworm as frontend-builder
 
-# MODE 0: create empty files. useful for backend tests
-FROM alpine:3.19 as frontend-builder-0
-RUN \
-  mkdir -p /frontend/client/dist && \
-  touch /frontend/client/dist/multi_org.html && \
-  touch /frontend/client/dist/index.html
+RUN npm install --global --force yarn@1.22.22
 
-# MODE 1: copy static frontend from host, useful for CI to ignore building static content multiple times
-FROM alpine:3.19 as frontend-builder-1
-COPY client/dist /frontend/client/dist
+# Controls whether to build the frontend assets
+ARG skip_frontend_build
 
-# MODE 2: build static content in docker, can be used for a local development
-FROM node:18-bookworm as frontend-builder-2
-RUN npm install --global --force yarn@1.22.22
 ENV CYPRESS_INSTALL_BINARY=0
 ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=1
+
 RUN useradd -m -d /frontend redash
 USER redash
+
 WORKDIR /frontend
 COPY --chown=redash package.json yarn.lock .yarnrc /frontend/
 COPY --chown=redash viz-lib /frontend/viz-lib
 COPY --chown=redash scripts /frontend/scripts
 
-RUN yarn --frozen-lockfile --network-concurrency 1;
+# Controls whether to instrument code for coverage information
+ARG code_coverage
+ENV BABEL_ENV=${code_coverage:+test}
+
+RUN if [ "x$skip_frontend_build" = "x" ] ; then yarn --frozen-lockfile --network-concurrency 1; fi
+
 COPY --chown=redash client /frontend/client
 COPY --chown=redash webpack.config.js /frontend/
-RUN yarn build
-
-FROM frontend-builder-${FRONTEND_BUILD_MODE} as frontend-builder
+RUN if [ "x$skip_frontend_build" = "x" ] ; then yarn build; else mkdir -p /frontend/client/dist && touch /frontend/client/dist/multi_org.html && touch /frontend/client/dist/index.html; fi
 
 FROM python:3.8-slim-bookworm
 
@@ -66,18 +61,17 @@ RUN apt-get update && \
   apt-get clean && \
   rm -rf /var/lib/apt/lists/*
 
-RUN \
-  curl https://packages.microsoft.com/config/debian/12/prod.list > /etc/apt/sources.list.d/mssql-release.list && \
-  curl https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor -o /usr/share/keyrings/microsoft-prod.gpg && \
-  apt update && \
-  ACCEPT_EULA=Y apt install  -y --no-install-recommends msodbcsql18 && \
-  apt clean && \
-  rm -rf /var/lib/apt/lists/*
 
 ARG TARGETPLATFORM
 ARG databricks_odbc_driver_url=https://databricks-bi-artifacts.s3.us-east-2.amazonaws.com/simbaspark-drivers/odbc/2.6.26/SimbaSparkODBC-2.6.26.1045-Debian-64bit.zip
 RUN if [ "$TARGETPLATFORM" = "linux/amd64" ]; then \
-  curl "$databricks_odbc_driver_url" --location --output /tmp/simba_odbc.zip \
+  curl https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor -o /usr/share/keyrings/microsoft-prod.gpg \
+  && curl https://packages.microsoft.com/config/debian/12/prod.list > /etc/apt/sources.list.d/mssql-release.list \
+  && apt-get update \
+  && ACCEPT_EULA=Y apt-get install  -y --no-install-recommends msodbcsql17 \
+  && apt-get clean \
+  && rm -rf /var/lib/apt/lists/* \
+  && curl "$databricks_odbc_driver_url" --location --output /tmp/simba_odbc.zip \
   && chmod 600 /tmp/simba_odbc.zip \
   && unzip /tmp/simba_odbc.zip -d /tmp/simba \
   && dpkg -i /tmp/simba/*.deb \
@@ -97,8 +91,8 @@ COPY pyproject.toml poetry.lock ./
 ARG POETRY_OPTIONS="--no-root --no-interaction --no-ansi"
 # for LDAP authentication, install with `ldap3` group
 # disabled by default due to GPL license conflict
-ARG INSTALL_GROUPS="main,all_ds,dev"
-RUN /etc/poetry/bin/poetry install --only $INSTALL_GROUPS $POETRY_OPTIONS
+ARG install_groups="main,all_ds,dev"
+RUN /etc/poetry/bin/poetry install --only $install_groups $POETRY_OPTIONS
 
 COPY --chown=redash . /app
 COPY --from=frontend-builder --chown=redash /frontend/client/dist /app/client/dist

Makefile

@@ -1,18 +1,10 @@
-.PHONY: compose_build up test_db create_database create_db clean clean-all down tests lint backend-unit-tests frontend-unit-tests pydeps test build watch start redis-cli bash
-
-export COMPOSE_DOCKER_CLI_BUILD=1
-export DOCKER_BUILDKIT=1
-export COMPOSE_PROFILES=local
+.PHONY: compose_build up test_db create_database clean clean-all down tests lint backend-unit-tests frontend-unit-tests test build watch start redis-cli bash
 
 compose_build: .env
-	docker compose build
+	COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 docker compose build
 
 up:
-	docker compose up -d redis postgres
-	docker compose exec -u postgres postgres psql postgres --csv \
-		-1tqc "SELECT table_name FROM information_schema.tables WHERE table_name = 'organizations'" 2> /dev/null \
-		| grep -q "organizations" || make create_database
-	docker compose up -d --build
+	COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 docker compose up -d --build
 
 test_db:
 	@for i in `seq 1 5`; do \
@@ -21,11 +13,9 @@ test_db:
 	done
 	docker compose exec postgres sh -c 'psql -U postgres -c "drop database if exists tests;" && psql -U postgres -c "create database tests;"'
 
-create_db: .env
+create_database: .env
 	docker compose run server create_db
 
-create_database: create_db
-
 clean:
 	docker compose down
 	docker compose --project-name cypress down
@@ -54,12 +44,6 @@ env: .env
 format:
 	pre-commit run --all-files
 
-pydeps:
-	pip3 install wheel
-	pip3 install --upgrade black ruff launchpadlib pip setuptools
-	pip3 install poetry
-	poetry install --only main,all_ds,dev
-
 tests:
 	docker compose run server tests