feat(cubesql): Flatten list expression rewrites to improve performance #3279
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build native | |
| on: | |
| push: | |
| paths: | |
| - '.github/workflows/rust-cubesql.yml' | |
| - 'packages/cubejs-backend-native/**' | |
| - 'rust/cubesql/**' | |
| branches: | |
| - 'master' | |
| pull_request: | |
| paths: | |
| - '.github/workflows/rust-cubesql.yml' | |
| - 'packages/cubejs-backend-native/**' | |
| - 'rust/cubesql/**' | |
| jobs: | |
| lint: | |
| runs-on: ubuntu-20.04 | |
| timeout-minutes: 20 | |
| name: Check fmt/clippy | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Install Rust | |
| uses: actions-rs/toolchain@v1 | |
| with: | |
| toolchain: nightly-2023-12-13 | |
| override: true | |
| components: rustfmt, clippy | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| workspaces: ./rust/cubesql -> target | |
| # default key | |
| key: cubesql-${{ runner.OS }}-x86_64-unknown-linux-gnu-16 | |
| shared-key: cubesql-${{ runner.OS }}-x86_64-unknown-linux-gnu-16 | |
| - name: Lint CubeSQL | |
| run: cd rust/cubesql/cubesql && cargo fmt --all -- --check | |
| - name: Lint Native | |
| run: cd packages/cubejs-backend-native && cargo fmt --all -- --check | |
| - name: Clippy Native | |
| run: cd packages/cubejs-backend-native && cargo clippy -- -D warnings | |
| # CubeSQL is not ready for Clippy | |
| #- name: Clippy CubeSQL | |
| # run: cd rust/cubesql && cargo clippy -- -D warnings | |
| unit: | |
| runs-on: ubuntu-20.04 | |
| timeout-minutes: 60 | |
| name: Unit (Rewrite Engine) | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| # pulls all commits (needed for codecov) | |
| fetch-depth: 2 | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| workspaces: ./rust/cubesql -> target | |
| # default key | |
| key: cubesql-${{ runner.OS }}-x86_64-unknown-linux-gnu-16 | |
| shared-key: cubesql-${{ runner.OS }}-x86_64-unknown-linux-gnu-16 | |
| - name: Install tarpaulin@0.28.0 | |
| uses: baptiste0928/cargo-install@v3 | |
| with: | |
| crate: cargo-tarpaulin | |
| version: "0.28.0" | |
| - name: Unit tests (Rewrite Engine) | |
| env: | |
| CUBESQL_TESTING_CUBE_TOKEN: ${{ secrets.CUBESQL_TESTING_CUBE_TOKEN }} | |
| CUBESQL_TESTING_CUBE_URL: ${{ secrets.CUBESQL_TESTING_CUBE_URL }} | |
| CUBESQL_SQL_PUSH_DOWN: true | |
| CUBESQL_REWRITE_CACHE: true | |
| CUBESQL_REWRITE_TIMEOUT: 60 | |
| run: cd rust/cubesql && cargo tarpaulin --workspace --no-fail-fast --avoid-cfg-tarpaulin --out Xml | |
| # - name: Upload code coverage | |
| # uses: codecov/codecov-action@v3 | |
| # with: | |
| # files: ./rust/cubesql/cobertura.xml | |
| # verbose: true | |
| # flags: cubesql | |
| # fail_ci_if_error: false | |
| native_linux: | |
| needs: [lint] | |
| runs-on: ubuntu-20.04 | |
| timeout-minutes: 60 | |
| name: Build Linux GNU ${{ matrix.node-version }}.x ${{ matrix.target }} with Python ${{ matrix.python-version }} | |
| strategy: | |
| matrix: | |
| # Current used version + 1 LTS | |
| node-version: [18, 20] | |
| python-version: ["3.9", "3.10", "3.11", "3.12", "fallback"] | |
| target: ["x86_64-unknown-linux-gnu", "aarch64-unknown-linux-gnu"] | |
| # minimize number of jobs | |
| exclude: | |
| - node-version: 18 | |
| target: "aarch64-unknown-linux-gnu" | |
| - python-version: 3.10 | |
| target: "aarch64-unknown-linux-gnu" | |
| - python-version: 3.11 | |
| target: "aarch64-unknown-linux-gnu" | |
| fail-fast: false | |
| container: | |
| image: cubejs/rust-cross:${{ matrix.target }}-02042024 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Install Rust | |
| uses: actions-rs/toolchain@v1 | |
| with: | |
| toolchain: nightly-2023-12-13 | |
| override: true | |
| components: rustfmt | |
| target: ${{ matrix.target }} | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| workspaces: ./rust/cubesql -> target | |
| key: cubesql-${{ runner.OS }}-${{ matrix.target }}-${{ matrix.node-version }} | |
| shared-key: cubesql-${{ runner.OS }}-${{ matrix.target }}-${{ matrix.node-version }} | |
| - name: Install Node.js ${{ matrix.node-version }} | |
| uses: actions/setup-node@v3 | |
| with: | |
| node-version: ${{ matrix.node-version }} | |
| - name: Install Yarn | |
| run: npm install -g yarn | |
| - name: Set Yarn version | |
| run: yarn policies set-version v1.22.19 | |
| - name: Get yarn cache directory path | |
| id: yarn-cache-dir-path | |
| run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT | |
| - name: Restore yarn cache | |
| uses: actions/cache@v3 | |
| with: | |
| path: ${{ steps.yarn-cache-dir-path.outputs.dir }} | |
| key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} | |
| restore-keys: | | |
| ${{ runner.os }}-yarn- | |
| - name: Yarn install | |
| uses: nick-invision/retry@v2 | |
| env: | |
| CUBESTORE_SKIP_POST_INSTALL: true | |
| with: | |
| max_attempts: 3 | |
| retry_on: error | |
| retry_wait_seconds: 15 | |
| timeout_minutes: 20 | |
| command: yarn install --frozen-lockfile | |
| - name: Lerna tsc | |
| run: yarn tsc | |
| - name: Build native (fallback) | |
| if: (matrix.python-version == 'fallback') | |
| env: | |
| CARGO_BUILD_TARGET: ${{ matrix.target }} | |
| working-directory: ./packages/cubejs-backend-native | |
| run: yarn run native:build-debug | |
| - name: Setup cross compilation | |
| if: (matrix.target == 'aarch64-unknown-linux-gnu') | |
| uses: allenevans/set-env@v3.0.0 | |
| with: | |
| PYO3_CROSS_PYTHON_VERSION: ${{ matrix.python-version }} | |
| - name: Build native (with Python) | |
| if: (matrix.python-version != 'fallback') | |
| env: | |
| PYO3_PYTHON: python${{ matrix.python-version }} | |
| CARGO_BUILD_TARGET: ${{ matrix.target }} | |
| working-directory: ./packages/cubejs-backend-native | |
| run: yarn run native:build-debug-python | |
| - name: Test native (GNU only) | |
| if: (matrix.target == 'x86_64-unknown-linux-gnu') | |
| env: | |
| CUBESQL_STREAM_MODE: true | |
| CUBEJS_NATIVE_INTERNAL_DEBUG: true | |
| working-directory: ./packages/cubejs-backend-native | |
| run: yarn run test:unit | |
| - name: Run E2E Smoke testing over whole Cube (GNU only) | |
| if: (matrix.target == 'x86_64-unknown-linux-gnu') | |
| env: | |
| CUBEJS_NATIVE_INTERNAL_DEBUG: true | |
| working-directory: ./packages/cubejs-testing | |
| run: yarn smoke:cubesql | |
| native_macos: | |
| needs: [lint] | |
| runs-on: ${{ matrix.os-version }} | |
| timeout-minutes: 60 | |
| name: Build ${{ matrix.os-version }} ${{ matrix.target }} ${{ matrix.node-version }} with Python ${{ matrix.python-version }} | |
| strategy: | |
| matrix: | |
| # We do not need to test under all versions, we do it under linux | |
| node-version: [18.x] | |
| os-version: ["macos-11"] | |
| target: ["x86_64-apple-darwin", "aarch64-apple-darwin"] | |
| include: | |
| - target: x86_64-apple-darwin | |
| python-version: "3.9" | |
| - target: x86_64-apple-darwin | |
| python-version: "3.10" | |
| - target: x86_64-apple-darwin | |
| python-version: "3.11" | |
| - target: x86_64-apple-darwin | |
| python-version: "3.12" | |
| - target: x86_64-apple-darwin | |
| python-version: "fallback" | |
| - target: aarch64-apple-darwin | |
| python-version: "fallback" | |
| fail-fast: false | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Install Rust | |
| uses: actions-rs/toolchain@v1 | |
| with: | |
| toolchain: nightly-2023-12-13 | |
| override: true | |
| components: rustfmt | |
| target: ${{ matrix.target }} | |
| - name: Install Python | |
| uses: actions/setup-python@v4 | |
| if: (matrix.python-version != 'fallback') | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install Node.js ${{ matrix.node-version }} | |
| uses: actions/setup-node@v3 | |
| with: | |
| node-version: ${{ matrix.node-version }} | |
| - name: Set Yarn version | |
| run: yarn policies set-version v1.22.19 | |
| - name: Get yarn cache directory path | |
| id: yarn-cache-dir-path | |
| run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT | |
| shell: bash | |
| - name: Restore yarn cache | |
| uses: actions/cache@v3 | |
| with: | |
| path: ${{ steps.yarn-cache-dir-path.outputs.dir }} | |
| key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} | |
| restore-keys: | | |
| ${{ runner.os }}-yarn- | |
| - name: Yarn install | |
| uses: nick-invision/retry@v2 | |
| env: | |
| CUBESTORE_SKIP_POST_INSTALL: true | |
| with: | |
| max_attempts: 3 | |
| retry_on: error | |
| retry_wait_seconds: 15 | |
| timeout_minutes: 20 | |
| command: yarn install --frozen-lockfile | |
| - name: Lerna tsc | |
| run: yarn tsc | |
| - name: Build native (fallback) | |
| if: (matrix.python-version == 'fallback') | |
| env: | |
| CUBESQL_STREAM_MODE: true | |
| CUBEJS_NATIVE_INTERNAL_DEBUG: true | |
| CARGO_BUILD_TARGET: ${{ matrix.target }} | |
| run: cd packages/cubejs-backend-native && yarn run native:build | |
| - name: Build native (with Python) | |
| if: (matrix.python-version != 'fallback') | |
| env: | |
| PYO3_PYTHON: python${{ matrix.python-version }} | |
| CARGO_BUILD_TARGET: ${{ matrix.target }} | |
| run: cd packages/cubejs-backend-native && yarn run native:build-debug-python | |
| - name: Tests | |
| # We cannot test arm64 on x64 | |
| if: (matrix.target == 'x86_64-apple-darwin') | |
| env: | |
| CUBESQL_STREAM_MODE: true | |
| CUBEJS_NATIVE_INTERNAL_DEBUG: true | |
| run: cd packages/cubejs-backend-native && yarn run test:unit | |
| native_windows: | |
| needs: [lint] | |
| runs-on: ${{ matrix.os-version }} | |
| timeout-minutes: 60 | |
| name: Build ${{ matrix.os-version }} ${{ matrix.node-version }} with Python ${{ matrix.python-version }} | |
| strategy: | |
| matrix: | |
| # We do not need to test under all versions, we do it under linux | |
| node-version: [18.x] | |
| os-version: [windows-2019] | |
| python-version: ["fallback"] | |
| fail-fast: false | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Disable rustup update (issue workaround for Windows) | |
| run: rustup set auto-self-update disable | |
| shell: bash | |
| - name: Install Rust | |
| uses: actions-rs/toolchain@v1 | |
| with: | |
| toolchain: nightly-2023-12-13 | |
| override: true | |
| components: rustfmt | |
| - name: Install Python | |
| uses: actions/setup-python@v4 | |
| if: (matrix.python-version != 'fallback') | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install Node.js ${{ matrix.node-version }} | |
| uses: actions/setup-node@v3 | |
| with: | |
| node-version: ${{ matrix.node-version }} | |
| - name: Set Yarn version | |
| run: yarn policies set-version v1.22.19 | |
| - name: Get yarn cache directory path | |
| id: yarn-cache-dir-path | |
| run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT | |
| shell: bash | |
| - name: Restore yarn cache | |
| uses: actions/cache@v3 | |
| with: | |
| path: ${{ steps.yarn-cache-dir-path.outputs.dir }} | |
| key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} | |
| restore-keys: | | |
| ${{ runner.os }}-yarn- | |
| - name: Yarn install | |
| uses: nick-invision/retry@v2 | |
| env: | |
| CUBESTORE_SKIP_POST_INSTALL: true | |
| with: | |
| max_attempts: 3 | |
| retry_on: error | |
| retry_wait_seconds: 15 | |
| timeout_minutes: 20 | |
| command: yarn install --frozen-lockfile | |
| - name: Lerna tsc | |
| run: yarn tsc | |
| - name: Build native (fallback) | |
| if: (matrix.python-version == 'fallback') | |
| env: | |
| CUBESQL_STREAM_MODE: true | |
| CUBEJS_NATIVE_INTERNAL_DEBUG: true | |
| run: cd packages/cubejs-backend-native && yarn run native:build | |
| - name: Build native (with Python) | |
| if: (matrix.python-version != 'fallback') | |
| env: | |
| PYO3_PYTHON: python${{ matrix.python-version }} | |
| run: cd packages/cubejs-backend-native && yarn run native:build-debug-python | |
| - name: Tests | |
| env: | |
| CUBESQL_STREAM_MODE: true | |
| CUBEJS_NATIVE_INTERNAL_DEBUG: true | |
| run: cd packages/cubejs-backend-native && yarn run test:unit |