Skip to content

Commit

Permalink
Merge branch 'main' into ssl/BoringSSL
Browse files Browse the repository at this point in the history
some manual fixups, more testing and post-fixing will be required.
  • Loading branch information
gpshead committed Mar 27, 2024
2 parents 0e5e1d5 + 48c0b05 commit 0cded81
Show file tree
Hide file tree
Showing 810 changed files with 31,295 additions and 19,505 deletions.
4 changes: 2 additions & 2 deletions .devcontainer/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,11 @@ FROM docker.io/library/fedora:37

ENV CC=clang

ENV WASI_SDK_VERSION=20
ENV WASI_SDK_VERSION=21
ENV WASI_SDK_PATH=/opt/wasi-sdk

ENV WASMTIME_HOME=/opt/wasmtime
ENV WASMTIME_VERSION=18.0.2
ENV WASMTIME_VERSION=18.0.3
ENV WASMTIME_CPU_ARCH=x86_64

RUN dnf -y --nodocs --setopt=install_weak_deps=False install /usr/bin/{blurb,clang,curl,git,ln,tar,xz} 'dnf-command(builddep)' && \
Expand Down
2 changes: 2 additions & 0 deletions .gitattributes
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,8 @@ Include/opcode_ids.h generated
Include/token.h generated
Lib/_opcode_metadata.py generated
Lib/keyword.py generated
Lib/test/certdata/*.pem generated
Lib/test/certdata/*.0 generated
Lib/test/levenshtein_examples.json generated
Lib/test/test_stable_abi_ctypes.py generated
Lib/token.py generated
Expand Down
2 changes: 1 addition & 1 deletion .github/CODEOWNERS
Validating CODEOWNERS rules …
Original file line number Diff line number Diff line change
Expand Up @@ -119,7 +119,7 @@ Python/dynload_*.c @ericsnowcurrently
Lib/test/test_module/ @ericsnowcurrently
Doc/c-api/module.rst @ericsnowcurrently
**/*importlib/resources/* @jaraco @warsaw @FFY00
**/importlib/metadata/* @jaraco @warsaw
**/*importlib/metadata/* @jaraco @warsaw

# Dates and times
**/*datetime* @pganssle @abalkin
Expand Down
41 changes: 39 additions & 2 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ jobs:
- name: Get a list of the changed documentation-related files
if: github.event_name == 'pull_request'
id: changed-docs-files
uses: Ana06/get-changed-files@v2.2.0
uses: Ana06/get-changed-files@v2.3.0
with:
filter: |
Doc/**
Expand Down Expand Up @@ -206,6 +206,8 @@ jobs:
uses: ./.github/workflows/reusable-macos.yml
with:
config_hash: ${{ needs.check_source.outputs.config_hash }}
# macos-14 is M1, macos-13 is Intel
os-matrix: '["macos-14", "macos-13"]'

build_macos_free_threading:
name: 'macOS (free-threading)'
Expand All @@ -215,6 +217,8 @@ jobs:
with:
config_hash: ${{ needs.check_source.outputs.config_hash }}
free-threading: true
# macos-14 is M1
os-matrix: '["macos-14"]'

build_ubuntu:
name: 'Ubuntu'
Expand Down Expand Up @@ -301,6 +305,14 @@ jobs:
- name: SSL tests
run: ./python Lib/test/ssltests.py

build_wasi:
name: 'WASI'
needs: check_source
if: needs.check_source.outputs.run_tests == 'true'
uses: ./.github/workflows/reusable-wasi.yml
with:
config_hash: ${{ needs.check_source.outputs.config_hash }}

test_hypothesis:
name: "Hypothesis tests on Ubuntu"
runs-on: ubuntu-20.04
Expand Down Expand Up @@ -472,6 +484,24 @@ jobs:
- name: Tests
run: xvfb-run make test

build_tsan:
name: 'Thread sanitizer'
needs: check_source
if: needs.check_source.outputs.run_tests == 'true'
uses: ./.github/workflows/reusable-tsan.yml
with:
config_hash: ${{ needs.check_source.outputs.config_hash }}
options: ./configure --config-cache --with-thread-sanitizer --with-pydebug

build_tsan_free_threading:
name: 'Thread sanitizer (free-threading)'
needs: check_source
if: needs.check_source.outputs.run_tests == 'true'
uses: ./.github/workflows/reusable-tsan.yml
with:
config_hash: ${{ needs.check_source.outputs.config_hash }}
options: ./configure --config-cache --disable-gil --with-thread-sanitizer --with-pydebug

# CIFuzz job based on https://google.github.io/oss-fuzz/getting-started/continuous-integration/
cifuzz:
name: CIFuzz
Expand All @@ -484,7 +514,8 @@ jobs:
strategy:
fail-fast: false
matrix:
sanitizer: [address, undefined, memory]
# sanitizer: [address, undefined, memory] -- memory skipped temporarily until GH-116886 is solved.
sanitizer: [address, undefined]
steps:
- name: Build fuzzers (${{ matrix.sanitizer }})
id: build
Expand Down Expand Up @@ -525,10 +556,13 @@ jobs:
- build_ubuntu
- build_ubuntu_free_threading
- build_ubuntu_ssltests
- build_wasi
- build_windows
- build_windows_free_threading
- test_hypothesis
- build_asan
- build_tsan
- build_tsan_free_threading
- cifuzz

runs-on: ubuntu-latest
Expand Down Expand Up @@ -558,9 +592,12 @@ jobs:
build_ubuntu,
build_ubuntu_free_threading,
build_ubuntu_ssltests,
build_wasi,
build_windows,
build_windows_free_threading,
build_asan,
build_tsan,
build_tsan_free_threading,
'
|| ''
}}
Expand Down
12 changes: 6 additions & 6 deletions .github/workflows/jit.yml
Original file line number Diff line number Diff line change
Expand Up @@ -108,8 +108,8 @@ jobs:
if: runner.os == 'macOS'
run: |
brew install llvm@${{ matrix.llvm }}
export SDKROOT="$(xcrun --show-sdk-path)"
./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }}
SDKROOT="$(xcrun --show-sdk-path)" \
./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }}
make all --jobs 4
./python.exe -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 3600 --verbose2 --verbose3
Expand All @@ -134,10 +134,10 @@ jobs:
sudo apt install --yes "gcc-$HOST" qemu-user
${{ !matrix.debug && matrix.compiler == 'clang' && './configure --enable-optimizations' || '' }}
${{ !matrix.debug && matrix.compiler == 'clang' && 'make profile-run-stamp --jobs 4' || '' }}
export CC="${{ matrix.compiler == 'clang' && 'clang --target=$HOST' || '$HOST-gcc' }}"
export CPP="$CC --preprocess"
export HOSTRUNNER=qemu-${{ matrix.architecture }}
export QEMU_LD_PREFIX="/usr/$HOST"
./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }} --build=x86_64-linux-gnu --host="$HOST" --with-build-python=../build/bin/python3 --with-pkg-config=no ac_cv_buggy_getaddrinfo=no ac_cv_file__dev_ptc=no ac_cv_file__dev_ptmx=yes
CC="${{ matrix.compiler == 'clang' && 'clang --target=$HOST' || '$HOST-gcc' }}" \
CPP="$CC --preprocess" \
HOSTRUNNER=qemu-${{ matrix.architecture }} \
./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }} --build=x86_64-linux-gnu --host="$HOST" --with-build-python=../build/bin/python3 --with-pkg-config=no ac_cv_buggy_getaddrinfo=no ac_cv_file__dev_ptc=no ac_cv_file__dev_ptmx=yes
make all --jobs 4
./python -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 3600 --verbose2 --verbose3
2 changes: 1 addition & 1 deletion .github/workflows/lint.yml
Original file line number Diff line number Diff line change
Expand Up @@ -23,4 +23,4 @@ jobs:
- uses: actions/setup-python@v5
with:
python-version: "3.x"
- uses: pre-commit/action@v3.0.0
- uses: pre-commit/action@v3.0.1
2 changes: 1 addition & 1 deletion .github/workflows/project-updater.yml
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ jobs:
- { project: 32, label: sprint }

steps:
- uses: actions/add-to-project@v0.1.0
- uses: actions/add-to-project@v0.6.0
with:
project-url: https://github.com/orgs/python/projects/${{ matrix.project }}
github-token: ${{ secrets.ADD_TO_PROJECT_PAT }}
Expand Down
8 changes: 4 additions & 4 deletions .github/workflows/reusable-macos.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,9 @@ on:
required: false
type: boolean
default: false
os-matrix:
required: false
type: string

jobs:
build_macos:
Expand All @@ -22,10 +25,7 @@ jobs:
strategy:
fail-fast: false
matrix:
os: [
"macos-14", # M1
"macos-13", # Intel
]
os: ${{fromJson(inputs.os-matrix)}}
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v4
Expand Down
51 changes: 51 additions & 0 deletions .github/workflows/reusable-tsan.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
on:
workflow_call:
inputs:
config_hash:
required: true
type: string
options:
required: true
type: string

jobs:
build_tsan_reusable:
name: 'Thread sanitizer'
runs-on: ubuntu-22.04
timeout-minutes: 60
steps:
- uses: actions/checkout@v4
- name: Runner image version
run: echo "IMAGE_VERSION=${ImageVersion}" >> $GITHUB_ENV
- name: Restore config.cache
uses: actions/cache@v4
with:
path: config.cache
key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ inputs.config_hash }}
- name: Install Dependencies
run: |
sudo ./.github/workflows/posix-deps-apt.sh
sudo apt install -y clang
# Reduce ASLR to avoid TSAN crashing
sudo sysctl -w vm.mmap_rnd_bits=28
- name: TSAN Option Setup
run: |
echo "TSAN_OPTIONS=suppressions=${GITHUB_WORKSPACE}/Tools/tsan/supressions.txt" >> $GITHUB_ENV
echo "CC=clang" >> $GITHUB_ENV
echo "CXX=clang++" >> $GITHUB_ENV
- name: Add ccache to PATH
run: |
echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV
- name: Configure ccache action
uses: hendrikmuhs/ccache-action@v1.2
with:
save: ${{ github.event_name == 'push' }}
max-size: "200M"
- name: Configure CPython
run: ${{ inputs.options }}
- name: Build CPython
run: make -j4
- name: Display build info
run: make pythoninfo
- name: Tests
run: ./python -m test --tsan -j4
71 changes: 71 additions & 0 deletions .github/workflows/reusable-wasi.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
on:
workflow_call:
inputs:
config_hash:
required: true
type: string

jobs:
build_wasi_reusable:
name: 'build and test'
timeout-minutes: 60
runs-on: ubuntu-20.04
env:
WASMTIME_VERSION: 18.0.3
WASI_SDK_VERSION: 21
WASI_SDK_PATH: /opt/wasi-sdk
CROSS_BUILD_PYTHON: cross-build/build
CROSS_BUILD_WASI: cross-build/wasm32-wasi
steps:
- uses: actions/checkout@v4
# No problem resolver registered as one doesn't currently exist for Clang.
- name: "Install wasmtime"
uses: jcbhmr/setup-wasmtime@v2
with:
wasmtime-version: ${{ env.WASMTIME_VERSION }}
- name: "Restore WASI SDK"
id: cache-wasi-sdk
uses: actions/cache@v4
with:
path: ${{ env.WASI_SDK_PATH }}
key: ${{ runner.os }}-wasi-sdk-${{ env.WASI_SDK_VERSION }}
- name: "Install WASI SDK"
if: steps.cache-wasi-sdk.outputs.cache-hit != 'true'
run: |
mkdir ${{ env.WASI_SDK_PATH }} && \
curl -s -S --location https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${{ env.WASI_SDK_VERSION }}/wasi-sdk-${{ env.WASI_SDK_VERSION }}.0-linux.tar.gz | \
tar --strip-components 1 --directory ${{ env.WASI_SDK_PATH }} --extract --gunzip
- name: "Configure ccache action"
uses: hendrikmuhs/ccache-action@v1.2
with:
save: ${{ github.event_name == 'push' }}
max-size: "200M"
- name: "Add ccache to PATH"
run: echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV
- name: "Install Python"
uses: actions/setup-python@v5
with:
python-version: '3.x'
- name: "Restore Python build config.cache"
uses: actions/cache@v4
with:
path: ${{ env.CROSS_BUILD_PYTHON }}/config.cache
key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ inputs.config_hash }}
- name: "Configure build Python"
run: python3 Tools/wasm/wasi.py configure-build-python -- --config-cache --with-pydebug
- name: "Make build Python"
run: python3 Tools/wasm/wasi.py make-build-python
- name: "Restore host config.cache"
uses: actions/cache@v4
with:
path: ${{ env.CROSS_BUILD_WASI }}/config.cache
key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-wasi-sdk-${{ env.WASI_SDK_VERSION }}-${{ inputs.config_hash }}
- name: "Configure host"
# `--with-pydebug` inferred from configure-build-python
run: python3 Tools/wasm/wasi.py configure-host -- --config-cache
- name: "Make host"
run: python3 Tools/wasm/wasi.py make-host
- name: "Display build info"
run: make --directory ${{ env.CROSS_BUILD_WASI }} pythoninfo
- name: "Test"
run: make --directory ${{ env.CROSS_BUILD_WASI }} test
2 changes: 1 addition & 1 deletion .github/workflows/stale.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ name: Mark stale pull requests

on:
schedule:
- cron: "0 0 * * *"
- cron: "0 */6 * * *"

permissions:
pull-requests: write
Expand Down
11 changes: 10 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,17 @@ Lib/test/data/*
/_bootstrap_python
/Makefile
/Makefile.pre
/iOSTestbed.*
iOS/Frameworks/
iOS/Resources/Info.plist
iOS/testbed/build
iOS/testbed/Python.xcframework/ios-*/bin
iOS/testbed/Python.xcframework/ios-*/include
iOS/testbed/Python.xcframework/ios-*/lib
iOS/testbed/Python.xcframework/ios-*/Python.framework
iOS/testbed/iOSTestbed.xcodeproj/project.xcworkspace
iOS/testbed/iOSTestbed.xcodeproj/xcuserdata
iOS/testbed/iOSTestbed.xcodeproj/xcshareddata
Mac/Makefile
Mac/PythonLauncher/Info.plist
Mac/PythonLauncher/Makefile
Expand Down Expand Up @@ -132,7 +142,6 @@ Tools/unicode/data/
/profile-clean-stamp
/profile-run-stamp
/profile-bolt-stamp
/Python/deepfreeze/*.c
/pybuilddir.txt
/pyconfig.h
/python-config
Expand Down
4 changes: 3 additions & 1 deletion .pre-commit-config.yaml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
repos:
- repo: https://github.com/astral-sh/ruff-pre-commit
rev: v0.2.0
rev: v0.3.4
hooks:
- id: ruff
name: Run Ruff on Lib/test/
Expand All @@ -14,6 +14,8 @@ repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v4.5.0
hooks:
- id: check-case-conflict
- id: check-merge-conflict
- id: check-toml
exclude: ^Lib/test/test_tomllib/
- id: check-yaml
Expand Down

0 comments on commit 0cded81

Please sign in to comment.