From 6e55e73da1ae6c967eb44e98334a2d5c9819ab33 Mon Sep 17 00:00:00 2001 From: Stephen Sadowski Date: Sun, 16 Jan 2022 12:42:38 -0600 Subject: [PATCH] fix: websocket dependency for websockets 9.1 security fix (#2366) Co-authored-by: Adam Hopkins --- .github/workflows/codeql-analysis.yml | 40 ++++++++++++++++++++++ .github/workflows/coverage.yml | 37 +++++++++++++++++++++ .github/workflows/on-demand.yml | 39 ++++++++++++++++++++++ .github/workflows/pr-bandit.yml | 36 ++++++++++++++++++++ .github/workflows/pr-docs.yml | 32 ++++++++++++++++++ .github/workflows/pr-linter.yml | 33 ++++++++++++++++++ .github/workflows/pr-python-pypy.yml | 41 +++++++++++++++++++++++ .github/workflows/pr-python37.yml | 35 +++++++++++++++++++ .github/workflows/pr-python38.yml | 35 +++++++++++++++++++ .github/workflows/pr-python39.yml | 47 ++++++++++++++++++++++++++ .github/workflows/pr-type-check.yml | 35 +++++++++++++++++++ .github/workflows/pr-windows.yml | 37 +++++++++++++++++++++ .github/workflows/publish-images.yml | 48 +++++++++++++++++++++++++++ .github/workflows/publish-package.yml | 28 ++++++++++++++++ CHANGELOG.rst | 10 ++++++ sanic/__version__.py | 2 +- sanic/compat.py | 1 - sanic/request.py | 2 +- setup.py | 2 +- tests/test_asgi.py | 19 ----------- tests/test_config.py | 2 +- tests/test_logging.py | 2 +- tests/test_requests.py | 6 ++-- tox.ini | 2 +- 24 files changed, 542 insertions(+), 29 deletions(-) create mode 100644 .github/workflows/codeql-analysis.yml create mode 100644 .github/workflows/coverage.yml create mode 100644 .github/workflows/on-demand.yml create mode 100644 .github/workflows/pr-bandit.yml create mode 100644 .github/workflows/pr-docs.yml create mode 100644 .github/workflows/pr-linter.yml create mode 100644 .github/workflows/pr-python-pypy.yml create mode 100644 .github/workflows/pr-python37.yml create mode 100644 .github/workflows/pr-python38.yml create mode 100644 .github/workflows/pr-python39.yml create mode 100644 .github/workflows/pr-type-check.yml create mode 100644 .github/workflows/pr-windows.yml create mode 100644 .github/workflows/publish-images.yml create mode 100644 .github/workflows/publish-package.yml diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 0000000000..1113fa80f9 --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,40 @@ +name: "CodeQL" + +on: + push: + branches: + - main + - "*LTS" + pull_request: + branches: + - main + - "*LTS" + types: [opened, synchronize, reopened, ready_for_review] + schedule: + - cron: '25 16 * * 0' + +jobs: + analyze: + if: github.event.pull_request.draft == false + name: Analyze + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + language: [ 'python' ] + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + with: + languages: ${{ matrix.language }} + + - name: Autobuild + uses: github/codeql-action/autobuild@v1 + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v1 diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml new file mode 100644 index 0000000000..9b5834fa13 --- /dev/null +++ b/.github/workflows/coverage.yml @@ -0,0 +1,37 @@ +name: Coverage check +on: + push: + branches: + - main + - "*LTS" + tags: + - "!*" # Do not execute on tags + pull_request: + types: [opened, synchronize, reopened, ready_for_review] +jobs: + test: + if: github.event.pull_request.draft == false + runs-on: ${{ matrix.os }} + strategy: + matrix: + python-version: [3.9] + os: [ubuntu-latest] + fail-fast: false + + steps: + - uses: actions/checkout@v2 + + - uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies 🔨 + run: | + python -m pip install --upgrade pip + pip install tox + - uses: paambaati/codeclimate-action@v2.5.3 + if: always() + env: + CC_TEST_REPORTER_ID: ${{ secrets.CODECLIMATE }} + with: + coverageCommand: tox -e coverage diff --git a/.github/workflows/on-demand.yml b/.github/workflows/on-demand.yml new file mode 100644 index 0000000000..1aa721f366 --- /dev/null +++ b/.github/workflows/on-demand.yml @@ -0,0 +1,39 @@ +name: On Demand Task +on: + workflow_dispatch: + inputs: + python-version: + description: 'Version of Python to use for running Test' + required: false + default: "3.8" + tox-env: + description: 'Test Environment to Run' + required: true + default: '' + os: + description: 'Operating System to Run Test on' + required: false + default: ubuntu-latest +jobs: + onDemand: + name: tox-${{ matrix.config.tox-env }}-on-${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: ["${{ github.event.inputs.os}}"] + config: + - { tox-env: "${{ github.event.inputs.tox-env }}", py-version: "${{ github.event.inputs.python-version }}"} + steps: + - name: Checkout Repository + uses: actions/checkout@v2 + + - name: Run tests + uses: harshanarayana/custom-actions@main + with: + python-version: ${{ matrix.config.py-version }} + test-infra-tool: tox + test-infra-version: latest + action: tests + test-additional-args: "-e=${{ matrix.config.tox-env }}" + experimental-ignore-error: "yes" diff --git a/.github/workflows/pr-bandit.yml b/.github/workflows/pr-bandit.yml new file mode 100644 index 0000000000..2bd702041b --- /dev/null +++ b/.github/workflows/pr-bandit.yml @@ -0,0 +1,36 @@ +name: Security Analysis +on: + pull_request: + branches: + - main + - "*LTS" + types: [opened, synchronize, reopened, ready_for_review] + +jobs: + bandit: + if: github.event.pull_request.draft == false + name: type-check-${{ matrix.config.python-version }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + config: + - { python-version: 3.7, tox-env: security} + - { python-version: 3.8, tox-env: security} + - { python-version: 3.9, tox-env: security} + - { python-version: "3.10", tox-env: security} + steps: + - name: Checkout the repository + uses: actions/checkout@v2 + id: checkout-branch + + - name: Run Linter Checks + id: linter-check + uses: harshanarayana/custom-actions@main + with: + python-version: ${{ matrix.config.python-version }} + test-infra-tool: tox + test-infra-version: latest + action: tests + test-additional-args: "-e=${{ matrix.config.tox-env }}" diff --git a/.github/workflows/pr-docs.yml b/.github/workflows/pr-docs.yml new file mode 100644 index 0000000000..8479aef547 --- /dev/null +++ b/.github/workflows/pr-docs.yml @@ -0,0 +1,32 @@ +name: Document Linter +on: + pull_request: + branches: + - main + - "*LTS" + types: [opened, synchronize, reopened, ready_for_review] + +jobs: + docsLinter: + if: github.event.pull_request.draft == false + name: Lint Documentation + runs-on: ubuntu-latest + strategy: + matrix: + config: + - {python-version: "3.8", tox-env: "docs"} + fail-fast: false + + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Run Document Linter + uses: harshanarayana/custom-actions@main + with: + python-version: ${{ matrix.config.python-version }} + test-infra-tool: tox + test-infra-version: latest + action: tests + test-additional-args: "-e=${{ matrix.config.tox-env }}" diff --git a/.github/workflows/pr-linter.yml b/.github/workflows/pr-linter.yml new file mode 100644 index 0000000000..11ad9d296e --- /dev/null +++ b/.github/workflows/pr-linter.yml @@ -0,0 +1,33 @@ +name: Linter Checks +on: + pull_request: + branches: + - main + - "*LTS" + types: [opened, synchronize, reopened, ready_for_review] + +jobs: + linter: + if: github.event.pull_request.draft == false + name: lint + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + config: + - { python-version: 3.8, tox-env: lint} + steps: + - name: Checkout the repository + uses: actions/checkout@v2 + id: checkout-branch + + - name: Run Linter Checks + id: linter-check + uses: harshanarayana/custom-actions@main + with: + python-version: ${{ matrix.config.python-version }} + test-infra-tool: tox + test-infra-version: latest + action: tests + test-additional-args: "-e=${{ matrix.config.tox-env }}" diff --git a/.github/workflows/pr-python-pypy.yml b/.github/workflows/pr-python-pypy.yml new file mode 100644 index 0000000000..c1a6c7cf79 --- /dev/null +++ b/.github/workflows/pr-python-pypy.yml @@ -0,0 +1,41 @@ +name: Python PyPy Tests +on: + workflow_dispatch: + inputs: + tox-env: + description: "Tox Env to run on the PyPy Infra" + required: false + default: "pypy37" + pypy-version: + description: "Version of PyPy to use" + required: false + default: "pypy-3.7" +jobs: + testPyPy: + name: ut-${{ matrix.config.tox-env }}-${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + # os: [ubuntu-latest, macos-latest] + os: [ubuntu-latest] + config: + - { + python-version: "${{ github.event.inputs.pypy-version }}", + tox-env: "${{ github.event.inputs.tox-env }}", + } + steps: + - name: Checkout the Repository + uses: actions/checkout@v2 + id: checkout-branch + + - name: Run Unit Tests + uses: harshanarayana/custom-actions@main + with: + python-version: ${{ matrix.config.python-version }} + test-infra-tool: tox + test-infra-version: latest + action: tests + test-additional-args: "-e=${{ matrix.config.tox-env }}" + experimental-ignore-error: "true" + command-timeout: "600000" diff --git a/.github/workflows/pr-python37.yml b/.github/workflows/pr-python37.yml new file mode 100644 index 0000000000..c0051d3372 --- /dev/null +++ b/.github/workflows/pr-python37.yml @@ -0,0 +1,35 @@ +name: Python 3.7 Tests +on: + pull_request: + branches: + - main + - "*LTS" + types: [opened, synchronize, reopened, ready_for_review] + +jobs: + testPy37: + if: github.event.pull_request.draft == false + name: ut-${{ matrix.config.tox-env }}-${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: true + matrix: + # os: [ubuntu-latest, macos-latest] + os: [ubuntu-latest] + config: + - { python-version: 3.7, tox-env: py37 } + - { python-version: 3.7, tox-env: py37-no-ext } + steps: + - name: Checkout the Repository + uses: actions/checkout@v2 + id: checkout-branch + + - name: Run Unit Tests + uses: harshanarayana/custom-actions@main + with: + python-version: ${{ matrix.config.python-version }} + test-infra-tool: tox + test-infra-version: latest + action: tests + test-additional-args: "-e=${{ matrix.config.tox-env }}" + test-failure-retry: "3" diff --git a/.github/workflows/pr-python38.yml b/.github/workflows/pr-python38.yml new file mode 100644 index 0000000000..09e93f3ff5 --- /dev/null +++ b/.github/workflows/pr-python38.yml @@ -0,0 +1,35 @@ +name: Python 3.8 Tests +on: + pull_request: + branches: + - main + - "*LTS" + types: [opened, synchronize, reopened, ready_for_review] + +jobs: + testPy38: + if: github.event.pull_request.draft == false + name: ut-${{ matrix.config.tox-env }}-${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: true + matrix: + # os: [ubuntu-latest, macos-latest] + os: [ubuntu-latest] + config: + - { python-version: 3.8, tox-env: py38 } + - { python-version: 3.8, tox-env: py38-no-ext } + steps: + - name: Checkout the Repository + uses: actions/checkout@v2 + id: checkout-branch + + - name: Run Unit Tests + uses: harshanarayana/custom-actions@main + with: + python-version: ${{ matrix.config.python-version }} + test-infra-tool: tox + test-infra-version: latest + action: tests + test-additional-args: "-e=${{ matrix.config.tox-env }}" + test-failure-retry: "3" diff --git a/.github/workflows/pr-python39.yml b/.github/workflows/pr-python39.yml new file mode 100644 index 0000000000..ff4794598e --- /dev/null +++ b/.github/workflows/pr-python39.yml @@ -0,0 +1,47 @@ +name: Python 3.9 Tests +on: + pull_request: + branches: + - main + - "*LTS" + types: [opened, synchronize, reopened, ready_for_review] + +jobs: + testPy39: + if: github.event.pull_request.draft == false + name: ut-${{ matrix.config.tox-env }}-${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: true + matrix: + # os: [ubuntu-latest, macos-latest] + os: [ubuntu-latest] + config: + - { + python-version: 3.9, + tox-env: py39, + ignore-error-flake: "false", + command-timeout: "0", + } + - { + python-version: 3.9, + tox-env: py39-no-ext, + ignore-error-flake: "true", + command-timeout: "600000", + } + steps: + - name: Checkout the Repository + uses: actions/checkout@v2 + id: checkout-branch + + - name: Run Unit Tests + uses: harshanarayana/custom-actions@main + with: + python-version: ${{ matrix.config.python-version }} + test-infra-tool: tox + test-infra-version: latest + action: tests + test-additional-args: "-e=${{ matrix.config.tox-env }},-vv=''" + experimental-ignore-error: "${{ matrix.config.ignore-error-flake }}" + command-timeout: "${{ matrix.config.command-timeout }}" + test-failure-retry: "3" diff --git a/.github/workflows/pr-type-check.yml b/.github/workflows/pr-type-check.yml new file mode 100644 index 0000000000..c071fdeee2 --- /dev/null +++ b/.github/workflows/pr-type-check.yml @@ -0,0 +1,35 @@ +name: Typing Checks +on: + pull_request: + branches: + - main + - "*LTS" + types: [opened, synchronize, reopened, ready_for_review] + +jobs: + typeChecking: + if: github.event.pull_request.draft == false + name: type-check-${{ matrix.config.python-version }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + config: + # - { python-version: 3.7, tox-env: type-checking} + - { python-version: 3.8, tox-env: type-checking} + - { python-version: 3.9, tox-env: type-checking} + steps: + - name: Checkout the repository + uses: actions/checkout@v2 + id: checkout-branch + + - name: Run Linter Checks + id: linter-check + uses: harshanarayana/custom-actions@main + with: + python-version: ${{ matrix.config.python-version }} + test-infra-tool: tox + test-infra-version: latest + action: tests + test-additional-args: "-e=${{ matrix.config.tox-env }}" diff --git a/.github/workflows/pr-windows.yml b/.github/workflows/pr-windows.yml new file mode 100644 index 0000000000..b789f2b643 --- /dev/null +++ b/.github/workflows/pr-windows.yml @@ -0,0 +1,37 @@ +name: Run Unit Tests on Windows +on: + pull_request: + branches: + - main + - "*LTS" + types: [opened, synchronize, reopened, ready_for_review] + +jobs: + testsOnWindows: + if: github.event.pull_request.draft == false + name: ut-${{ matrix.config.tox-env }} + runs-on: windows-latest + strategy: + fail-fast: false + matrix: + config: + - { python-version: 3.7, tox-env: py37-no-ext } + - { python-version: 3.8, tox-env: py38-no-ext } + - { python-version: 3.9, tox-env: py39-no-ext } + - { python-version: pypy-3.7, tox-env: pypy37-no-ext } + + steps: + - name: Checkout Repository + uses: actions/checkout@v2 + + - name: Run Unit Tests + uses: ahopkins/custom-actions@pip-extra-args + with: + python-version: ${{ matrix.config.python-version }} + test-infra-tool: tox + test-infra-version: latest + action: tests + test-additional-args: "-e=${{ matrix.config.tox-env }}" + experimental-ignore-error: "true" + command-timeout: "600000" + pip-extra-args: "--user" diff --git a/.github/workflows/publish-images.yml b/.github/workflows/publish-images.yml new file mode 100644 index 0000000000..8c78f96c8f --- /dev/null +++ b/.github/workflows/publish-images.yml @@ -0,0 +1,48 @@ +name: Publish Docker Images +on: + workflow_run: + workflows: + - 'Publish Artifacts' + types: + - completed + +jobs: + publishDockerImages: + name: Docker Image Build [${{ matrix.python-version }}] + runs-on: ubuntu-latest + + strategy: + fail-fast: true + matrix: + python-version: ["3.7", "3.8", "3.9"] + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Build Latest Base images for ${{ matrix.python-version }} + uses: harshanarayana/custom-actions@main + with: + docker-image-base-name: sanicframework/sanic-build + ignore-python-setup: 'true' + dockerfile-base-dir: './docker' + action: 'image-publish' + docker-image-tag: "${{ matrix.python-version }}" + docker-file-suffix: "base" + docker-build-args: "PYTHON_VERSION=${{ matrix.python-version }}" + registry-auth-user: ${{ secrets.DOCKER_ACCESS_USER }} + registry-auth-password: ${{ secrets.DOCKER_ACCESS_TOKEN }} + push-images: 'true' + + - name: Publish Sanic Docker Image for ${{ matrix.python-version }} + uses: harshanarayana/custom-actions@main + with: + docker-image-base-name: sanicframework/sanic + ignore-python-setup: 'true' + dockerfile-base-dir: './docker' + action: 'image-publish' + docker-build-args: "BASE_IMAGE_TAG=${{ matrix.python-version }}" + docker-image-prefix: "${{ matrix.python-version }}" + registry-auth-user: ${{ secrets.DOCKER_ACCESS_USER }} + registry-auth-password: ${{ secrets.DOCKER_ACCESS_TOKEN }} + push-images: 'true' diff --git a/.github/workflows/publish-package.yml b/.github/workflows/publish-package.yml new file mode 100644 index 0000000000..faa3b7e6dd --- /dev/null +++ b/.github/workflows/publish-package.yml @@ -0,0 +1,28 @@ +name: Publish Artifacts +on: + release: + types: [created] + +jobs: + publishPythonPackage: + name: Publishing Sanic Release Artifacts + runs-on: ubuntu-latest + + strategy: + fail-fast: true + matrix: + python-version: ["3.8"] + + steps: + - name: Checkout Repository + uses: actions/checkout@v2 + + - name: Publish Python Package + uses: harshanarayana/custom-actions@main + with: + python-version: ${{ matrix.python-version }} + package-infra-name: "twine" + pypi-user: __token__ + pypi-access-token: ${{ secrets.PYPI_ACCESS_TOKEN }} + action: "package-publish" + pypi-verify-metadata: "true" diff --git a/CHANGELOG.rst b/CHANGELOG.rst index cb828d0092..41f30c07e3 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,3 +1,13 @@ +Version 20.12.5 +=============== + +Bugfixes +******** + + * + `#2366 `_ + websocket dependency for websockets 9.1 security fix + Version 20.12.0 =============== diff --git a/sanic/__version__.py b/sanic/__version__.py index 7e720e33ca..43f6a244ce 100644 --- a/sanic/__version__.py +++ b/sanic/__version__.py @@ -1 +1 @@ -__version__ = "20.12.4" +__version__ = "20.12.5" diff --git a/sanic/compat.py b/sanic/compat.py index f624442615..48c18a40bc 100644 --- a/sanic/compat.py +++ b/sanic/compat.py @@ -20,7 +20,6 @@ def get_all(self, key): def stat_async(path): return Path(path).stat() - else: from aiofiles import open as aio_open # type: ignore from aiofiles.os import stat as stat_async # type: ignore # noqa: F401 diff --git a/sanic/request.py b/sanic/request.py index 95b64d34b1..092264f1d9 100644 --- a/sanic/request.py +++ b/sanic/request.py @@ -50,7 +50,7 @@ def __init__(self, buffer_size=100): self._queue = asyncio.Queue(buffer_size) async def read(self): - """ Stop reading when gets None """ + """Stop reading when gets None""" payload = await self._queue.get() self._queue.task_done() return payload diff --git a/setup.py b/setup.py index ab5c0f3a73..0cc5f595ea 100644 --- a/setup.py +++ b/setup.py @@ -88,7 +88,7 @@ def open_local(paths, mode="r", encoding="utf8"): uvloop, ujson, "aiofiles>=0.6.0", - "websockets>=8.1,<9.0", + "websockets>=8.1,<=9.1", "multidict>=5.0,<6.0", "httpx==0.15.4", ] diff --git a/tests/test_asgi.py b/tests/test_asgi.py index 0c728493f9..a6d0287fb7 100644 --- a/tests/test_asgi.py +++ b/tests/test_asgi.py @@ -1,6 +1,3 @@ -import asyncio -import sys - from collections import deque, namedtuple import pytest @@ -82,14 +79,6 @@ def install_signal_handlers(self): with pytest.warns(UserWarning): server.run() - all_tasks = ( - asyncio.Task.all_tasks() - if sys.version_info < (3, 7) - else asyncio.all_tasks(asyncio.get_event_loop()) - ) - for task in all_tasks: - task.cancel() - assert before_server_start assert after_server_start assert before_server_stop @@ -132,14 +121,6 @@ def install_signal_handlers(self): with pytest.warns(UserWarning): server.run() - all_tasks = ( - asyncio.Task.all_tasks() - if sys.version_info < (3, 7) - else asyncio.all_tasks(asyncio.get_event_loop()) - ) - for task in all_tasks: - task.cancel() - assert before_server_start assert after_server_start assert before_server_stop diff --git a/tests/test_config.py b/tests/test_config.py index 7c232d7556..754d29c30f 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -13,7 +13,7 @@ @contextmanager def temp_path(): - """ a simple cross platform replacement for NamedTemporaryFile """ + """a simple cross platform replacement for NamedTemporaryFile""" with TemporaryDirectory() as td: yield Path(td, "file") diff --git a/tests/test_logging.py b/tests/test_logging.py index faa83571bc..95236be4f5 100644 --- a/tests/test_logging.py +++ b/tests/test_logging.py @@ -102,7 +102,7 @@ def test_logging_pass_customer_logconfig(): @pytest.mark.parametrize("debug", (True, False)) def test_log_connection_lost(app, debug, monkeypatch): - """ Should not log Connection lost exception on non debug """ + """Should not log Connection lost exception on non debug""" stream = StringIO() root = logging.getLogger("sanic.root") root.addHandler(logging.StreamHandler(stream)) diff --git a/tests/test_requests.py b/tests/test_requests.py index 7a47c3f9ef..63ab59d4a7 100644 --- a/tests/test_requests.py +++ b/tests/test_requests.py @@ -289,18 +289,18 @@ async def handler(request): assert request.args.getlist("test1") == ["1"] assert request.args.get("test3", default="My value") == "My value" + def test_popped_stays_popped(app): @app.route("/") async def handler(request): return text("OK") - request, response = app.test_client.get( - "/", params=[("test1", "1")] - ) + request, response = app.test_client.get("/", params=[("test1", "1")]) assert request.args.pop("test1") == ["1"] assert "test1" not in request.args + @pytest.mark.asyncio async def test_query_string_asgi(app): @app.route("/") diff --git a/tox.ini b/tox.ini index 2110bd2b58..6f5c0adcb7 100644 --- a/tox.ini +++ b/tox.ini @@ -20,7 +20,7 @@ deps = beautifulsoup4 gunicorn==20.0.4 uvicorn - websockets>=8.1,<9.0 + websockets>=8.1,<=9.1 commands = pytest {posargs:tests --cov sanic} - coverage combine --append