From efcb3219cfa22e43ebb48cb64ca8eeab82fbe9cd Mon Sep 17 00:00:00 2001 From: Bart Kroon Date: Tue, 26 Jul 2022 11:06:14 +0200 Subject: [PATCH 01/16] Eat stderr messages from git commands. Prevents: "fatal: not a git repository (or any of the parent directories): .git" in cases where safety is ran outside a git repository. --- safety/util.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/safety/util.py b/safety/util.py index 4b469a79..1b806235 100644 --- a/safety/util.py +++ b/safety/util.py @@ -213,8 +213,11 @@ def build_telemetry_data(telemetry=True): def build_git_data(): import subprocess + def git_command(commandline): + return subprocess.run(commandline, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL).stdout.decode('utf-8').strip() + try: - is_git = subprocess.run(["git", "rev-parse", "--is-inside-work-tree"], stdout=subprocess.PIPE).stdout.decode('utf-8').strip() + is_git = git_command(["git", "rev-parse", "--is-inside-work-tree"]) except Exception: is_git = False @@ -228,14 +231,14 @@ def build_git_data(): } try: - result['branch'] = subprocess.run(["git", "symbolic-ref", "--short", "-q", "HEAD"], stdout=subprocess.PIPE).stdout.decode('utf-8').strip() - result['tag'] = subprocess.run(["git", "describe", "--tags", "--exact-match"], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL).stdout.decode('utf-8').strip() + result['branch'] = git_command(["git", "symbolic-ref", "--short", "-q", "HEAD"]) + result['tag'] = git_command(["git", "describe", "--tags", "--exact-match"]) - commit = subprocess.run(["git", "describe", '--match=""', '--always', '--abbrev=40', '--dirty'], stdout=subprocess.PIPE).stdout.decode('utf-8').strip() + commit = git_command(["git", "describe", '--match=""', '--always', '--abbrev=40', '--dirty']) result['dirty'] = commit.endswith('-dirty') result['commit'] = commit.split("-dirty")[0] - result['origin'] = subprocess.run(["git", "remote", "get-url", "origin"], stdout=subprocess.PIPE).stdout.decode('utf-8').strip() + result['origin'] = git_command(["git", "remote", "get-url", "origin"]) except Exception: pass From e010baa7771aa6bc5ccc95e64aa17e8ec76df27d Mon Sep 17 00:00:00 2001 From: Bart Kroon Date: Tue, 26 Jul 2022 11:06:44 +0200 Subject: [PATCH 02/16] Remove unused import of safety package in utils. --- safety/util.py | 1 - 1 file changed, 1 deletion(-) diff --git a/safety/util.py b/safety/util.py index 1b806235..54d4b6f9 100644 --- a/safety/util.py +++ b/safety/util.py @@ -15,7 +15,6 @@ from packaging.version import parse as parse_version from ruamel.yaml import YAML from ruamel.yaml.error import MarkedYAMLError -import safety from safety.constants import EXIT_CODE_FAILURE, EXIT_CODE_OK from safety.models import Package, RequirementFile From 69af89693e37065ab691258b6d3cb9bf66281534 Mon Sep 17 00:00:00 2001 From: Yeison Vargas Date: Wed, 14 Sep 2022 16:14:15 -0500 Subject: [PATCH 03/16] Starting version 2.2.0.dev --- CHANGELOG.md | 2 ++ appveyor.yml | 2 +- safety/VERSION | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a347341c..51f2e5b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ All notable changes to this project will be documented in this file. The format is partly based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) and [PEP 440](https://peps.python.org/pep-0440/) +## [Unreleased] 2.2.0.dev + ## [2.1.1] - 2022-07-18 - Fix crash when running on systems without git present (Thanks @andyjones) diff --git a/appveyor.yml b/appveyor.yml index 47b13ee2..3742eda9 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,4 +1,4 @@ -version: 2.1.1-{build} +version: 2.2.0.dev-{build} image: - Visual Studio 2019 - Ubuntu diff --git a/safety/VERSION b/safety/VERSION index 3e3c2f1e..7fff5300 100644 --- a/safety/VERSION +++ b/safety/VERSION @@ -1 +1 @@ -2.1.1 +2.2.0.dev From 341507b69f2a31cbe0944c7e1bb24bfca06aa02d Mon Sep 17 00:00:00 2001 From: Yeison Vargas Date: Wed, 14 Sep 2022 16:26:53 -0500 Subject: [PATCH 04/16] Adding basic github action workflow --- .github/workflows/main.yml | 107 +++++++++++++++++++++++++++++++++++++ .travis.yml | 44 --------------- appveyor.yml | 66 ----------------------- appveyor.py => binaries.py | 45 ++++++++-------- test_requirements.txt | 8 +++ 5 files changed, 139 insertions(+), 131 deletions(-) create mode 100644 .github/workflows/main.yml delete mode 100644 .travis.yml delete mode 100644 appveyor.yml rename appveyor.py => binaries.py (71%) create mode 100644 test_requirements.txt diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 00000000..08c1ef7c --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,107 @@ +name: Python package + +on: [ push ] + +jobs: + test: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [ "3.6", "3.7", "3.8", "3.9", "3.10" ] + steps: + - uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Display Python version + run: python -c "import sys; print(sys.version)" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r test_requirements.txt + - name: Test with pytest + run: | + pytest -rP tests/ --cov=dparse/ --cov-report=xml --cov-report=html + + build-binaries: + needs: test + runs-on: ${{ matrix.os }} + + if: contains(fromJson('["refs/heads/master", "refs/heads/develop", "refs/heads/binaries-fixes"]'), github.ref) || startsWith(github.ref, 'refs/tags') + + strategy: + matrix: + os: ['windows-latest', 'ubuntu-latest', 'macos-latest'] + env: + BINARY_OS: '${{ matrix.os }}' + steps: + - uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.10' + - name: Install Dependencies + run: python binaries.py install + - name: Test Safety + run: python binaries.py test + - name: Producing Binaries + run: python binaries.py dist + - uses: actions/upload-artifact@v3 + with: + name: safety-win-i686.exe + path: dist/safety-win-i686.exe + - uses: actions/upload-artifact@v3 + with: + name: safety-win-x86_64.exe + path: dist/safety-win-x86_64.exe + - uses: actions/upload-artifact@v3 + with: + name: safety-linux-i686 + path: dist/safety-linux-i686 + - uses: actions/upload-artifact@v3 + with: + name: safety-linux-x86_64 + path: dist/safety-linux-x86_64 + - uses: actions/upload-artifact@v3 + with: + name: safety-macos-x86_64 + path: dist/safety-macos-x86_64 + + + deploy-pypi: + needs: build-binaries + runs-on: ubuntu-latest + + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') + + steps: + - uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.10' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install build + - name: Build package + run: python -m build + - name: Publish package + uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29 + with: + user: __token__ + password: ${{ secrets.SAFETY_PYPI_API_TOKEN }} + + create-gh-release: + needs: deploy-pypi + runs-on: ubuntu-latest + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') + permissions: + contents: write + steps: + - uses: actions/checkout@v2 + - uses: ncipollo/release-action@v1 + with: + artifacts: "dist/safety-win-i686.exe,dist/safety-win-x86_64.exe,dist/safety-linux-i686,dist/safety-linux-x86_64,dist/safety-macos-x86_64" + token: ${{ secrets.SAFETY_GITHUB_TOKEN }} diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 02b1b4ae..00000000 --- a/.travis.yml +++ /dev/null @@ -1,44 +0,0 @@ -dist: focal - -env: - global: - - TWINE_NON_INTERACTIVE: true - # TWINE_USERNAME - - secure: uLzl+gKc867pGfuFicfHHZ18VETx4dcNLtpEHwoU+mBAwIz7geu7EtwIpzs717XX0wviCizeEJ1f/fVhs/uKnd/uHSfjL+z1/VCY5pr++O5pz2nmAbAQFIehTryF74A2NZi0LlJ+5brJcIt3/bVBsi3nhcmhNDw1eKfqDZV/wIA92Z10jE//JGp1P9qErD7sp0GRbo1NWR6MPwl/6doNK12583avVSuyKptNCKAQd6avxpzQ5PUHoklGq5Uc384VKc5hqcRoZxD3MeygvCZoclcVa+DHcHKfepmqys7A1i+/Kj48/lrlHVwLSbqE+bME0E5H0Js+4ltkEslnPuKvYgUne9tQArwNKWzdnJ6RE3Jij164QmeEbbdkmiYeNlIgq/fEsv0TaLZFEsDTM/+81DGbDwEm196nlbReSMTQ4UKYw/5bVi9k3MVAfYC87eLv8kx3Of7/vc0U3BqbJBBJFNRELCnIpWSR7Q42ycOyPjxmTaZ6rIoTBpyhWqnlQO98zrgMwE9Zkahw9cdGhnkQzPhxWzcyt0ULurN7URxgcaGMxSpajc9uYToppmT/wTWMrLVqN/8tQ6wrMKMVQ9oJTkkZayAyvdFBNwR8bMY8+K0ts5U51F2DvC32WCjdUoUpIiaEGJO37Ca7vU8kN92fLxqdF14Rm8QdWs0fzYTlkLY= - # TWINE_PASSWORD - - secure: wMkT6y9WTicPLeRCCzVxd02WHAHfegO9yJMC9yyQOJM8FqOGQmNQJMd+rS5FcryxTLBnPQgMGWXxOpy1HEUFzjmjX0OPFO9zkheD5PuMPgQpdOKMVVEEhAfr8ndVdwUKGCOXrf5PVOMkDu/dyQx9OdS9TZMWP46HWYOSI0LJ6z/u4ak0QgJszkXANSBvTOGGDvMmsiHnDyqTEFIhepZxkB1v/p2mOif/wQWRwe9cq7NRkl092QkE8PlTKm1HMu5P69n8Uz8OTrqKlmOD3S6mNr94dkNgqrui/6iRq6ofN91LENIlg32ZvMDgRpr2dmPg8IhiJtC4wDtFpkxB7vsg8MIIFiECzBBfTrOXohq/GWqEvlN0bRYkXOiexLMNHHQ+Vylt4LhI8osrjElUCp7TuBf9NBcom5lttQQNsgDGCr1zRxiGH0iRBiW/JJUQ0UFUl90E00JH8z1Np36I7oyIUolB0y/C0UFotq321MqxFUwXv1r/lNHUOwrBElNicGwIAU148tt6iAeNpEvcbXl0NMTMmzFvWQAvIZF2cjw7JX5QgxD+2cLHCOaBPLFXfm4kY6Bo44PlIJD8kdE/RxDN+4KlyBu2N7txgUf7b7s1Qsaat0sSYJzvh4XNcIXLWilPpaRS53aCQ4AyXVGHyicblsO/CmnjyLUNBlGQHQJf0J4= - -branches: - only: - - master - - develop - - /^\d+\.\d+(\.\d+)?((a|b|rc)?\d+)?$/ - -language: python - -python: - - '3.6' - - '3.7' - - '3.8' - - '3.9' - - '3.10' -install: - - pip install tox-travis - - pip install codecov -script: - - tox -after_success: - - codecov - -before_deploy: - - pip install --upgrade build - - pip install --upgrade twine - -deploy: - - provider: script - script: python3 -m build && twine upload dist/* - on: - repo: pyupio/safety - branch: master - python: 3.9 - tags: true diff --git a/appveyor.yml b/appveyor.yml deleted file mode 100644 index 3742eda9..00000000 --- a/appveyor.yml +++ /dev/null @@ -1,66 +0,0 @@ -version: 2.2.0.dev-{build} -image: - - Visual Studio 2019 - - Ubuntu - - macOS - -# Do not build feature branch with open Pull Requests -skip_branch_with_pr: true -branches: - only: - - master - - develop - - appveyor-fixes - - /^\d+\.\d+(\.\d+)?((a|b|rc)?\d+)?$/ - -# note: on macOS the binary is built using Python 3.7 (installed via Homebrew), because -# the shipped Python lacks libraries PyInstaller needs. -stack: python 3.6 - -# note: 32 bit linux binary is build using docker -for: -- - matrix: - only: - - image: Ubuntu - - services: - - docker - -environment: - PY_DIR: C:\Python36-x64 - -init: - - cmd: set PATH=%PY_DIR%;%PY_DIR%\Scripts;%PATH% - -build: off - -artifacts: - - path: "dist\\safety-win-i686.exe" - name: "safety-win-i686.exe" - - path: "dist\\safety-win-x86_64.exe" - name: "safety-win-x86_64.exe" - - path: "dist\\safety-linux-i686" - name: "safety-linux-i686" - - path: "dist\\safety-linux-x86_64" - name: "safety-linux-x86_64" - - path: "dist\\safety-macos-x86_64" - name: "safety-macos-x86_64" - -install: - - "python --version" - - "python appveyor.py install" - -test_script: - - "python appveyor.py test" - - "python appveyor.py dist" - - -deploy: - - provider: GitHub - description: 'Safety Binary Release' - auth_token: - secure: FQDI6RodnfAg03naBo+mYvDjtNmLhDSH5d11mQPrAns7Cj83JZWpQc36GsOcW+JM - on: - branch: master - APPVEYOR_REPO_TAG: true diff --git a/appveyor.py b/binaries.py similarity index 71% rename from appveyor.py rename to binaries.py index b5dfd683..0752f932 100644 --- a/appveyor.py +++ b/binaries.py @@ -1,7 +1,7 @@ -"""AppVeyor Build +"""Github Action Build -This file is used to build and distribute the safety binary on appveyor. Take -a look at the corresponding appveyor.yml as well. +This file is used to build and distribute the safety binary on Github actions. +Take a look at the corresponding main.yml as well. """ import os @@ -18,11 +18,11 @@ class environment: def __init__(self): os_mapping = { - "Visual Studio 2019": self.WIN, - "Ubuntu": self.LINUX, - "macOS": self.MACOS + "windows-latest": self.WIN, + "ubuntu-latest": self.LINUX, + "macos-latest": self.MACOS } - self.os = os_mapping[os.getenv("APPVEYOR_BUILD_WORKER_IMAGE")] + self.os = os_mapping[os.getenv("BINARY_OS")] @property def python(self): @@ -31,22 +31,23 @@ def python(self): PYTHON_BINARIES = { WIN: { - 64: "C:\\Python38-x64\\python.exe", - 32: "C:\\Python38\\python.exe", + 64: "C:\\Python39-x64\\python.exe", + 32: "C:\\Python39\\python.exe", }, # Order is important. If the 32 bit release gets built first, # you'll run into permission problems due to docker clobbering # up the current working directory. LINUX: OrderedDict([ - (64, "python"), - (32, f"docker run -t -v {os.getcwd()}:/app 32-bit-linux python3"), + (64, "python3"), + (32, + f"docker run --platform linux/386 -t " + f"-v {os.getcwd()}:/app 32-bit-linux " + f"python3"), ]), MACOS: { - # Trying to use Python 3 compatible with PyInstaller according - # https://www.appveyor.com/docs/macos-images-software/#python - 64: "~/venv3.8/bin/python", + 64: "python3", } } @@ -59,7 +60,10 @@ def run(self, command): try: print(f"RUNNING: {command}") print("-" * 80) - subprocess.run(command, shell=True, check=True) + result = subprocess.run(command, shell=True, check=True, + stdout=subprocess.PIPE) + if result: + print(result.stdout.decode('utf-8').strip()) except subprocess.CalledProcessError as e: print(f"ERROR calling '{command}'") print("-" * 20) @@ -74,13 +78,12 @@ def install(self): # - build the 32 bit binary for linux on docker # - create dist/ path to circumvent permission errors if self.os == self.LINUX: - self.run("docker build -t 32-bit-linux -f Dockerfilei386 .") + self.run("docker build --platform linux/386 " + "-t 32-bit-linux -f Dockerfilei386 .") for arch, python in self.python: - self.run(f"{python} -m pip install setuptools") self.run(f"{python} -m pip install pyinstaller") - self.run(f"{python} -m pip install pytest") - self.run(f"{python} -m pip install -e .") + self.run(f"{python} -m pip install -r test_requirements.txt") def dist(self): """Runs Pyinstaller producing a binary for every platform arch.""" @@ -92,7 +95,7 @@ def dist(self): f" --distpath {build_path}") # There seems to be no way to tell pyinstaller the binary name. - # This leads to problems with appveyors artifact collector because + # This leads to problems with artifact collector because # every binary is named the same. # # Move them around so they can be picked up correctly @@ -119,7 +122,7 @@ def test(self): if __name__ == "__main__": if len(sys.argv) <= 1 or sys.argv[1] not in ['install', 'test', 'dist']: - print("usage: appveyor.py [install|test|dist]") + print("usage: binaries.py [install|test|dist]") sys.exit(-1) env = environment() diff --git a/test_requirements.txt b/test_requirements.txt new file mode 100644 index 00000000..b556e1c6 --- /dev/null +++ b/test_requirements.txt @@ -0,0 +1,8 @@ +pytest +pytest-cov +setuptools>=19.3 +Click>=8.0.2 +requests +packaging>=21.0 +dparse>=0.6.0 +ruamel.yaml>=0.17.21 From 718bb7b5e52708e490450b83dd013bcf388e7731 Mon Sep 17 00:00:00 2001 From: Yeison Vargas Date: Wed, 14 Sep 2022 16:38:06 -0500 Subject: [PATCH 05/16] Test action only in main branches --- .github/workflows/test-insecure.yml | 4 +++- .github/workflows/test-secure.yml | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-insecure.yml b/.github/workflows/test-insecure.yml index f5f29d28..8178539e 100644 --- a/.github/workflows/test-insecure.yml +++ b/.github/workflows/test-insecure.yml @@ -2,7 +2,9 @@ ######## set on the action step, and a further step to ensure the previous step failed (and actually fail if it _didn't_) name: Safety Action Insecure Tests -on: [push] +on: + push: + branches: [master, develop] jobs: ##### Auto mode tests diff --git a/.github/workflows/test-secure.yml b/.github/workflows/test-secure.yml index 357f2213..0bd29484 100644 --- a/.github/workflows/test-secure.yml +++ b/.github/workflows/test-secure.yml @@ -3,7 +3,9 @@ ######## fail, the pinned version might need to be updated. name: Safety Action Secure Tests -on: [push] +on: + push: + branches: [master, develop] jobs: ##### Auto mode tests From 51ad4ada3e4b89d3ec1d8b07f9847c095a638319 Mon Sep 17 00:00:00 2001 From: Yeison Vargas Date: Wed, 14 Sep 2022 16:49:01 -0500 Subject: [PATCH 06/16] Building image only on master branch --- .github/workflows/build.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 74832bc2..b0bd278d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,6 +1,8 @@ name: Safety Action Build And Publish -on: [push] +on: + push: + branches: [master] env: DOCKER_BUILDKIT: 1 From 815e812e0bffa895235cf8b3f0045ad57cb0055c Mon Sep 17 00:00:00 2001 From: Yeison Vargas Date: Wed, 14 Sep 2022 17:18:20 -0500 Subject: [PATCH 07/16] Fixes on WIN binaries generation --- binaries.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/binaries.py b/binaries.py index 0752f932..9f5550a7 100644 --- a/binaries.py +++ b/binaries.py @@ -29,10 +29,12 @@ def python(self): for arch, python in self.PYTHON_BINARIES[self.os].items(): yield arch, python + WIN_BASE_PATH = "C:\\hostedtoolcache\\windows\\Python\\3.10.7" + PYTHON_BINARIES = { WIN: { - 64: "C:\\Python39-x64\\python.exe", - 32: "C:\\Python39\\python.exe", + 64: f"{WIN_BASE_PATH}\\x64\\python.exe", # setup-python default + 32: f"{WIN_BASE_PATH}\\x86\\python.exe" }, # Order is important. If the 32 bit release gets built first, From de5ba12f87be9c9699d9ce93aa0b791355e0f55d Mon Sep 17 00:00:00 2001 From: Yeison Vargas Date: Wed, 14 Sep 2022 18:18:46 -0500 Subject: [PATCH 08/16] Uploading binaries depending on the right os --- .github/workflows/main.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 08c1ef7c..4b324189 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -48,25 +48,35 @@ jobs: - name: Producing Binaries run: python binaries.py dist - uses: actions/upload-artifact@v3 + if: ${{ matrix.os == 'windows-latest' }} with: name: safety-win-i686.exe path: dist/safety-win-i686.exe + if-no-files-found: error - uses: actions/upload-artifact@v3 + if: ${{ matrix.os == 'windows-latest' }} with: name: safety-win-x86_64.exe path: dist/safety-win-x86_64.exe + if-no-files-found: error - uses: actions/upload-artifact@v3 + if: ${{ matrix.os == 'ubuntu-latest' }} with: name: safety-linux-i686 path: dist/safety-linux-i686 + if-no-files-found: error - uses: actions/upload-artifact@v3 + if: ${{ matrix.os == 'ubuntu-latest' }} with: name: safety-linux-x86_64 path: dist/safety-linux-x86_64 + if-no-files-found: error - uses: actions/upload-artifact@v3 + if: ${{ matrix.os == 'macos-latest' }} with: name: safety-macos-x86_64 path: dist/safety-macos-x86_64 + if-no-files-found: error deploy-pypi: From 3a7f560f9e309d462f33130251d22395ced55331 Mon Sep 17 00:00:00 2001 From: Yeison Vargas Date: Wed, 14 Sep 2022 18:27:06 -0500 Subject: [PATCH 09/16] Upgrading dparse because a ReDoS security issue - PVE-2022-50571 - GHSA-8fg9-p83m-x5pq --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 4b1c7b31..e9ec4554 100644 --- a/setup.cfg +++ b/setup.cfg @@ -39,7 +39,7 @@ install_requires = Click>=8.0.2 requests packaging>=21.0 - dparse>=0.5.1 + dparse>=0.6.0 ruamel.yaml>=0.17.21 [options.entry_points] From 1aea37ecf112a4b702132575dfbf41db010c83c3 Mon Sep 17 00:00:00 2001 From: Yeison Vargas Date: Wed, 14 Sep 2022 18:37:22 -0500 Subject: [PATCH 10/16] Using dparse to read requirements and fixes for custom integrations --- safety/cli.py | 6 +-- safety/output_utils.py | 17 +++++-- safety/safety.py | 30 ++++++++--- safety/util.py | 113 ++++++++++------------------------------- 4 files changed, 67 insertions(+), 99 deletions(-) diff --git a/safety/cli.py b/safety/cli.py index cba8adde..58e73b2a 100644 --- a/safety/cli.py +++ b/safety/cli.py @@ -17,7 +17,7 @@ from safety.safety import get_packages, read_vulnerabilities, fetch_policy, post_results from safety.util import get_proxy_dict, get_packages_licenses, output_exception, \ MutuallyExclusiveOption, DependentOption, transform_ignore, SafetyPolicyFile, active_color_if_needed, \ - get_processed_options, get_safety_version, json_alias, bare_alias, SafetyContext + get_processed_options, get_safety_version, json_alias, bare_alias, SafetyContext, is_a_remote_mirror LOG = logging.getLogger(__name__) @@ -51,7 +51,7 @@ def cli(ctx, debug, telemetry, disable_optional_telemetry_data): help="API Key for pyup.io's vulnerability database. Can be set as SAFETY_API_KEY " "environment variable. Default: empty") @click.option("--db", default="", - help="Path to a local vulnerability database. Default: empty") + help="Path to a local or remote vulnerability database. Default: empty") @click.option("--full-report/--short-report", default=False, cls=MutuallyExclusiveOption, mutually_exclusive=["output", "json", "bare"], with_values={"output": ['json', 'bare'], "json": [True, False], "bare": [True, False]}, @@ -105,7 +105,7 @@ def check(ctx, key, db, full_report, stdin, files, cache, ignore, output, json, proxy_dictionary = get_proxy_dict(proxy_protocol, proxy_host, proxy_port) announcements = [] - if not db: + if not db or is_a_remote_mirror(db): LOG.info('Not local DB used, Getting announcements') announcements = safety.get_announcements(key=key, proxy=proxy_dictionary, telemetry=ctx.parent.telemetry) diff --git a/safety/output_utils.py b/safety/output_utils.py index 23bd5396..73738f20 100644 --- a/safety/output_utils.py +++ b/safety/output_utils.py @@ -1,12 +1,14 @@ import json import logging +import os import textwrap from datetime import datetime import click from safety.constants import RED, YELLOW -from safety.util import get_safety_version, Package, get_terminal_size, SafetyContext, build_telemetry_data, build_git_data +from safety.util import get_safety_version, Package, get_terminal_size, \ + SafetyContext, build_telemetry_data, build_git_data, is_a_remote_mirror LOG = logging.getLogger(__name__) @@ -485,16 +487,24 @@ def build_report_for_review_vuln_report(as_dict=False): def build_using_sentence(key, db): key_sentence = [] + custom_integration = os.environ.get('SAFETY_CUSTOM_INTEGRATION', + 'false').lower() == 'true' if key: key_sentence = [{'style': True, 'value': 'an API KEY'}, {'style': False, 'value': ' and the '}] db_name = 'PyUp Commercial' + elif db and custom_integration and is_a_remote_mirror(db): + return [] else: db_name = 'non-commercial' if db: - db_name = "local file {0}".format(db) + db_type = 'local file' + if is_a_remote_mirror(db): + db_type = 'remote URL' + + db_name = f"{db_type} {db}" database_sentence = [{'style': True, 'value': db_name + ' database'}] @@ -629,6 +639,7 @@ def get_report_brief_info(as_dict=False, report_type=1, **kwargs): brief_data['json_version'] = 1 using_sentence = build_using_sentence(key, db) + using_sentence_section = [nl] if not using_sentence else [nl] + [build_using_sentence(key, db)] scanned_count_sentence = build_scanned_count_sentence(packages) timestamp = [{'style': False, 'value': 'Timestamp '}, {'style': True, 'value': current_time}] @@ -638,7 +649,7 @@ def get_report_brief_info(as_dict=False, report_type=1, **kwargs): {'style': False, 'value': ' is scanning for '}, {'style': True, 'value': scanning_types.get(context.command, {}).get('name', '')}, {'style': True, 'value': '...'}] + safety_policy_used + audit_and_monitor, action_executed - ] + [nl] + scanned_items + [nl] + [using_sentence] + [scanned_count_sentence] + [timestamp] + ] + [nl] + scanned_items + using_sentence_section + [scanned_count_sentence] + [timestamp] brief_info.extend(additional_data) diff --git a/safety/safety.py b/safety/safety.py index 5140124c..d65b0edf 100644 --- a/safety/safety.py +++ b/safety/safety.py @@ -19,7 +19,7 @@ RequestTimeoutError, ServerError, MalformedDatabase) from .models import Vulnerability, CVE, Severity from .util import RequirementFile, read_requirements, Package, build_telemetry_data, sync_safety_context, SafetyContext, \ - validate_expiration_date + validate_expiration_date, is_a_remote_mirror session = requests.session() @@ -216,7 +216,7 @@ def fetch_database(full=False, key=False, db=False, cached=0, proxy=None, teleme db_name = "insecure_full.json" if full else "insecure.json" for mirror in mirrors: # mirror can either be a local path or a URL - if mirror.startswith("http://") or mirror.startswith("https://"): + if is_a_remote_mirror(mirror): data = fetch_database_url(mirror, db_name=db_name, key=key, cached=cached, proxy=proxy, telemetry=telemetry) else: data = fetch_database_file(mirror, db_name=db_name) @@ -509,7 +509,7 @@ def get_licenses(key=False, db_mirror=False, cached=0, proxy=None, telemetry=Tru for mirror in mirrors: # mirror can either be a local path or a URL - if mirror.startswith("http://") or mirror.startswith("https://"): + if is_a_remote_mirror(mirror): licenses = fetch_database_url(mirror, db_name=db_name, key=key, cached=cached, proxy=proxy, telemetry=telemetry) else: @@ -522,8 +522,6 @@ def get_licenses(key=False, db_mirror=False, cached=0, proxy=None, telemetry=Tru def get_announcements(key, proxy, telemetry=True): LOG.info('Getting announcements') - body = build_telemetry_data(telemetry=telemetry) - announcements = [] headers = {} @@ -531,11 +529,29 @@ def get_announcements(key, proxy, telemetry=True): headers["X-Api-Key"] = key url = f"{API_BASE_URL}announcements/" + method = 'post' + data = build_telemetry_data(telemetry=telemetry) + request_kwargs = {'headers': headers, 'proxies': proxy, 'timeout': 3} + data_keyword = 'json' + + source = os.environ.get('SAFETY_ANNOUNCEMENTS_URL', None) + + if source: + LOG.debug(f'Getting the announcement from a different source: {source}') + url = source + method = 'get' + data = { + 'telemetry': json.dumps(build_telemetry_data(telemetry=telemetry))} + data_keyword = 'params' + + request_kwargs[data_keyword] = data + request_kwargs['url'] = url - LOG.debug(f'Telemetry body sent: {body}') + LOG.debug(f'Telemetry data sent: {data}') try: - r = session.post(url=url, json=body, headers=headers, timeout=2, proxies=proxy) + request_func = getattr(session, method) + r = request_func(**request_kwargs) LOG.debug(r.text) except Exception as e: LOG.info('Unexpected but HANDLED Exception happened getting the announcements: %s', e) diff --git a/safety/util.py b/safety/util.py index 54d4b6f9..10a1dbab 100644 --- a/safety/util.py +++ b/safety/util.py @@ -1,3 +1,4 @@ +import json import logging import os import platform @@ -9,8 +10,10 @@ from typing import List import click +import dparse.parser from click import BadParameter from dparse.parser import setuptools_parse_requirements_backport as _parse_requirements +from dparse import parse, filetypes from packaging.utils import canonicalize_name from packaging.version import parse as parse_version from ruamel.yaml import YAML @@ -21,21 +24,12 @@ LOG = logging.getLogger(__name__) -def iter_lines(fh, lineno=0): - for line in fh.readlines()[lineno:]: - yield line +def is_a_remote_mirror(mirror): + return mirror.startswith("http://") or mirror.startswith("https://") -def parse_line(line): - if line.startswith('-e') or line.startswith('http://') or line.startswith('https://'): - if "#egg=" in line: - line = line.split("#egg=")[-1] - if ' --hash' in line: - line = line.split(" --hash")[0] - return _parse_requirements(line) - -def read_requirements(fh, resolve=False): +def read_requirements(fh, resolve=True): """ Reads requirements from a file like object and (optionally) from referenced files. :param fh: file like object to read from @@ -43,80 +37,27 @@ def read_requirements(fh, resolve=False): :return: generator """ is_temp_file = not hasattr(fh, 'name') - for num, line in enumerate(iter_lines(fh)): - line = line.strip() - if not line: - # skip empty lines - continue - if line.startswith('#') or \ - line.startswith('-i') or \ - line.startswith('--index-url') or \ - line.startswith('--extra-index-url') or \ - line.startswith('-f') or line.startswith('--find-links') or \ - line.startswith('--no-index') or line.startswith('--allow-external') or \ - line.startswith('--allow-unverified') or line.startswith('-Z') or \ - line.startswith('--always-unzip'): - # skip unsupported lines - continue - elif line.startswith('-r') or line.startswith('--requirement'): - # got a referenced file here, try to resolve the path - # if this is a tempfile, skip - if is_temp_file: - continue - - # strip away the recursive flag - prefixes = ["-r", "--requirement"] - filename = line.strip() - for prefix in prefixes: - if filename.startswith(prefix): - filename = filename[len(prefix):].strip() - - # if there is a comment, remove it - if " #" in filename: - filename = filename.split(" #")[0].strip() - req_file_path = os.path.join(os.path.dirname(fh.name), filename) - if resolve: - # recursively yield the resolved requirements - if os.path.exists(req_file_path): - with open(req_file_path) as _fh: - for req in read_requirements(_fh, resolve=True): - yield req - else: - yield RequirementFile(path=req_file_path) - else: - try: - parseable_line = line - # multiline requirements are not parseable - if "\\" in line: - parseable_line = line.replace("\\", "") - for next_line in iter_lines(fh, num + 1): - parseable_line += next_line.strip().replace("\\", "") - line += "\n" + next_line - if "\\" in next_line: - continue - break - req, = parse_line(parseable_line) - if len(req.specifier._specs) == 1 and \ - next(iter(req.specifier._specs))._spec[0] == "==": - yield Package(name=req.name, version=next(iter(req.specifier._specs))._spec[1], - found='temp_file' if is_temp_file else fh.name, insecure_versions=[], - secure_versions=[], latest_version=None, - latest_version_without_known_vulnerabilities=None, more_info_url=None) - else: - try: - fname = fh.name - except AttributeError: - fname = line - - click.secho( - "Warning: unpinned requirement '{req}' found in {fname}, " - "unable to check.".format(req=req.name, - fname=fname), - fg="yellow", - file=sys.stderr - ) - except ValueError: - continue + path = None + found = 'temp_file' + file_type = filetypes.requirements_txt + + if not is_temp_file: + path = fh.name + found = path + file_type = None + + dependency_file = parse(fh.read(), path=path, resolve=resolve, + file_type=file_type) + for dep in dependency_file.resolved_dependencies: + spec = next(iter(dep.specs))._spec + version = spec[1] + if spec[0] == '==': + yield Package(name=dep.name, version=version, + found=found, + insecure_versions=[], + secure_versions=[], latest_version=None, + latest_version_without_known_vulnerabilities=None, + more_info_url=None) def get_proxy_dict(proxy_protocol, proxy_host, proxy_port): From d91950b9d43fe0844428debafa05040823bf73c2 Mon Sep 17 00:00:00 2001 From: Yeison Vargas Date: Thu, 15 Sep 2022 08:44:12 -0500 Subject: [PATCH 11/16] Handling unpinned dep and make a warn about that --- safety/util.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/safety/util.py b/safety/util.py index 10a1dbab..375c4494 100644 --- a/safety/util.py +++ b/safety/util.py @@ -29,6 +29,12 @@ def is_a_remote_mirror(mirror): return mirror.startswith("http://") or mirror.startswith("https://") +def is_supported_by_parser(path): + supported_types = (".txt", ".in", ".yml", ".ini", "Pipfile", + "Pipfile.lock", "setup.cfg", "poetry.lock") + return path.endswith(supported_types) + + def read_requirements(fh, resolve=True): """ Reads requirements from a file like object and (optionally) from referenced files. @@ -41,7 +47,7 @@ def read_requirements(fh, resolve=True): found = 'temp_file' file_type = filetypes.requirements_txt - if not is_temp_file: + if not is_temp_file and is_supported_by_parser(fh.name): path = fh.name found = path file_type = None @@ -49,7 +55,17 @@ def read_requirements(fh, resolve=True): dependency_file = parse(fh.read(), path=path, resolve=resolve, file_type=file_type) for dep in dependency_file.resolved_dependencies: - spec = next(iter(dep.specs))._spec + try: + spec = next(iter(dep.specs))._spec + except StopIteration: + click.secho( + f"Warning: unpinned requirement '{dep.name}' found in {path}, " + "unable to check.", + fg="yellow", + file=sys.stderr + ) + return + version = spec[1] if spec[0] == '==': yield Package(name=dep.name, version=version, From c95eeca5e2b46dd3ad51fee891bb113f93544050 Mon Sep 17 00:00:00 2001 From: Yeison Vargas Date: Thu, 15 Sep 2022 11:37:50 -0500 Subject: [PATCH 12/16] Adding more logging --- safety/util.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/safety/util.py b/safety/util.py index 375c4494..8ff8a70c 100644 --- a/safety/util.py +++ b/safety/util.py @@ -10,9 +10,7 @@ from typing import List import click -import dparse.parser from click import BadParameter -from dparse.parser import setuptools_parse_requirements_backport as _parse_requirements from dparse import parse, filetypes from packaging.utils import canonicalize_name from packaging.version import parse as parse_version @@ -48,12 +46,17 @@ def read_requirements(fh, resolve=True): file_type = filetypes.requirements_txt if not is_temp_file and is_supported_by_parser(fh.name): + LOG.debug('temp or not a compatible file') path = fh.name found = path file_type = None + LOG.debug(f'Path: {path}') + LOG.debug(f'File Type: {file_type}') + LOG.debug('Trying to parse file using dparse...') dependency_file = parse(fh.read(), path=path, resolve=resolve, file_type=file_type) + LOG.debug(f'Parsed, dependencies: {[dep.serialize() for dep in dependency_file.resolved_dependencies]}') for dep in dependency_file.resolved_dependencies: try: spec = next(iter(dep.specs))._spec From af2776f2c2275f312c9948e49f84ceadf0dcef2a Mon Sep 17 00:00:00 2001 From: Yeison Vargas Date: Thu, 15 Sep 2022 11:46:42 -0500 Subject: [PATCH 13/16] Debug log level on binaries testing --- binaries.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/binaries.py b/binaries.py index 9f5550a7..df320950 100644 --- a/binaries.py +++ b/binaries.py @@ -118,7 +118,7 @@ def test(self): Runs tests for every available arch on the current platform. """ for arch, python in self.python: - self.run(f"{python} -m pytest") + self.run(f"{python} -m pytest --log-level=DEBUG") if __name__ == "__main__": From 5b953cf0462d7cc8e94a45b5ec17ccb8a4f2043e Mon Sep 17 00:00:00 2001 From: Yeison Vargas Date: Thu, 15 Sep 2022 11:57:40 -0500 Subject: [PATCH 14/16] More logging... --- safety/util.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/safety/util.py b/safety/util.py index 8ff8a70c..b9673458 100644 --- a/safety/util.py +++ b/safety/util.py @@ -46,7 +46,7 @@ def read_requirements(fh, resolve=True): file_type = filetypes.requirements_txt if not is_temp_file and is_supported_by_parser(fh.name): - LOG.debug('temp or not a compatible file') + LOG.debug('not temp and a compatible file') path = fh.name found = path file_type = None @@ -54,7 +54,9 @@ def read_requirements(fh, resolve=True): LOG.debug(f'Path: {path}') LOG.debug(f'File Type: {file_type}') LOG.debug('Trying to parse file using dparse...') - dependency_file = parse(fh.read(), path=path, resolve=resolve, + content = fh.read() + LOG.debug(f'Content: {content}') + dependency_file = parse(content, path=path, resolve=resolve, file_type=file_type) LOG.debug(f'Parsed, dependencies: {[dep.serialize() for dep in dependency_file.resolved_dependencies]}') for dep in dependency_file.resolved_dependencies: From 6423099ab284f8577f2d33f7d3e25a5f74027389 Mon Sep 17 00:00:00 2001 From: Yeison Vargas Date: Thu, 15 Sep 2022 12:07:11 -0500 Subject: [PATCH 15/16] Including the dep file in the logging --- safety/util.py | 1 + 1 file changed, 1 insertion(+) diff --git a/safety/util.py b/safety/util.py index b9673458..9d805bc0 100644 --- a/safety/util.py +++ b/safety/util.py @@ -58,6 +58,7 @@ def read_requirements(fh, resolve=True): LOG.debug(f'Content: {content}') dependency_file = parse(content, path=path, resolve=resolve, file_type=file_type) + LOG.debug(f'Dependency file: {dependency_file.serialize()}') LOG.debug(f'Parsed, dependencies: {[dep.serialize() for dep in dependency_file.resolved_dependencies]}') for dep in dependency_file.resolved_dependencies: try: From 003eb3a7db32a473b6ee23c7479e151593de4224 Mon Sep 17 00:00:00 2001 From: Yeison Vargas Date: Mon, 19 Sep 2022 15:40:30 -0500 Subject: [PATCH 16/16] Using the latest dparse version --- setup.cfg | 2 +- test_requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index e9ec4554..35ef688a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -39,7 +39,7 @@ install_requires = Click>=8.0.2 requests packaging>=21.0 - dparse>=0.6.0 + dparse>=0.6.2 ruamel.yaml>=0.17.21 [options.entry_points] diff --git a/test_requirements.txt b/test_requirements.txt index b556e1c6..60167d4e 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -4,5 +4,5 @@ setuptools>=19.3 Click>=8.0.2 requests packaging>=21.0 -dparse>=0.6.0 +dparse>=0.6.2 ruamel.yaml>=0.17.21