diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 2ebf445..f3eedc9 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,16 +1,31 @@ version: 2 updates: -- package-ecosystem: docker - directory: "/" - schedule: - interval: daily - pull-request-branch-name: - separator: "-" - open-pull-requests-limit: 99 -- package-ecosystem: pip - directory: "/" - schedule: - interval: daily - pull-request-branch-name: - separator: "-" - open-pull-requests-limit: 99 + - package-ecosystem: pip + directory: '/' + schedule: + interval: 'daily' + pull-request-branch-name: + separator: '-' + open-pull-requests-limit: 99 + target-branch: 'develop' + commit-message: + prefix: 'chore' + - package-ecosystem: 'docker' + directory: '/' + schedule: + interval: 'daily' + pull-request-branch-name: + separator: '-' + open-pull-requests-limit: 99 + commit-message: + prefix: 'chore' + target-branch: 'develop' + - package-ecosystem: 'github-actions' + directory: '/' + schedule: + interval: 'daily' + commit-message: + prefix: 'chore' + pull-request-branch-name: + separator: '-' + target-branch: 'develop' diff --git a/.github/herald_rule_label_pr_breaking.json b/.github/herald_rule_label_pr_breaking.json new file mode 100644 index 0000000..708a45b --- /dev/null +++ b/.github/herald_rule_label_pr_breaking.json @@ -0,0 +1,5 @@ +{ + "action": "label", + "eventJsonPath": "$..[?(@.body.match(/BREAKING.*?/))].body", + "labels": "breaking" +} diff --git a/.github/herald_rule_label_pr_bug.json b/.github/herald_rule_label_pr_bug.json new file mode 100644 index 0000000..9462093 --- /dev/null +++ b/.github/herald_rule_label_pr_bug.json @@ -0,0 +1,5 @@ +{ + "action": "label", + "eventJsonPath": "$..[?(@.title.match(/^fix.*?/))].title", + "labels": "bug" +} diff --git a/.github/herald_rule_label_pr_documentation.json b/.github/herald_rule_label_pr_documentation.json new file mode 100644 index 0000000..dcaacfe --- /dev/null +++ b/.github/herald_rule_label_pr_documentation.json @@ -0,0 +1,5 @@ +{ + "action": "label", + "eventJsonPath": "$..[?(@.title.match(/^docs.*?/))].title", + "labels": "documentation" +} diff --git a/.github/herald_rule_label_pr_enhancement.json b/.github/herald_rule_label_pr_enhancement.json new file mode 100644 index 0000000..ad0d611 --- /dev/null +++ b/.github/herald_rule_label_pr_enhancement.json @@ -0,0 +1,5 @@ +{ + "action": "label", + "eventJsonPath": "$..[?(@.title.match(/^feat.*?/))].title", + "labels": "enhancement" +} diff --git a/.github/workflows/auto-release.yaml b/.github/workflows/auto-release.yaml new file mode 100644 index 0000000..202e754 --- /dev/null +++ b/.github/workflows/auto-release.yaml @@ -0,0 +1,54 @@ +name: 'Create Release' + +on: + push: + branches: ['main'] + +jobs: + create_release: + runs-on: 'ubuntu-18.04' + name: 'Create Release' + steps: + - name: 'Checkout' + uses: 'actions/checkout@v2' + with: + fetch-depth: 0 + - name: 'Get Latest Tag' + id: 'previoustag' + uses: 'WyriHaximus/github-action-get-previous-tag@v1.0.1' + - name: 'Bump version and push tag' + id: 'tag_version' + uses: 'mathieudutour/github-tag-action@v5.2' + with: + github_token: '${{ secrets.PAT_REPO_ONLY }}' + release_branches: 'main' + default_bump: 'patch' + - name: 'Generate incremental changelog' + uses: 'charmixer/auto-changelog-action@v1.2' + with: + token: '${{ secrets.GITHUB_TOKEN }}' + release_branch: 'main' + since_tag: '${{ steps.previoustag.outputs.tag }}' + output: 'release.md' + - name: 'Read incremental changelog' + id: 'changelog' + uses: 'juliangruber/read-file-action@v1' + with: + path: './release.md' + - name: 'Create a GitHub release' + uses: 'actions/create-release@v1' + env: + GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}' + with: + tag_name: '${{ steps.tag_version.outputs.new_tag }}' + release_name: '${{ steps.tag_version.outputs.new_tag }}' + body: '${{ steps.changelog.outputs.content }}' + - name: 'Generate changelog' + uses: 'charmixer/auto-changelog-action@v1.2' + with: + token: '${{ secrets.GITHUB_TOKEN }}' + release_branch: 'main' + - uses: 'stefanzweifel/git-auto-commit-action@v4' + with: + commit_message: 'chore: Generating CHANGELOG.md for ${{ steps.tag_version.outputs.new_tag }}' + file_pattern: 'CHANGELOG.md' diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..022c13d --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,70 @@ +name: 'CI' + +on: + push: + branches: ['**'] + +env: + CONTAINER_NAME: 'handbrake-job-creator' + +jobs: + lint: + runs-on: 'ubuntu-20.04' + name: 'Python Lint' + steps: + - uses: 'actions/checkout@v2' + - uses: 'actions/setup-python@v2' + with: + python-version: '3.9' + - name: 'Install dependencies' + run: | + echo "Installing CI requirements" + pip install --upgrade --user pip flake8 + echo "Installing project requirements" + pip install -r requirements.txt + - name: 'Flake8 :allthethings:' + run: 'python -m flake8' + test: + runs-on: 'ubuntu-20.04' + name: 'Python Tests' + steps: + - uses: 'actions/checkout@v2' + - uses: 'actions/setup-python@v2' + with: + python-version: '3.9' + - name: 'Install dependencies' + run: | + echo "Installing CI requirements" + pip install --upgrade --user pip pytest coverage + echo "Installing project requirements" + pip install -r requirements.txt + - name: 'Run tests' + run: 'python -m coverage run --branch --source=app -m pytest' + - name: 'Display coverage report' + run: 'python -m coverage report --fail-under 80 -m' + container_build: + needs: ['lint', 'test'] + name: 'Build and Push Docker Container' + runs-on: 'ubuntu-20.04' + steps: + - uses: 'actions/checkout@v2' + - name: 'Build the Docker image' + run: 'docker build . -t chrisjohnson00/${CONTAINER_NAME}:${GITHUB_REF##*/}' + - name: 'Login to docker hub' + run: 'docker login --username=chrisjohnson00 --password=${{ secrets.DOCKER_HUB_PASSWORD }}' + - name: 'Push the Docker image' + run: 'docker push chrisjohnson00/${CONTAINER_NAME}:${GITHUB_REF##*/}' + actionslint: + name: 'Actions Yaml Lint' + runs-on: 'ubuntu-20.04' + steps: + - name: 'Checkout' + uses: 'actions/checkout@v2' + - name: 'Actions Yaml Lint' + uses: 'karancode/yamllint-github-action@v2.0.0' + with: + yamllint_file_or_dir: '.github/workflows' + yamllint_comment: 'true' + yamllint_config_datapath: '{"extends":"default","rules":{"line-length":{"max":360,"level":"warning"},"quoted-strings":{"quote-type":"single","required":true,"level":"error"},"truthy":{"check-keys":false},"document-start":{"present":false}}}' + env: + GITHUB_ACCESS_TOKEN: '${{ secrets.GITHUB_TOKEN }}' diff --git a/.github/workflows/pr_checks.yaml b/.github/workflows/pr_checks.yaml new file mode 100644 index 0000000..dc43c0f --- /dev/null +++ b/.github/workflows/pr_checks.yaml @@ -0,0 +1,42 @@ +name: 'PR Standards Check' +on: + pull_request: + types: + - 'opened' + - 'edited' + - 'reopened' + - 'synchronize' + +jobs: + check-pr-title: + name: 'PR Title' + runs-on: 'ubuntu-18.04' + steps: + - uses: 'actions/checkout@v2' + - name: 'Install Dependencies' + run: | + npm install @commitlint/config-conventional + echo "module.exports = {extends: ['@commitlint/config-conventional'], rules: { 'subject-case': [2, 'always', ['sentence-case','lower-case', 'start-case']], 'header-max-length': [2, 'always', 120], 'body-max-line-length': [0, 'always', 120] }};" > commitlint.config.js + - uses: 'JulienKode/pull-request-name-linter-action@v0.2.0' + check-pr-commits: + name: 'PR Commit Messages' + runs-on: 'ubuntu-18.04' + steps: + - uses: 'actions/checkout@v2' + with: + fetch-depth: 0 + - name: 'Configure commit lint' + run: | + echo "module.exports = {extends: ['@commitlint/config-conventional'], rules: { 'subject-case': [2, 'always', ['sentence-case','lower-case', 'start-case']], 'header-max-length': [2, 'always', 120], 'body-max-line-length': [0, 'always', 120] }};" > commitlint.config.js + - uses: 'wagoid/commitlint-github-action@v3.0.6' + apply-labels: + name: 'PR Labels' + runs-on: 'ubuntu-18.04' + steps: + - uses: 'actions/checkout@v2' + - name: 'Apply Herald rules' + uses: 'gagoar/use-herald-action@v2.3.0' + with: + GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}' + rulesLocation: '.github/herald_rule_*.json' + base: 'main' diff --git a/.github/workflows/push_latest.yaml b/.github/workflows/push_latest.yaml new file mode 100644 index 0000000..3658bcd --- /dev/null +++ b/.github/workflows/push_latest.yaml @@ -0,0 +1,21 @@ +name: 'Push Latest Tag' + +on: + push: + tags: + - 'v**' + +env: + CONTAINER_NAME: 'handbrake-job-creator' + +jobs: + push_latest: + runs-on: 'ubuntu-18.04' + steps: + - uses: 'actions/checkout@v2' + - name: 'Build the Docker image' + run: 'docker build . -t chrisjohnson00/${CONTAINER_NAME}:latest' + - name: 'Login to docker hub' + run: 'docker login --username=chrisjohnson00 --password=${{ secrets.DOCKER_HUB_PASSWORD }}' + - name: 'Push the Docker image' + run: 'docker push chrisjohnson00/${CONTAINER_NAME}:latest' diff --git a/.github/workflows/push_release_container.yaml b/.github/workflows/push_release_container.yaml new file mode 100644 index 0000000..0b71f77 --- /dev/null +++ b/.github/workflows/push_release_container.yaml @@ -0,0 +1,20 @@ +name: 'Push release version tag' + +on: + push: + tags: ['**'] + +env: + CONTAINER_NAME: 'handbrake-job-creator' + +jobs: + build_container: + runs-on: 'ubuntu-18.04' + steps: + - uses: 'actions/checkout@v2' + - name: 'Build the Docker image' + run: 'docker build . -t chrisjohnson00/${CONTAINER_NAME}:${GITHUB_REF##*/}' + - name: 'Login to docker hub' + run: 'docker login --username=chrisjohnson00 --password=${{ secrets.DOCKER_HUB_PASSWORD }}' + - name: 'Push the Docker image' + run: 'docker push chrisjohnson00/${CONTAINER_NAME}:${GITHUB_REF##*/}' diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 0ad59b7..0000000 --- a/.travis.yml +++ /dev/null @@ -1,26 +0,0 @@ -language: bash -arch: - - amd64 -services: - - docker -jobs: - include: - - stage: build branch - script: - - set -e - - docker build -t chrisjohnson00/handbrake-job-creator -f Dockerfile . - - docker run chrisjohnson00/handbrake-job-creator python -m flake8 - - docker run chrisjohnson00/handbrake-job-creator python -m pytest - - docker login --username=chrisjohnson00 --password=$DOCKER_HUB_PASSWORD - - docker tag chrisjohnson00/handbrake-job-creator chrisjohnson00/handbrake-job-creator:$TRAVIS_BRANCH - - docker push chrisjohnson00/handbrake-job-creator:$TRAVIS_BRANCH - if: tag is blank - - stage: build tag - script: - - set -e - - docker build -t chrisjohnson00/handbrake-job-creator -f Dockerfile . - - docker login --username=chrisjohnson00 --password=$DOCKER_HUB_PASSWORD - - docker tag chrisjohnson00/handbrake-job-creator chrisjohnson00/handbrake-job-creator:$TRAVIS_TAG - - docker push chrisjohnson00/handbrake-job-creator:$TRAVIS_TAG - - docker push chrisjohnson00/handbrake-job-creator - if: tag IS present diff --git a/app.py b/app.py index 6081227..af23254 100644 --- a/app.py +++ b/app.py @@ -21,7 +21,6 @@ def main(): directory = get_watch_path() move_path = get_move_path() namespace = get_namespace() - encoding_profile = get_encoding_profile() file_discovered_metrics = Gauge('handbrake_job_creator_files_in_process', 'Job Creator Found A File', labelnames=["type", "quality"]) files_to_process_metrics = Gauge('handbrake_job_creator_files_to_process', 'Job Creator Found Some Files', @@ -62,6 +61,7 @@ def main(): batch_v1 = client.BatchV1Api() if job_exists(batch_v1, generate_job_name(file), namespace): print("INFO: Done with {} did not create any new job".format(filename), flush=True) + # @TODO remove the file from encoding_queue! else: output_filename = filename # (with 1080p in the name), it will rename it to 720p @@ -69,8 +69,7 @@ def main(): replace_value = get_file_name_replace_value() if find_value and replace_value: output_filename = filename.replace(find_value, replace_value) - job = create_job_object(generate_job_name(file), filename, output_filename, encoding_profile, - file_size) + job = create_job_object(generate_job_name(file), filename, output_filename, file_size) create_job(batch_v1, job, namespace) # @TODO move the file back if the create_job call fails print("INFO: Done with {}".format(filename), flush=True) @@ -94,11 +93,11 @@ def get_container_version(): def get_watch_path(): - return "/watch" + return get_config("WATCH_PATH") def get_move_path(): - return "/move" + return get_config("MOVE_PATH") def get_nfs_server(): @@ -129,10 +128,6 @@ def get_namespace(): return get_config("JOB_NAMESPACE") # expected as an env value only -def get_encoding_profile(): - return get_config("JOB_PROFILE") # expected as an env value only - - def get_quality_level(): quality = get_config("QUALITY") # expected as an env value only if quality not in ['720p', '1080p', '4k']: @@ -168,14 +163,13 @@ def get_config(key, config_path=CONFIG_PATH): return data['Value'].decode("utf-8") -def create_job_object(job_name, input_filename, output_filename, encoding_profile, file_size): +def create_job_object(job_name, input_filename, output_filename, file_size): # Configureate Pod template container container = client.V1Container( name=job_name, image="chrisjohnson00/handbrakecli:{}".format(get_container_version()), image_pull_policy=get_job_container_pull_policy(), - command=["./wrapper.sh", "{}".format(input_filename), "{}".format(output_filename), - "{}".format(encoding_profile)], + command=["python3", "/wrapper.py", "{}".format(input_filename), "{}".format(output_filename)], volume_mounts=[ client.V1VolumeMount( mount_path="/input", @@ -194,6 +188,10 @@ def create_job_object(job_name, input_filename, output_filename, encoding_profil client.V1EnvVar( name="JOB_TYPE", value=get_job_type() + ), + client.V1EnvVar( + name="CONSUL_HTTP_ADDR", + value=get_config('CONSUL_HTTP_ADDR') ) ] ) diff --git a/app/tests/test_utils.py b/app/tests/test_utils.py index 45bac99..9c58583 100644 --- a/app/tests/test_utils.py +++ b/app/tests/test_utils.py @@ -13,3 +13,10 @@ def test_cleanup_job_suffix_period(): expected_result = "thepantypiatapolarizationhdtv1080p" result = cleanup_job_suffix(input_with_period_character) assert result == expected_result + + +def test_cleanup_job_suffix_underscore(): + input_with_bad_character = "The_Panty_Piata_Polarization_HDTV-1080p" + expected_result = "thepantypiatapolarizationhdtv1080p" + result = cleanup_job_suffix(input_with_bad_character) + assert result == expected_result diff --git a/app/utils.py b/app/utils.py index 037e6ea..4033fb1 100644 --- a/app/utils.py +++ b/app/utils.py @@ -5,4 +5,5 @@ def cleanup_job_suffix(input_string): # keep only word character, removing all non-word chars job_suffix = re.sub(r"\W", "", input_string.lower(), flags=re.I) job_suffix = re.sub(r"[à-ú]|[À-Ú]", "", job_suffix, flags=re.I) + job_suffix = job_suffix.replace("_", "") return job_suffix diff --git a/requirements.txt b/requirements.txt index 15b7842..bd881ba 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,6 +6,7 @@ flake8==3.8.4 google-auth==1.24.0 idna==2.10 importlib-metadata==3.3.0 +iniconfig==1.1.1 kubernetes==12.0.1 mccabe==0.6.1 more-itertools==8.6.0 @@ -27,7 +28,9 @@ requests==2.25.1 requests-oauthlib==1.3.0 rsa==4.6 six==1.15.0 -urllib3==1.26.2 +toml==0.10.2 +typing-extensions==3.7.4.3 +urllib3==1.26.4 wcwidth==0.2.5 websocket-client==0.57.0 zipp==3.4.0