From f40d5557880660863c69d146e7ed6ae585932936 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Sun, 20 Jun 2021 16:31:28 +0200 Subject: [PATCH] Adds automated generation of provider issue to track test progress (#16419) Allows to automatically generate draft of the issue which can be used to track progress of testing released providers. --- .github/workflows/ci.yml | 5 +- BREEZE.rst | 4 + breeze | 17 ++- breeze-complete | 2 +- dev/README_RELEASE_PROVIDER_PACKAGES.md | 24 +++- .../PROVIDER_ISSUE_TEMPLATE.md.jinja2 | 35 +++++ .../prepare_provider_packages.py | 127 +++++++++++++++++- .../ci/docker-compose/forward-credentials.yml | 2 + scripts/ci/libraries/_runs.sh | 2 + .../run_prepare_provider_documentation.sh | 27 +++- setup.py | 1 + 11 files changed, 233 insertions(+), 13 deletions(-) create mode 100644 dev/provider_packages/PROVIDER_ISSUE_TEMPLATE.md.jinja2 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e097556ad9795..8f233fc517f87 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -320,8 +320,6 @@ jobs: uses: actions/checkout@v2 with: persist-credentials: false - # Needs to fetch all history to be able to verify changelog generation - fetch-depth: 0 - name: "Setup python" uses: actions/setup-python@v2 with: @@ -522,6 +520,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" VERSION_SUFFIX_FOR_PYPI: ".dev0" GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }} NON_INTERACTIVE: "true" + GENERATE_PROVIDERS_ISSUE: "true" if: needs.build-info.outputs.image-build == 'true' && needs.build-info.outputs.default-branch == 'main' steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" @@ -569,6 +568,8 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}} VERSION_SUFFIX_FOR_PYPI: ".dev0" GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }} + NON_INTERACTIVE: "true" + GENERATE_PROVIDERS_ISSUE: "true" if: needs.build-info.outputs.image-build == 'true' && needs.build-info.outputs.default-branch == 'main' steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" diff --git a/BREEZE.rst b/BREEZE.rst index eaa3f676be48a..ec939980608d7 100644 --- a/BREEZE.rst +++ b/BREEZE.rst @@ -2141,6 +2141,10 @@ This is the current syntax for `./breeze <./breeze>`_: Runs the command in non-interactive mode. + --generate-providers-issue + + Generate providers issue that should be created. + -v, --verbose Show verbose information about executed docker, kind, kubectl, helm commands. Useful for debugging - when you run breeze with --verbose flags you will be able to see the commands diff --git a/breeze b/breeze index e01e2e434c3a4..d358b682997e7 100755 --- a/breeze +++ b/breeze @@ -1260,6 +1260,13 @@ function breeze::parse_arguments() { echo shift ;; + --generate-providers-issue) + export GENERATE_PROVIDERS_ISSUE="true" + export FORWARD_CREDENTIALS="true" + echo "Generating providers issue (includes forwarding credentials)" + echo + shift + ;; --installation-method) export AIRFLOW_INSTALLATION_METHOD="${2}" echo "Airflow installation method: ${AIRFLOW_INSTALLATION_METHOD}" @@ -1866,7 +1873,7 @@ ${CMDNAME} prepare-provider-documentation [FLAGS] [PACKAGE_ID ...] Flags: $(breeze::flag_version_suffix) $(breeze::flag_packages) -$(breeze::flag_non_interactive) +$(breeze::prepare_providers_documentation) $(breeze::flag_verbosity) " readonly DETAILED_USAGE_PREPARE_PROVIDER_DOCUMENTATION @@ -2596,17 +2603,21 @@ ${FORMATTED_PACKAGE_FORMATS} ####################################################################################################### # -# Prints flags for non-interactive run +# Prints flags for prepare-providers-documentation # # Outputs: # Flag information. ####################################################################################################### -function breeze::flag_non_interactive() { +function breeze::prepare_providers_documentation() { echo " --non-interactive Runs the command in non-interactive mode. +--generate-providers-issue + + Generate providers issue that should be created. + " } diff --git a/breeze-complete b/breeze-complete index 162d7478f20ed..5c28da876b934 100644 --- a/breeze-complete +++ b/breeze-complete @@ -189,7 +189,7 @@ additional-extras: additional-python-deps: disable-pypi-when-building skip-insta dev-apt-deps: additional-dev-apt-deps: dev-apt-command: additional-dev-apt-command: additional-dev-apt-env: runtime-apt-deps: additional-runtime-apt-deps: runtime-apt-command: additional-runtime-apt-command: additional-runtime-apt-env: load-default-connections load-example-dags -use-packages-from-dist no-rbac-ui package-format: upgrade-to-newer-dependencies installation-method: continue-on-pip-check-failure non-interactive +use-packages-from-dist no-rbac-ui package-format: upgrade-to-newer-dependencies installation-method: continue-on-pip-check-failure non-interactive generate-providers-issue use-airflow-version: cleanup-docker-context-files test-type: preserve-volumes dry-run-docker diff --git a/dev/README_RELEASE_PROVIDER_PACKAGES.md b/dev/README_RELEASE_PROVIDER_PACKAGES.md index 619083c4a100a..dcb729a3cac49 100644 --- a/dev/README_RELEASE_PROVIDER_PACKAGES.md +++ b/dev/README_RELEASE_PROVIDER_PACKAGES.md @@ -31,6 +31,7 @@ - [Publish the Regular convenience package to PyPI](#publish-the-regular-convenience-package-to-pypi) - [Add tags in git](#add-tags-in-git) - [Prepare documentation](#prepare-documentation) + - [Prepare issue in GitHub to keep status of testing](#prepare-issue-in-github-to-keep-status-of-testing) - [Prepare voting email for Providers release candidate](#prepare-voting-email-for-providers-release-candidate) - [Verify the release by PMC members](#verify-the-release-by-pmc-members) - [Verify by Contributors](#verify-by-contributors) @@ -109,6 +110,17 @@ are updated, run it in non-interactive mode: ./breeze --non-interactive prepare-provider-documentation [packages] ``` +When you run the command and documentation generation is successful you will get a command that you can run to +create GitHub issue where you will be tracking status of tests for the providers you release. + +You can also trigger automated execution of the issue by running: + +```shell script +./breeze --non-interactive --generate-providers-issue prepare-provider-documentation [packages] +``` + +Once you release packages, you should create the issue with the content specified and link to it in +the email sent to the devlist. ## Build provider packages for SVN apache upload @@ -338,6 +350,11 @@ git commit -m "Add documentation for packages - $(date "+%Y-%m-%d%n")" git push --set-upstream origin "${branch}" ``` +## Prepare issue in GitHub to keep status of testing + +Create GitHub issue with the content generated via prepare-provider-documentation or manual +execution of the script above. You will use link to that issue in the next step. + ## Prepare voting email for Providers release candidate Make sure the packages are in https://dist.apache.org/repos/dist/dev/airflow/providers/ @@ -350,7 +367,7 @@ subject: ```shell script cat < + Airflow Providers are available at: https://dist.apache.org/repos/dist/dev/airflow/providers/ @@ -399,6 +416,9 @@ Please note that the version number excludes the 'rcX' string. This will allow us to rename the artifact without modifying the artifact checksums when we actually release. +The status of testing the providers by the community is kept here: + + You can find packages as well as detailed changelog following the below links: diff --git a/dev/provider_packages/PROVIDER_ISSUE_TEMPLATE.md.jinja2 b/dev/provider_packages/PROVIDER_ISSUE_TEMPLATE.md.jinja2 new file mode 100644 index 0000000000000..a436bcbe84b7e --- /dev/null +++ b/dev/provider_packages/PROVIDER_ISSUE_TEMPLATE.md.jinja2 @@ -0,0 +1,35 @@ +I have a kind request for all the contributors to the latest provider packages release. +Could you help us to test the RC versions of the providers and let us know in the comment, +if the issue is addressed there. + +## Providers that need testing + +Those are providers that require testing as there were some substantial changes introduced: + +{% for provider_id, provider_pr_info in interesting_providers.items() %} +### Provider [{{ provider_id }}: {{ provider_pr_info.provider_details.versions[0] }}{{ suffix }}](https://pypi.org/project/{{ provider_pr_info.provider_details.pypi_package_name }}/{{ provider_pr_info.provider_details.versions[0] }}{{ suffix }}) +{%- for pr in provider_pr_info.pr_list %} + - [ ] [{{ pr.title }} (#{{ pr.number }})]({{ pr.html_url }}): @{{ pr.user.login }} +{%- endfor %} +{%- endfor %} + +## Providers that do not need testing + +Those are providers that were either doc-only or had changes that do not require testing. + +{% for provider_id, provider_pr_info in non_interesting_providers.items() %} +* Provider [{{ provider_id }}: {{ provider_pr_info.provider_details.versions[0] }}{{ suffix }}](https://pypi.org/project/{{ provider_pr_info.provider_details.pypi_package_name }}/{{ provider_pr_info.provider_details.versions[0] }}{{ suffix }}) +{%- endfor %} + + diff --git a/dev/provider_packages/prepare_provider_packages.py b/dev/provider_packages/prepare_provider_packages.py index fcb8467d1a437..c27c88bae2800 100755 --- a/dev/provider_packages/prepare_provider_packages.py +++ b/dev/provider_packages/prepare_provider_packages.py @@ -44,9 +44,11 @@ import click import jsonschema import yaml +from github import Github, PullRequest, UnknownObjectException from packaging.version import Version from rich import print from rich.console import Console +from rich.progress import Progress from rich.syntax import Syntax try: @@ -155,6 +157,7 @@ def cli(): ) argument_package_id = click.argument('package_id') argument_changelog_files = click.argument('changelog_files', nargs=-1) +argument_package_ids = click.argument('package_ids', nargs=-1) @contextmanager @@ -200,6 +203,7 @@ class VerifiedEntities(NamedTuple): class ProviderPackageDetails(NamedTuple): provider_package_id: str full_package_name: str + pypi_package_name: str source_provider_package_path: str documentation_provider_package_path: str provider_description: str @@ -1436,6 +1440,7 @@ def get_provider_details(provider_package_id: str) -> ProviderPackageDetails: return ProviderPackageDetails( provider_package_id=provider_package_id, full_package_name=f"airflow.providers.{provider_package_id}", + pypi_package_name=f"apache-airflow-providers-{provider_package_id.replace('.', '-')}", source_provider_package_path=get_source_package_path(provider_package_id), documentation_provider_package_path=get_documentation_package_path(provider_package_id), provider_description=provider_info['description'], @@ -1797,7 +1802,7 @@ def get_all_providers() -> List[str]: return list(PROVIDERS_REQUIREMENTS.keys()) -def verify_provider_package(provider_package_id: str) -> None: +def verify_provider_package(provider_package_id: str) -> str: """ Verifies if the provider package is good. :param provider_package_id: package id to verify @@ -2246,6 +2251,7 @@ def get_package_from_changelog(changelog_path: str): @option_git_update @option_verbose def update_changelogs(changelog_files: List[str], git_update: bool, verbose: bool): + """Updates changelogs for multiple packages.""" if git_update: make_sure_remote_apache_exists_and_fetch(git_update, verbose) for changelog_file in changelog_files: @@ -2253,6 +2259,125 @@ def update_changelogs(changelog_files: List[str], git_update: bool, verbose: boo _update_changelog(package_id=package_id, verbose=verbose) +def get_prs_for_package(package_id: str) -> List[int]: + pr_matcher = re.compile(r".*\(#([0-9]*)\)``$") + verify_provider_package(package_id) + changelog_path = verify_changelog_exists(package_id) + provider_details = get_provider_details(package_id) + current_release_version = provider_details.versions[0] + prs = [] + with open(changelog_path) as changelog_file: + changelog_lines = changelog_file.readlines() + extract_prs = False + skip_line = False + for line in changelog_lines: + if skip_line: + # Skip first "....." header + skip_line = False + continue + if line.strip() == current_release_version: + extract_prs = True + skip_line = True + continue + if extract_prs: + if all(c == '.' for c in line): + # Header for next version reached + break + if line.startswith('.. Below changes are excluded from the changelog'): + # The reminder of PRs is not important skipping it + break + match_result = pr_matcher.match(line.strip()) + if match_result: + prs.append(int(match_result.group(1))) + return prs + + +class ProviderPRInfo(NamedTuple): + provider_details: ProviderPackageDetails + pr_list: List[PullRequest.PullRequest] + + +@cli.command() +@click.option('--github-token', envvar='GITHUB_TOKEN') +@click.option('--suffix', default='rc1') +@click.option('--excluded-pr-list', type=str, help="Coma-separated list of PRs to exclude from the issue.") +@argument_package_ids +def generate_issue_content(package_ids: List[str], github_token: str, suffix: str, excluded_pr_list: str): + if not package_ids: + package_ids = get_all_providers() + """Generates content for issue to test the release.""" + with with_group("Generates GitHub issue content with people who can test it"): + if excluded_pr_list: + excluded_prs = [int(pr) for pr in excluded_pr_list.split(",")] + else: + excluded_prs = [] + console = Console(width=200, color_system="standard") + all_prs: Set[int] = set() + provider_prs: Dict[str, List[int]] = {} + for package_id in package_ids: + console.print(f"Extracting PRs for provider {package_id}") + prs = get_prs_for_package(package_id) + provider_prs[package_id] = list(filter(lambda pr: pr not in excluded_prs, prs)) + all_prs.update(provider_prs[package_id]) + g = Github(github_token) + repo = g.get_repo("apache/airflow") + pull_requests: Dict[int, PullRequest.PullRequest] = {} + with Progress(console=console) as progress: + task = progress.add_task(f"Retrieving {len(all_prs)} PRs ", total=len(all_prs)) + pr_list = list(all_prs) + for i in range(len(pr_list)): + pr_number = pr_list[i] + progress.console.print( + f"Retrieving PR#{pr_number}: " f"https://github.com/apache/airflow/pull/{pr_number}" + ) + try: + pull_requests[pr_number] = repo.get_pull(pr_number) + except UnknownObjectException: + # Fallback to issue if PR not found + try: + pull_requests[pr_number] = repo.get_issue(pr_number) # noqa (same fields as PR) + except UnknownObjectException: + console.print(f"[red]The PR #{pr_number} could not be found[/]") + progress.advance(task) + interesting_providers: Dict[str, ProviderPRInfo] = {} + non_interesting_providers: Dict[str, ProviderPRInfo] = {} + for package_id in package_ids: + pull_request_list = [pull_requests[pr] for pr in provider_prs[package_id] if pr in pull_requests] + provider_details = get_provider_details(package_id) + if pull_request_list: + interesting_providers[package_id] = ProviderPRInfo(provider_details, pull_request_list) + else: + non_interesting_providers[package_id] = ProviderPRInfo(provider_details, pull_request_list) + context = { + 'interesting_providers': interesting_providers, + 'date': datetime.now(), + 'suffix': suffix, + 'non_interesting_providers': non_interesting_providers, + } + issue_content = render_template(template_name="PROVIDER_ISSUE", context=context, extension=".md") + console.print() + console.print( + "[green]Below you can find the issue content that you can use " + "to ask contributor to test providers![/]" + ) + console.print() + console.print() + console.print( + "Issue title: [yellow]Status of testing Providers that were " + f"prepared on { datetime.now().strftime('%B %d, %Y') }[/]" + ) + console.print() + syntax = Syntax(issue_content, "markdown", theme="ansi_dark") + console.print(syntax) + console.print() + users: Set[str] = set() + for provider_info in interesting_providers.values(): + for pr in provider_info.pr_list: + users.add("@" + pr.user.login) + console.print("All users involved in the PRs:") + console.print(" ".join(users)) + + if __name__ == "__main__": # The cli exit code is: # * 0 in case of success diff --git a/scripts/ci/docker-compose/forward-credentials.yml b/scripts/ci/docker-compose/forward-credentials.yml index 0eb0dbb00491f..ebbd2ec803afc 100644 --- a/scripts/ci/docker-compose/forward-credentials.yml +++ b/scripts/ci/docker-compose/forward-credentials.yml @@ -24,6 +24,8 @@ services: # Everything you install in Docker # If you add it here - also add it to "in_container_fix_ownership" method in # the _in_container_utils.sh file to make it friendly for Linux users + environment: + - GITHUB_TOKEN volumes: - ${HOME}/.aws:/root/.aws:cached - ${HOME}/.azure:/root/.azure:cached diff --git a/scripts/ci/libraries/_runs.sh b/scripts/ci/libraries/_runs.sh index d15e68bf657d0..f7bb69ff5f370 100644 --- a/scripts/ci/libraries/_runs.sh +++ b/scripts/ci/libraries/_runs.sh @@ -73,6 +73,8 @@ function runs::run_prepare_provider_documentation() { "${term_flag}" \ -v "${AIRFLOW_SOURCES}:/opt/airflow" \ -e "NON_INTERACTIVE" \ + -e "GENERATE_PROVIDERS_ISSUE" \ + -e "GITHUB_TOKEN" \ "${AIRFLOW_CI_IMAGE}" \ "--" "/opt/airflow/scripts/in_container/run_prepare_provider_documentation.sh" "${@}" } diff --git a/scripts/in_container/run_prepare_provider_documentation.sh b/scripts/in_container/run_prepare_provider_documentation.sh index bbc889b5b7eae..9130c79b9681d 100755 --- a/scripts/in_container/run_prepare_provider_documentation.sh +++ b/scripts/in_container/run_prepare_provider_documentation.sh @@ -96,19 +96,19 @@ function run_prepare_documentation() { echo if [[ "${#prepared_documentation[@]}" != "0" ]]; then echo "${COLOR_GREEN} Success:${COLOR_RESET}" - echo "${prepared_documentation[@]}" | fold -w 100 + echo "${prepared_documentation[@]}" | fold -sw 100 fi if [[ "${#skipped_documentation[@]}" != "0" ]]; then echo "${COLOR_YELLOW} Skipped:${COLOR_RESET}" - echo "${skipped_documentation[@]}" | fold -w 100 + echo "${skipped_documentation[@]}" | fold -sw 100 fi if [[ "${#doc_only_documentation[@]}" != "0" ]]; then echo "${COLOR_YELLOW} Marked as doc-only (please commit those!):${COLOR_RESET}" - echo "${doc_only_documentation[@]}" | fold -w 100 + echo "${doc_only_documentation[@]}" | fold -sw 100 fi if [[ "${#error_documentation[@]}" != "0" ]]; then echo "${COLOR_RED} Errors:${COLOR_RESET}" - echo "${error_documentation[@]}" | fold -w 100 + echo "${error_documentation[@]}" | fold -sw 100 | sed "s/^/ /" | sed "s/$/\\/" fi echo echo "${COLOR_BLUE}===================================================================================${COLOR_RESET}" @@ -117,9 +117,28 @@ function run_prepare_documentation() { echo "${COLOR_RED}There were errors when preparing documentation. Exiting! ${COLOR_RESET}" exit 1 else + if [[ ${GENERATE_PROVIDERS_ISSUE=} == "true" ]]; then + echo + python3 dev/provider_packages/prepare_provider_packages.py generate-issue-content "${prepared_documentation[@]}" + echo + fi + echo "${COLOR_BLUE}===================================================================================${COLOR_RESET}" + echo + echo "${COLOR_YELLOW}You can separately generate content of the issue to create to track testing status by running this command${COLOR_RESET}" + echo "${COLOR_YELLOW}You can optionally exclude some PRs via --excluded-pr-list option containing coma-separated list of pr numbers${COLOR_RESET}" + echo + echo "python3 dev/provider_packages/prepare_provider_packages.py generate-issue-content \\" + echo "${prepared_documentation[@]}" | fold -sw 100 | sed "s/^/ /" | sed "s/$/ \\\\/" + echo " [--excluded-pr-list=\"\" ]" + echo + echo "${COLOR_YELLOW}You can also run it here by rerunning the prepare-providers-documentation${COLOR_RESET}" + echo "${COLOR_YELLOW}with --generate-providers-issue and --forward-credentials flag (you have to have GITHUB_TOKEN variable set)${COLOR_RESET}" + echo + echo "${COLOR_BLUE}===================================================================================${COLOR_RESET}" echo echo "${COLOR_YELLOW}Please review the updated files, classify the changelog entries and commit the changes!${COLOR_RESET}" echo + fi } diff --git a/setup.py b/setup.py index e2ba9d4fd1094..0655c02153f95 100644 --- a/setup.py +++ b/setup.py @@ -522,6 +522,7 @@ def get_sphinx_theme_version() -> str: 'paramiko', 'pipdeptree', 'pre-commit', + 'pygithub', 'pylint~=2.8.1', 'pysftp', 'pytest~=6.0',