From b2e0505e1089eb89818e506cb2af8a5b7f89a91e Mon Sep 17 00:00:00 2001 From: Lo Ferris Date: Tue, 24 Aug 2021 09:15:48 -0700 Subject: [PATCH 01/19] adding python_notebooks template to synthtool --- synthtool/gcp/common.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/synthtool/gcp/common.py b/synthtool/gcp/common.py index 1f54ad3f4..e20ff7b1c 100644 --- a/synthtool/gcp/common.py +++ b/synthtool/gcp/common.py @@ -204,6 +204,12 @@ def py_samples_override( overridden_samples_kwargs["subdir"] = override_path return self._generic_library("python_samples", **overridden_samples_kwargs) + + def python_notebooks(self, **kwargs) -> Path: + # kwargs["metadata"] is required to load values from .repo-metadata.json + if "metadata" not in kwargs: + kwargs["metadata"] = {} + return self._generic_library("python_notebooks", **kwargs) def py_library(self, **kwargs) -> Path: # kwargs["metadata"] is required to load values from .repo-metadata.json From f22fe84364121afb1b9ceece2b2e47a5b0ce973c Mon Sep 17 00:00:00 2001 From: Lo Ferris Date: Wed, 25 Aug 2021 16:02:34 -0700 Subject: [PATCH 02/19] adding python notebook temlate repo --- synthtool/gcp/common.py | 2 +- .../.cloud-build/check_python_version.py | 28 +++ .../cleanup/cleanup-cloudbuild.yaml | 8 + .../cleanup/cleanup-requirements.txt | 1 + .../.cloud-build/cleanup/cleanup.py | 56 +++++ .../cleanup/resource_cleanup_manager.py | 101 +++++++++ .../.cloud-build/execute_changed_notebooks.py | 159 ++++++++++++++ .../.cloud-build/execute_notebook.py | 89 ++++++++ .../notebook-execution-test-cloudbuild.yaml | 40 ++++ .../.cloud-build/notebook_processors.py | 62 ++++++ .../.cloud-build/requirements.txt | 6 + .../.cloud-build/test_folders.txt | 0 .../.cloud-build/update_notebook_variables.py | 74 +++++++ .../.github/ISSUE_TEMPLATE/bug_report.md | 31 +++ .../ISSUE_TEMPLATE/sample_feature_request.md | 20 ++ .../.github/pull-request-template.md | 12 ++ .../python_notebooks/.github/snippet-bot.yml | 0 .../.github/workflows/ci.yaml | 34 +++ .../.github/workflows/linter/requirements.txt | 9 + .../.github/workflows/linter/run_linter.sh | 152 +++++++++++++ .../gcp/templates/python_notebooks/.gitignore | 152 +++++++++++++ .../gcp/templates/python_notebooks/CODEOWNERS | 4 + .../gcp/templates/python_notebooks/LICENSE | 202 ++++++++++++++++++ .../gcp/templates/python_notebooks/README.md | 20 ++ .../python_notebooks/docs/code-of-conduct.md | 93 ++++++++ .../python_notebooks/docs/contributing.md | 28 +++ .../notebooks/community/README.md | 3 + .../notebooks/official/README.md | 3 + 28 files changed, 1388 insertions(+), 1 deletion(-) create mode 100644 synthtool/gcp/templates/python_notebooks/.cloud-build/check_python_version.py create mode 100644 synthtool/gcp/templates/python_notebooks/.cloud-build/cleanup/cleanup-cloudbuild.yaml create mode 100644 synthtool/gcp/templates/python_notebooks/.cloud-build/cleanup/cleanup-requirements.txt create mode 100644 synthtool/gcp/templates/python_notebooks/.cloud-build/cleanup/cleanup.py create mode 100644 synthtool/gcp/templates/python_notebooks/.cloud-build/cleanup/resource_cleanup_manager.py create mode 100644 synthtool/gcp/templates/python_notebooks/.cloud-build/execute_changed_notebooks.py create mode 100644 synthtool/gcp/templates/python_notebooks/.cloud-build/execute_notebook.py create mode 100644 synthtool/gcp/templates/python_notebooks/.cloud-build/notebook-execution-test-cloudbuild.yaml create mode 100644 synthtool/gcp/templates/python_notebooks/.cloud-build/notebook_processors.py create mode 100644 synthtool/gcp/templates/python_notebooks/.cloud-build/requirements.txt create mode 100644 synthtool/gcp/templates/python_notebooks/.cloud-build/test_folders.txt create mode 100644 synthtool/gcp/templates/python_notebooks/.cloud-build/update_notebook_variables.py create mode 100644 synthtool/gcp/templates/python_notebooks/.github/ISSUE_TEMPLATE/bug_report.md create mode 100644 synthtool/gcp/templates/python_notebooks/.github/ISSUE_TEMPLATE/sample_feature_request.md create mode 100644 synthtool/gcp/templates/python_notebooks/.github/pull-request-template.md create mode 100644 synthtool/gcp/templates/python_notebooks/.github/snippet-bot.yml create mode 100644 synthtool/gcp/templates/python_notebooks/.github/workflows/ci.yaml create mode 100644 synthtool/gcp/templates/python_notebooks/.github/workflows/linter/requirements.txt create mode 100644 synthtool/gcp/templates/python_notebooks/.github/workflows/linter/run_linter.sh create mode 100644 synthtool/gcp/templates/python_notebooks/.gitignore create mode 100644 synthtool/gcp/templates/python_notebooks/CODEOWNERS create mode 100644 synthtool/gcp/templates/python_notebooks/LICENSE create mode 100644 synthtool/gcp/templates/python_notebooks/README.md create mode 100644 synthtool/gcp/templates/python_notebooks/docs/code-of-conduct.md create mode 100644 synthtool/gcp/templates/python_notebooks/docs/contributing.md create mode 100644 synthtool/gcp/templates/python_notebooks/notebooks/community/README.md create mode 100644 synthtool/gcp/templates/python_notebooks/notebooks/official/README.md diff --git a/synthtool/gcp/common.py b/synthtool/gcp/common.py index e20ff7b1c..9420c89ca 100644 --- a/synthtool/gcp/common.py +++ b/synthtool/gcp/common.py @@ -204,7 +204,7 @@ def py_samples_override( overridden_samples_kwargs["subdir"] = override_path return self._generic_library("python_samples", **overridden_samples_kwargs) - + def python_notebooks(self, **kwargs) -> Path: # kwargs["metadata"] is required to load values from .repo-metadata.json if "metadata" not in kwargs: diff --git a/synthtool/gcp/templates/python_notebooks/.cloud-build/check_python_version.py b/synthtool/gcp/templates/python_notebooks/.cloud-build/check_python_version.py new file mode 100644 index 000000000..d2c542088 --- /dev/null +++ b/synthtool/gcp/templates/python_notebooks/.cloud-build/check_python_version.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python +# # Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import sys + +MINIMUM_MAJOR_VERSION = 3 +MINIMUM_MINOR_VERSION = 3 + +if ( + sys.version_info.major < MINIMUM_MAJOR_VERSION + and sys.version_info.minor < MINIMUM_MINOR_VERSION +): + print("Error: Python version less than 3.5") + exit(1) +else: + print(f"Python version acceptable: {sys.version}") + exit(0) \ No newline at end of file diff --git a/synthtool/gcp/templates/python_notebooks/.cloud-build/cleanup/cleanup-cloudbuild.yaml b/synthtool/gcp/templates/python_notebooks/.cloud-build/cleanup/cleanup-cloudbuild.yaml new file mode 100644 index 000000000..890f5c4e9 --- /dev/null +++ b/synthtool/gcp/templates/python_notebooks/.cloud-build/cleanup/cleanup-cloudbuild.yaml @@ -0,0 +1,8 @@ +steps: + # Install Python dependencies and run cleanup script + - name: ${_PYTHON_IMAGE} + entrypoint: /bin/sh + args: + - -c + - 'python3 -m pip install -U -r .cloud-build/cleanup/cleanup-requirements.txt && python3 .cloud-build/cleanup/cleanup.py' +timeout: 86400s \ No newline at end of file diff --git a/synthtool/gcp/templates/python_notebooks/.cloud-build/cleanup/cleanup-requirements.txt b/synthtool/gcp/templates/python_notebooks/.cloud-build/cleanup/cleanup-requirements.txt new file mode 100644 index 000000000..983ca6b4c --- /dev/null +++ b/synthtool/gcp/templates/python_notebooks/.cloud-build/cleanup/cleanup-requirements.txt @@ -0,0 +1 @@ +google-cloud-aiplatform \ No newline at end of file diff --git a/synthtool/gcp/templates/python_notebooks/.cloud-build/cleanup/cleanup.py b/synthtool/gcp/templates/python_notebooks/.cloud-build/cleanup/cleanup.py new file mode 100644 index 000000000..2a107619e --- /dev/null +++ b/synthtool/gcp/templates/python_notebooks/.cloud-build/cleanup/cleanup.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python +# # Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import List +from resource_cleanup_manager import ( + ResourceCleanupManager, + DatasetResourceCleanupManager, + EndpointResourceCleanupManager, + ModelResourceCleanupManager, +) + + +def run_cleanup_managers(managers: List[ResourceCleanupManager], is_dry_run: bool): + for manager in managers: + type_name = manager.type_name + + print(f"Fetching {type_name}'s...") + resources = manager.list() + print(f"Found {len(resources)} {type_name}'s") + for resource in resources: + if not manager.is_deletable(resource): + continue + + if is_dry_run: + resource_name = manager.resource_name(resource) + print(f"Will delete '{type_name}': {resource_name}") + else: + manager.delete(resource) + + print("") + + +is_dry_run = False + +if is_dry_run: + print("Starting cleanup in dry run mode...") + +# List of all cleanup managers +managers = [ + DatasetResourceCleanupManager(), + EndpointResourceCleanupManager(), + ModelResourceCleanupManager(), +] + +run_cleanup_managers(managers=managers, is_dry_run=is_dry_run) \ No newline at end of file diff --git a/synthtool/gcp/templates/python_notebooks/.cloud-build/cleanup/resource_cleanup_manager.py b/synthtool/gcp/templates/python_notebooks/.cloud-build/cleanup/resource_cleanup_manager.py new file mode 100644 index 000000000..c23535b47 --- /dev/null +++ b/synthtool/gcp/templates/python_notebooks/.cloud-build/cleanup/resource_cleanup_manager.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python +# # Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import abc +from google.cloud import aiplatform +from typing import Any +from proto.datetime_helpers import DatetimeWithNanoseconds +from google.cloud.aiplatform import base + +# If a resource was updated within this number of seconds, do not delete. +RESOURCE_UPDATE_BUFFER_IN_SECONDS = 60 * 60 * 8 + + +class ResourceCleanupManager(abc.ABC): + @property + @abc.abstractmethod + def type_name(str) -> str: + pass + + @abc.abstractmethod + def list(self) -> Any: + pass + + @abc.abstractmethod + def resource_name(self, resource: Any) -> str: + pass + + @abc.abstractmethod + def delete(self, resource: Any): + pass + + @abc.abstractmethod + def get_seconds_since_modification(self, resource: Any) -> float: + pass + + def is_deletable(self, resource: Any) -> bool: + time_difference = self.get_seconds_since_modification(resource) + + if self.resource_name(resource).startswith("perm"): + print(f"Skipping '{resource}' due to name starting with 'perm'.") + return False + + # Check that it wasn't created too recently, to prevent race conditions + if time_difference <= RESOURCE_UPDATE_BUFFER_IN_SECONDS: + print( + f"Skipping '{resource}' due update_time being '{time_difference}', which is less than '{RESOURCE_UPDATE_BUFFER_IN_SECONDS}'." + ) + return False + + return True + + +class VertexAIResourceCleanupManager(ResourceCleanupManager): + @property + @abc.abstractmethod + def vertex_ai_resource(self) -> base.VertexAiResourceNounWithFutureManager: + pass + + @property + def type_name(self) -> str: + return self.vertex_ai_resource._resource_noun + + def list(self) -> Any: + return self.vertex_ai_resource.list() + + def resource_name(self, resource: Any) -> str: + return resource.display_name + + def delete(self, resource): + resource.delete() + + def get_seconds_since_modification(self, resource: Any) -> bool: + update_time = resource.update_time + current_time = DatetimeWithNanoseconds.now(tz=update_time.tzinfo) + return (current_time - update_time).total_seconds() + + +class DatasetResourceCleanupManager(VertexAIResourceCleanupManager): + vertex_ai_resource = aiplatform.datasets._Dataset + + +class EndpointResourceCleanupManager(VertexAIResourceCleanupManager): + vertex_ai_resource = aiplatform.Endpoint + + def delete(self, resource): + resource.delete(force=True) + + +class ModelResourceCleanupManager(VertexAIResourceCleanupManager): + vertex_ai_resource = aiplatform.Model \ No newline at end of file diff --git a/synthtool/gcp/templates/python_notebooks/.cloud-build/execute_changed_notebooks.py b/synthtool/gcp/templates/python_notebooks/.cloud-build/execute_changed_notebooks.py new file mode 100644 index 000000000..be02ec9ff --- /dev/null +++ b/synthtool/gcp/templates/python_notebooks/.cloud-build/execute_changed_notebooks.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python +# # Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import argparse +import pathlib +import subprocess +from pathlib import Path +from typing import Dict, List, Optional + +import execute_notebook + + +def run_changed_notebooks( + test_paths_file: str, + output_folder: str, + variable_project_id: str, + variable_region: str, + base_branch: Optional[str], +): + """ + Run the notebooks that exist under the folders defined in the test_paths_file. + It only runs notebooks that have differences from the Git base_branch. + The executed notebooks are saved in the output_folder. + Variables are also injected into the notebooks such as the variable_project_id and variable_region. + Args: + test_paths_file (str): + Required. The new-line delimited file to folders and files that need checking. + Folders are checked recursively. + base_branch (str): + Optional. If provided, only the files that have changed from the base_branch will be checked. + If not provided, all files will be checked. + output_folder (str): + Required. The folder to write executed notebooks to. + variable_project_id (str): + Required. The value for PROJECT_ID to inject into notebooks. + variable_region (str): + Required. The value for REGION to inject into notebooks. + """ + + test_paths = [] + with open(test_paths_file) as file: + lines = [line.strip() for line in file.readlines()] + lines = [line for line in lines if len(line) > 0] + test_paths = [line for line in lines] + + if len(test_paths) == 0: + raise RuntimeError("No test folders found.") + + print(f"Checking folders: {test_paths}") + + # Find notebooks + notebooks = [] + if base_branch: + print(f"Looking for notebooks that changed from branch: {base_branch}") + notebooks = subprocess.check_output( + ["git", "diff", "--name-only", f"origin/{base_branch}...", "--"] + test_paths + ) + else: + print(f"Looking for all notebooks.") + notebooks = subprocess.check_output(["git", "ls-files"] + test_paths) + + notebooks = notebooks.decode("utf-8").split("\n") + notebooks = [notebook for notebook in notebooks if notebook.endswith(".ipynb")] + notebooks = [notebook for notebook in notebooks if len(notebook) > 0] + notebooks = [notebook for notebook in notebooks if Path(notebook).exists()] + + # Create paths + artifacts_path = Path(output_folder) + artifacts_path.mkdir(parents=True, exist_ok=True) + artifacts_path.joinpath("success").mkdir(parents=True, exist_ok=True) + artifacts_path.joinpath("failure").mkdir(parents=True, exist_ok=True) + + passed_notebooks: List[str] = [] + failed_notebooks: List[str] = [] + + if len(notebooks) > 0: + print(f"Found {len(notebooks)} modified notebooks: {notebooks}") + + for notebook in notebooks: + print(f"Running notebook: {notebook}") + + # TODO: Handle cases where multiple notebooks have the same name + try: + execute_notebook.execute_notebook( + notebook_file_path=notebook, + output_file_folder=artifacts_path, + replacement_map={ + "PROJECT_ID": variable_project_id, + "REGION": variable_region, + }, + ) + print(f"Notebook finished successfully.") + passed_notebooks.append(notebook) + except Exception as error: + print(f"Notebook finished with failure: {error}") + failed_notebooks.append(notebook) + else: + print("No notebooks modified in this pull request.") + + if len(failed_notebooks) > 0: + print(f"{len(failed_notebooks)} notebooks failed:") + print(failed_notebooks) + print(f"{len(passed_notebooks)} notebooks passed:") + print(passed_notebooks) + elif len(passed_notebooks) > 0: + print("All notebooks executed successfully:") + print(passed_notebooks) + + +parser = argparse.ArgumentParser(description="Run changed notebooks.") +parser.add_argument( + "--test_paths_file", + type=pathlib.Path, + help="The path to the file that has newline-limited folders of notebooks that should be tested.", + required=True, +) +parser.add_argument( + "--base_branch", + help="The base git branch to diff against to find changed files.", + required=False, +) +parser.add_argument( + "--output_folder", + type=pathlib.Path, + help="The path to the folder to store executed notebooks.", + required=True, +) +parser.add_argument( + "--variable_project_id", + type=str, + help="The GCP project id. This is used to inject a variable value into the notebook before running.", + required=True, +) +parser.add_argument( + "--variable_region", + type=str, + help="The GCP region. This is used to inject a variable value into the notebook before running.", + required=True, +) + +args = parser.parse_args() +run_changed_notebooks( + test_paths_file=args.test_paths_file, + base_branch=args.base_branch, + output_folder=args.output_folder, + variable_project_id=args.variable_project_id, + variable_region=args.variable_region, +) \ No newline at end of file diff --git a/synthtool/gcp/templates/python_notebooks/.cloud-build/execute_notebook.py b/synthtool/gcp/templates/python_notebooks/.cloud-build/execute_notebook.py new file mode 100644 index 000000000..3fdda714a --- /dev/null +++ b/synthtool/gcp/templates/python_notebooks/.cloud-build/execute_notebook.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python +# # Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import sys +import nbformat +from nbconvert.preprocessors import ExecutePreprocessor, CellExecutionError +import os +import errno +from notebook_processors import RemoveNoExecuteCells, UpdateVariablesPreprocessor +from typing import Dict +import papermill as pm + +# This script is used to execute a notebook and write out the output notebook. +# The replaces calling the nbconvert via command-line, which doesn't write the output notebook correctly when there are errors during execution. + + +def execute_notebook( + notebook_file_path: str, output_file_folder: str, replacement_map: Dict[str, str] +): + file_name = os.path.basename(os.path.normpath(notebook_file_path)) + + # Read notebook + with open(notebook_file_path) as f: + nb = nbformat.read(f, as_version=4) + + has_error = False + + # Execute notebook + try: + # Create preprocessors + remove_no_execute_cells_preprocessor = RemoveNoExecuteCells() + update_variables_preprocessor = UpdateVariablesPreprocessor( + replacement_map=replacement_map + ) + execute_preprocessor = ExecutePreprocessor(timeout=-1, kernel_name="python3") + + # Use no-execute preprocessor + ( + nb, + resources, + ) = remove_no_execute_cells_preprocessor.preprocess(nb) + + (nb, resources) = update_variables_preprocessor.preprocess(nb, resources) + + # Execute notebook + # out = execute_preprocessor.preprocess(nb, resources) + output_file_path = os.path.join( + output_file_folder, "failure" if has_error else "success", file_name + ) + + # Create directories if they don't exist + if not os.path.exists(os.path.dirname(output_file_path)): + try: + os.makedirs(os.path.dirname(output_file_path)) + except OSError as exc: # Guard against race condition + if exc.errno != errno.EEXIST: + raise + + print(f"Writing modified notebook to: {output_file_path}") + with open(output_file_path, mode="w", encoding="utf-8") as f: + nbformat.write(nb, f) + + pm.execute_notebook( + input_path=output_file_path, + output_path=output_file_path, + progress_bar=True, + request_save_on_cell_execute=True, + log_output=True, + stdout_file=sys.stdout, + stderr_file=sys.stderr, + ) + + except Exception as error: + out = None + print(f"Error executing the notebook: {notebook_file_path}.\n\n") + has_error = True + + raise \ No newline at end of file diff --git a/synthtool/gcp/templates/python_notebooks/.cloud-build/notebook-execution-test-cloudbuild.yaml b/synthtool/gcp/templates/python_notebooks/.cloud-build/notebook-execution-test-cloudbuild.yaml new file mode 100644 index 000000000..de036e877 --- /dev/null +++ b/synthtool/gcp/templates/python_notebooks/.cloud-build/notebook-execution-test-cloudbuild.yaml @@ -0,0 +1,40 @@ +steps: + # Show the gcloud info and check if gcloud exists + - name: ${_PYTHON_IMAGE} + entrypoint: /bin/sh + args: + - -c + - 'gcloud config list' + # Check the Python version + - name: ${_PYTHON_IMAGE} + entrypoint: /bin/sh + args: + - -c + - 'python3 .cloud-build/check_python_version.py' + # Fetch base branch if required + - name: ${_PYTHON_IMAGE} + entrypoint: /bin/sh + args: + - -c + - 'if [ -n "${_BASE_BRANCH}" ]; then git fetch origin "${_BASE_BRANCH}":refs/remotes/origin/"${_BASE_BRANCH}"; else echo "Skipping fetch."; fi' + # Install Python dependencies and run testing script + - name: ${_PYTHON_IMAGE} + entrypoint: /bin/sh + args: + - -c + - 'python3 -m pip install -U -r .cloud-build/requirements.txt && python3 -m pip freeze && python3 .cloud-build/execute_changed_notebooks.py --test_paths_file .cloud-build/test_folders.txt --base_branch "${_FORCED_BASE_BRANCH}" --output_folder ${BUILD_ID} --variable_project_id ${PROJECT_ID} --variable_region ${_GCP_REGION}' + env: + - 'IS_TESTING=1' + # Manually copy artifacts to GCS + - name: gcr.io/cloud-builders/gsutil + entrypoint: /bin/sh + args: + - -c + - 'if [ $(ls -pR "/workspace/${BUILD_ID}" | grep -v / | grep -v ^$ | wc -l) -ne 0 ]; then gsutil rsync -r "/workspace/${BUILD_ID}" "gs://${_GCS_ARTIFACTS_BUCKET}/test-artifacts/PR_${_PR_NUMBER}/BUILD_${BUILD_ID}/"; else echo "No artifacts to copy."; fi' + # Fail if there is anything in the failure folder + - name: ${_PYTHON_IMAGE} + entrypoint: /bin/sh + args: + - -c + - 'echo "Download executed notebooks with this command: \"mkdir -p artifacts && gsutil rsync -r gs://${_GCS_ARTIFACTS_BUCKET}/test-artifacts/PR_${_PR_NUMBER}/BUILD_${BUILD_ID} artifacts/\"" && if [ "$(ls -A /workspace/${BUILD_ID}/failure | wc -l)" -ne 0 ]; then exit 1; else exit 0; fi' +timeout: 86400s \ No newline at end of file diff --git a/synthtool/gcp/templates/python_notebooks/.cloud-build/notebook_processors.py b/synthtool/gcp/templates/python_notebooks/.cloud-build/notebook_processors.py new file mode 100644 index 000000000..44c628f31 --- /dev/null +++ b/synthtool/gcp/templates/python_notebooks/.cloud-build/notebook_processors.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python +# # Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from nbconvert.preprocessors import Preprocessor +from typing import Dict +import update_notebook_variables + + +class RemoveNoExecuteCells(Preprocessor): + def preprocess(self, notebook, resources=None): + executable_cells = [] + for cell in notebook.cells: + if cell.metadata.get("tags"): + if "no_execute" in cell.metadata.get("tags"): + continue + executable_cells.append(cell) + notebook.cells = executable_cells + return notebook, resources + + +class UpdateVariablesPreprocessor(Preprocessor): + def __init__(self, replacement_map: Dict): + self._replacement_map = replacement_map + + @staticmethod + def update_variables(content: str, replacement_map: Dict[str, str]): + # replace variables inside .ipynb files + # looking for this format inside notebooks: + # VARIABLE_NAME = '[description]' + + for variable_name, variable_value in replacement_map.items(): + content = update_notebook_variables.get_updated_value( + content=content, + variable_name=variable_name, + variable_value=variable_value, + ) + + return content + + def preprocess(self, notebook, resources=None): + executable_cells = [] + for cell in notebook.cells: + if cell.cell_type == "code": + cell.source = self.update_variables( + content=cell.source, + replacement_map=self._replacement_map, + ) + + executable_cells.append(cell) + notebook.cells = executable_cells + return notebook, resources \ No newline at end of file diff --git a/synthtool/gcp/templates/python_notebooks/.cloud-build/requirements.txt b/synthtool/gcp/templates/python_notebooks/.cloud-build/requirements.txt new file mode 100644 index 000000000..c6a9a18ac --- /dev/null +++ b/synthtool/gcp/templates/python_notebooks/.cloud-build/requirements.txt @@ -0,0 +1,6 @@ +ipython +jupyter>=1.0 +nbconvert>=6.0 +papermill +numpy +pandas \ No newline at end of file diff --git a/synthtool/gcp/templates/python_notebooks/.cloud-build/test_folders.txt b/synthtool/gcp/templates/python_notebooks/.cloud-build/test_folders.txt new file mode 100644 index 000000000..e69de29bb diff --git a/synthtool/gcp/templates/python_notebooks/.cloud-build/update_notebook_variables.py b/synthtool/gcp/templates/python_notebooks/.cloud-build/update_notebook_variables.py new file mode 100644 index 000000000..9c74627e5 --- /dev/null +++ b/synthtool/gcp/templates/python_notebooks/.cloud-build/update_notebook_variables.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python +# # Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import re + +""" + This script is used to update variables in the notebook via regex + It requires variables to be defined in particular format +For example, if your variable was PROJECT_ID, use: + PROJECT_ID = "[your_project_here]" +Single-quotes also work: + PROJECT_ID = '[your_project_here]' +Variables in conditionals can also be replaced: + PROJECT_ID == "[your_project_here]" +""" + + +def get_updated_value(content: str, variable_name: str, variable_value: str) -> str: + return re.sub( + rf"({variable_name}.*?=.*?[\",\'])\[.+?\]([\",\'].*?)", + rf"\1{variable_value}\2", + content, + flags=re.M, + ) + + +def test_update_value(): + new_content = get_updated_value( + content='asdf\nPROJECT_ID = "[your-project-id]" #@param {type:"string"} \nasdf', + variable_name="PROJECT_ID", + variable_value="sample-project", + ) + assert ( + new_content + == 'asdf\nPROJECT_ID = "sample-project" #@param {type:"string"} \nasdf' + ) + + +def test_update_value_single_quotes(): + new_content = get_updated_value( + content="PROJECT_ID = '[your-project-id]'", + variable_name="PROJECT_ID", + variable_value="sample-project", + ) + assert new_content == "PROJECT_ID = 'sample-project'" + + +def test_update_value_avoidance(): + new_content = get_updated_value( + content="PROJECT_ID = shell_output[0] ", + variable_name="PROJECT_ID", + variable_value="sample-project", + ) + assert new_content == "PROJECT_ID = shell_output[0] " + + +def test_region(): + new_content = get_updated_value( + content='REGION = "[your-region]" # @param {type:"string"}', + variable_name="REGION", + variable_value="us-central1", + ) + assert new_content == 'REGION = "us-central1" # @param {type:"string"}' \ No newline at end of file diff --git a/synthtool/gcp/templates/python_notebooks/.github/ISSUE_TEMPLATE/bug_report.md b/synthtool/gcp/templates/python_notebooks/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 000000000..ebb13c156 --- /dev/null +++ b/synthtool/gcp/templates/python_notebooks/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,31 @@ +--- +name: Bug report +about: For problems running the sample code please provide the following information. + +--- + +**Describe the bug** +A clear and concise description of what the bug is. Be sure to convey here whether it occurred locally or on the server (BigQuery, AI Platform, Google Dataflow) + +**What sample is this bug related to?** + +**Source code / logs** +Include any logs or source code that would be helpful to diagnose the problem. If including tracebacks, please include the full traceback. Large logs and files should be attached. Try to provide a reproducible test case that is the bare minimum necessary to generate the problem. + +**To Reproduce** +Steps to reproduce the behavior: +1. Go to '...' +2. Click on '....' +3. Scroll down to '....' +4. See error + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**System Information** +- **OS Platform and Distribution (e.g., Linux Ubuntu 16.04)**: +- **Python version**: +- **Exact command to reproduce**: + +**Additional context** +Add any other context about the problem here. \ No newline at end of file diff --git a/synthtool/gcp/templates/python_notebooks/.github/ISSUE_TEMPLATE/sample_feature_request.md b/synthtool/gcp/templates/python_notebooks/.github/ISSUE_TEMPLATE/sample_feature_request.md new file mode 100644 index 000000000..a825dfbdf --- /dev/null +++ b/synthtool/gcp/templates/python_notebooks/.github/ISSUE_TEMPLATE/sample_feature_request.md @@ -0,0 +1,20 @@ +--- +name: Sample/Feature request +about: Suggest an idea for this project + +--- + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Key technologies to include** +* AI Platform specfic features (CPU, GPU, HP Tuning, TPU): +* Framework (BigQuery ML, AutoML, Tensorflow, Keras, scikit-learn, XGBoost, ...): +* Model: +* Dataset: + +**Additional context** +Add any other context or screenshots about the feature request here. \ No newline at end of file diff --git a/synthtool/gcp/templates/python_notebooks/.github/pull-request-template.md b/synthtool/gcp/templates/python_notebooks/.github/pull-request-template.md new file mode 100644 index 000000000..7fac8f48f --- /dev/null +++ b/synthtool/gcp/templates/python_notebooks/.github/pull-request-template.md @@ -0,0 +1,12 @@ +Fixes # + +Before submitting a Jupyter notebook, follow this mandatory checklist: + +- [ ] Use the [notebook template](#) as a starting point. +- [ ] Double check that all links, including the Colab and Github links to the notebook, are valid. +- [ ] Follow the style and grammar rules outlined in the above notebook template. +- [ ] Verify the notebook runs successfully in Colab since the automated tests cannot guarantee this even when it passes. +- [ ] Passes all the required automated checks +- [ ] You have consulted with a tech writer to see if tech writer review is necessary. If so, the notebook has been reviewed by a tech writer, and they have approved it. +- [ ] This notebook has been added to the CODEOWNERS file, pointing to the author or the author's team. If the CODEOWNERS file doesn't exist, create one in the nearest folder that makes sense. +- [ ] The Jupyter notebook cleans up any artifacts it has created (datasets, ML models, endpoints, etc) so as not to eat up unnecessary resources. \ No newline at end of file diff --git a/synthtool/gcp/templates/python_notebooks/.github/snippet-bot.yml b/synthtool/gcp/templates/python_notebooks/.github/snippet-bot.yml new file mode 100644 index 000000000..e69de29bb diff --git a/synthtool/gcp/templates/python_notebooks/.github/workflows/ci.yaml b/synthtool/gcp/templates/python_notebooks/.github/workflows/ci.yaml new file mode 100644 index 000000000..d8b78f268 --- /dev/null +++ b/synthtool/gcp/templates/python_notebooks/.github/workflows/ci.yaml @@ -0,0 +1,34 @@ +name: ci +on: pull_request + +jobs: + format_and_lint: + name: notebook format and lint + runs-on: ubuntu-latest + steps: + - name: Set up Python + uses: actions/setup-python@v2 + - name: Fetch pull request branch + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Fetch base main branch + run: git fetch -u "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY" main:main + - name: Install requirements + run: python3 -m pip install -U -r .github/workflows/linter/requirements.txt + - name: Format and lint notebooks + run: | + set +e + .github/workflows/linter/run_linter.sh -t + RTN=$? + if [ "$RTN" != "0" ]; then + echo "There were problems formatting/linting the notebooks." + echo "Please run the following commands locally from the root directory to attempt to autofix the issues:" + echo "" + echo "python3 -m pip install -U -r .github/workflows/linter/requirements.txt" + echo ".github/workflows/linter/run_linter.sh" + echo "" + echo "If it can't be autofixed, please fix them manually." + echo "Then, commit the fixes and push again." + exit 1 + fi \ No newline at end of file diff --git a/synthtool/gcp/templates/python_notebooks/.github/workflows/linter/requirements.txt b/synthtool/gcp/templates/python_notebooks/.github/workflows/linter/requirements.txt new file mode 100644 index 000000000..02f8896b7 --- /dev/null +++ b/synthtool/gcp/templates/python_notebooks/.github/workflows/linter/requirements.txt @@ -0,0 +1,9 @@ +git+https://github.com/tensorflow/docs +ipython +jupyter +nbconvert +black==20.8b1 +pyupgrade==2.7.3 +isort==5.6.4 +flake8==3.9.0 +nbqa==0.6.0 \ No newline at end of file diff --git a/synthtool/gcp/templates/python_notebooks/.github/workflows/linter/run_linter.sh b/synthtool/gcp/templates/python_notebooks/.github/workflows/linter/run_linter.sh new file mode 100644 index 000000000..4ba3a8b06 --- /dev/null +++ b/synthtool/gcp/templates/python_notebooks/.github/workflows/linter/run_linter.sh @@ -0,0 +1,152 @@ +#!/bin/bash +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script automatically formats and lints all notebooks that have changed from the head of the master branch. +# +# Options: +# -t: Test-mode. Only test if format and linting are required but make no changes to files. +# +# Returns: +# This script will return 0 if linting was successful/unneeded and 1 if there were any errors. + +# `+e` enables the script to continue even when a command fails +set +e + +# `-o pipefail` sets the exit code to the rightmost comment to exit with a non-zero +set -o pipefail + +# Use RTN to return a non-zero value if the test fails. +RTN="0" + +is_test=false + +# Process all options supplied on the command line +while getopts 'tc' arg; do + case $arg in + 't') + is_test=true + ;; + *) + echo "Unimplemented flag" + exit 1 + ;; + esac +done + +echo "Test mode: $is_test" + +# Only check notebooks in test folders modified in this pull request. +# Note: Use process substitution to persist the data in the array +notebooks=() +while read -r file || [ -n "$line" ]; +do + notebooks+=("$file") +done < <(git diff --name-only main... | grep '\.ipynb$') + +problematic_notebooks=() +if [ ${#notebooks[@]} -gt 0 ]; then + for notebook in "${notebooks[@]}" + do + if [ -f "$notebook" ]; then + echo "Checking notebook: ${notebook}" + + NBFMT_RTN="0" + BLACK_RTN="0" + PYUPGRADE_RTN="0" + ISORT_RTN="0" + FLAKE8_RTN="0" + + if [ "$is_test" = true ] ; then + echo "Running nbfmt..." + python3 -m tensorflow_docs.tools.nbfmt --remove_outputs --test "$notebook" + NBFMT_RTN=$? + # echo "Running black..." + # python3 -m nbqa black "$notebook" --check + # BLACK_RTN=$? + echo "Running pyupgrade..." + python3 -m nbqa pyupgrade "$notebook" + PYUPGRADE_RTN=$? + echo "Running isort..." + python3 -m nbqa isort "$notebook" --check + ISORT_RTN=$? + echo "Running flake8..." + python3 -m nbqa flake8 "$notebook" --show-source --extend-ignore=W391,E501,F821,E402,F404,W503,W291,E203,E999,E111,E113 + FLAKE8_RTN=$? + else + echo "Running black..." + python3 -m nbqa black "$notebook" --nbqa-mutate + BLACK_RTN=$? + echo "Running pyupgrade..." + python3 -m nbqa pyupgrade "$notebook" --nbqa-mutate + PYUPGRADE_RTN=$? + echo "Running isort..." + python3 -m nbqa isort "$notebook" --nbqa-mutate + ISORT_RTN=$? + echo "Running nbfmt..." + python3 -m tensorflow_docs.tools.nbfmt --remove_outputs "$notebook" + NBFMT_RTN=$? + echo "Running flake8..." + python3 -m nbqa flake8 "$notebook" --show-source --extend-ignore=W391,E501,F821,E402,F404,W503,W291,E203,E999,E111,E113 + FLAKE8_RTN=$? + fi + + NOTEBOOK_RTN="0" + + if [ "$NBFMT_RTN" != "0" ]; then + NOTEBOOK_RTN="$NBFMT_RTN" + printf "nbfmt: Failed\n" + fi + + if [ "$BLACK_RTN" != "0" ]; then + NOTEBOOK_RTN="$BLACK_RTN" + printf "black: Failed\n" + fi + + if [ "$PYUPGRADE_RTN" != "0" ]; then + NOTEBOOK_RTN="$PYUPGRADE_RTN" + printf "pyupgrade: Failed\n" + fi + + if [ "$ISORT_RTN" != "0" ]; then + NOTEBOOK_RTN="$ISORT_RTN" + printf "isort: Failed\n" + fi + + if [ "$FLAKE8_RTN" != "0" ]; then + NOTEBOOK_RTN="$FLAKE8_RTN" + printf "flake8: Failed\n" + fi + + echo "Notebook lint finished with return code = $NOTEBOOK_RTN" + echo "" + if [ "$NOTEBOOK_RTN" != "0" ] + then + problematic_notebooks+=("$notebook") + RTN=$NOTEBOOK_RTN + fi + fi + done +else + echo "No notebooks modified in this pull request." +fi + +echo "All tests finished. Exiting with return code = $RTN" + +if [ ${#problematic_notebooks[@]} -gt 0 ]; then + echo "The following notebooks could not be automatically linted:" + printf '%s\n' "${problematic_notebooks[@]}" +fi + +exit "$RTN" \ No newline at end of file diff --git a/synthtool/gcp/templates/python_notebooks/.gitignore b/synthtool/gcp/templates/python_notebooks/.gitignore new file mode 100644 index 000000000..d6703f5f8 --- /dev/null +++ b/synthtool/gcp/templates/python_notebooks/.gitignore @@ -0,0 +1,152 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# OS junk files +[Tt]humbs.db +*.DS_Store + +# PyCharm +.idea/* + +# VS Code +.vscode/* + +*.log +*.err +*.pyc \ No newline at end of file diff --git a/synthtool/gcp/templates/python_notebooks/CODEOWNERS b/synthtool/gcp/templates/python_notebooks/CODEOWNERS new file mode 100644 index 000000000..3ceb126e2 --- /dev/null +++ b/synthtool/gcp/templates/python_notebooks/CODEOWNERS @@ -0,0 +1,4 @@ +# These owners will be the default owners for everything in +# the repo. Unless a later match takes precedence, +# @global-owner1 and @global-owner2 will be requested for +# review when someone opens a pull request. \ No newline at end of file diff --git a/synthtool/gcp/templates/python_notebooks/LICENSE b/synthtool/gcp/templates/python_notebooks/LICENSE new file mode 100644 index 000000000..7a4a3ea24 --- /dev/null +++ b/synthtool/gcp/templates/python_notebooks/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/synthtool/gcp/templates/python_notebooks/README.md b/synthtool/gcp/templates/python_notebooks/README.md new file mode 100644 index 000000000..f74eb61ed --- /dev/null +++ b/synthtool/gcp/templates/python_notebooks/README.md @@ -0,0 +1,20 @@ +# Google Cloud [PRODUCT] Notebooks + +[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](LICENSE) + +Welcome to the Google Cloud [PRODUCT](#) Notebooks sample repository. + +## Overview + +The repository contains [Notebooks](#) + +## Contributing + +Contributions welcome! See the [Contributing Guide](#). + +## Getting help + +Please use the [issues page](#) to provide feedback or submit a bug report. + +## Disclaimer +This is not an officially supported Google product. The code in this repository is for demonstrative purposes only. \ No newline at end of file diff --git a/synthtool/gcp/templates/python_notebooks/docs/code-of-conduct.md b/synthtool/gcp/templates/python_notebooks/docs/code-of-conduct.md new file mode 100644 index 000000000..8ba5767ed --- /dev/null +++ b/synthtool/gcp/templates/python_notebooks/docs/code-of-conduct.md @@ -0,0 +1,93 @@ +# Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to making participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, gender identity and expression, level of +experience, education, socio-economic status, nationality, personal appearance, +race, religion, or sexual identity and orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment +include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or + advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic + address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, or to ban temporarily or permanently any +contributor for other behaviors that they deem inappropriate, threatening, +offensive, or harmful. + +## Scope + +This Code of Conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. Examples of +representing a project or community include using an official project e-mail +address, posting via an official social media account, or acting as an appointed +representative at an online or offline event. Representation of a project may be +further defined and clarified by project maintainers. + +This Code of Conduct also applies outside the project spaces when the Project +Steward has a reasonable belief that an individual's behavior may have a +negative impact on the project or its community. + +## Conflict Resolution + +We do not believe that all conflict is bad; healthy debate and disagreement +often yield positive results. However, it is never okay to be disrespectful or +to engage in behavior that violates the project’s code of conduct. + +If you see someone violating the code of conduct, you are encouraged to address +the behavior directly with those involved. Many issues can be resolved quickly +and easily, and this gives people more control over the outcome of their +dispute. If you are unable to resolve the matter for any reason, or if the +behavior is threatening or harassing, report it. We are dedicated to providing +an environment where participants feel welcome and safe. + +Reports should be directed to *[PROJECT STEWARD NAME(s) AND EMAIL(s)]*, the +Project Steward(s) for *[PROJECT NAME]*. It is the Project Steward’s duty to +receive and address reported violations of the code of conduct. They will then +work with a committee consisting of representatives from the Open Source +Programs Office and the Google Open Source Strategy team. If for any reason you +are uncomfortable reaching out to the Project Steward, please email +opensource@google.com. + +We will investigate every complaint, but you may not receive a direct response. +We will use our discretion in determining when and how to follow up on reported +incidents, which may range from not taking action to permanent expulsion from +the project and project-sponsored spaces. We will notify the accused of the +report and provide them an opportunity to discuss it before any action is taken. +The identity of the reporter will be omitted from the details of the report +supplied to the accused. In potentially harmful situations, such as ongoing +harassment or threats to anyone's safety, we may take action without notice. + +## Attribution + +This Code of Conduct is adapted from the Contributor Covenant, version 1.4, +available at +https://www.contributor-covenant.org/version/1/4/code-of-conduct.html \ No newline at end of file diff --git a/synthtool/gcp/templates/python_notebooks/docs/contributing.md b/synthtool/gcp/templates/python_notebooks/docs/contributing.md new file mode 100644 index 000000000..3abd08301 --- /dev/null +++ b/synthtool/gcp/templates/python_notebooks/docs/contributing.md @@ -0,0 +1,28 @@ +# How to Contribute + +We'd love to accept your patches and contributions to this project. There are +just a few small guidelines you need to follow. + +## Contributor License Agreement + +Contributions to this project must be accompanied by a Contributor License +Agreement. You (or your employer) retain the copyright to your contribution; +this simply gives us permission to use and redistribute your contributions as +part of the project. Head over to to see +your current agreements on file or to sign a new one. + +You generally only need to submit a CLA once, so if you've already submitted one +(even if it was for a different project), you probably don't need to do it +again. + +## Code Reviews + +All submissions, including submissions by project members, require review. We +use GitHub pull requests for this purpose. Consult +[GitHub Help](https://help.github.com/articles/about-pull-requests/) for more +information on using pull requests. + +## Community Guidelines + +This project follows [Google's Open Source Community +Guidelines](https://opensource.google/conduct/). \ No newline at end of file diff --git a/synthtool/gcp/templates/python_notebooks/notebooks/community/README.md b/synthtool/gcp/templates/python_notebooks/notebooks/community/README.md new file mode 100644 index 000000000..c148606fc --- /dev/null +++ b/synthtool/gcp/templates/python_notebooks/notebooks/community/README.md @@ -0,0 +1,3 @@ +# Google Cloud [PRODUCT] Official Notebooks + +The official tutorials are organized by Google Cloud [PRODUCT] products. \ No newline at end of file diff --git a/synthtool/gcp/templates/python_notebooks/notebooks/official/README.md b/synthtool/gcp/templates/python_notebooks/notebooks/official/README.md new file mode 100644 index 000000000..50071b817 --- /dev/null +++ b/synthtool/gcp/templates/python_notebooks/notebooks/official/README.md @@ -0,0 +1,3 @@ +# Google Cloud [PRODUCT] Community Notebooks + +Community tutorials are organized by Google Cloud [PRODUCT] products and not officially maintined. \ No newline at end of file From d7c07136df3428159c376a5926660816297a329f Mon Sep 17 00:00:00 2001 From: Lo Ferris Date: Wed, 25 Aug 2021 16:13:04 -0700 Subject: [PATCH 03/19] adding python notebooks template --- .../python_notebooks/.cloud-build/check_python_version.py | 2 +- .../python_notebooks/.cloud-build/cleanup/cleanup.py | 2 +- .../.cloud-build/cleanup/resource_cleanup_manager.py | 2 +- .../.cloud-build/execute_changed_notebooks.py | 5 +++-- .../python_notebooks/.cloud-build/execute_notebook.py | 7 ++----- .../python_notebooks/.cloud-build/notebook_processors.py | 5 ++--- .../.cloud-build/update_notebook_variables.py | 2 +- 7 files changed, 11 insertions(+), 14 deletions(-) diff --git a/synthtool/gcp/templates/python_notebooks/.cloud-build/check_python_version.py b/synthtool/gcp/templates/python_notebooks/.cloud-build/check_python_version.py index d2c542088..a40348a3b 100644 --- a/synthtool/gcp/templates/python_notebooks/.cloud-build/check_python_version.py +++ b/synthtool/gcp/templates/python_notebooks/.cloud-build/check_python_version.py @@ -25,4 +25,4 @@ exit(1) else: print(f"Python version acceptable: {sys.version}") - exit(0) \ No newline at end of file + exit(0) diff --git a/synthtool/gcp/templates/python_notebooks/.cloud-build/cleanup/cleanup.py b/synthtool/gcp/templates/python_notebooks/.cloud-build/cleanup/cleanup.py index 2a107619e..f3976bf61 100644 --- a/synthtool/gcp/templates/python_notebooks/.cloud-build/cleanup/cleanup.py +++ b/synthtool/gcp/templates/python_notebooks/.cloud-build/cleanup/cleanup.py @@ -53,4 +53,4 @@ def run_cleanup_managers(managers: List[ResourceCleanupManager], is_dry_run: boo ModelResourceCleanupManager(), ] -run_cleanup_managers(managers=managers, is_dry_run=is_dry_run) \ No newline at end of file +run_cleanup_managers(managers=managers, is_dry_run=is_dry_run) diff --git a/synthtool/gcp/templates/python_notebooks/.cloud-build/cleanup/resource_cleanup_manager.py b/synthtool/gcp/templates/python_notebooks/.cloud-build/cleanup/resource_cleanup_manager.py index c23535b47..11f45dcc4 100644 --- a/synthtool/gcp/templates/python_notebooks/.cloud-build/cleanup/resource_cleanup_manager.py +++ b/synthtool/gcp/templates/python_notebooks/.cloud-build/cleanup/resource_cleanup_manager.py @@ -98,4 +98,4 @@ def delete(self, resource): class ModelResourceCleanupManager(VertexAIResourceCleanupManager): - vertex_ai_resource = aiplatform.Model \ No newline at end of file + vertex_ai_resource = aiplatform.Model diff --git a/synthtool/gcp/templates/python_notebooks/.cloud-build/execute_changed_notebooks.py b/synthtool/gcp/templates/python_notebooks/.cloud-build/execute_changed_notebooks.py index be02ec9ff..b3af75362 100644 --- a/synthtool/gcp/templates/python_notebooks/.cloud-build/execute_changed_notebooks.py +++ b/synthtool/gcp/templates/python_notebooks/.cloud-build/execute_changed_notebooks.py @@ -64,7 +64,8 @@ def run_changed_notebooks( if base_branch: print(f"Looking for notebooks that changed from branch: {base_branch}") notebooks = subprocess.check_output( - ["git", "diff", "--name-only", f"origin/{base_branch}...", "--"] + test_paths + ["git", "diff", "--name-only", f"origin/{base_branch}...", "--"] + + test_paths ) else: print(f"Looking for all notebooks.") @@ -156,4 +157,4 @@ def run_changed_notebooks( output_folder=args.output_folder, variable_project_id=args.variable_project_id, variable_region=args.variable_region, -) \ No newline at end of file +) diff --git a/synthtool/gcp/templates/python_notebooks/.cloud-build/execute_notebook.py b/synthtool/gcp/templates/python_notebooks/.cloud-build/execute_notebook.py index 3fdda714a..83ee38c77 100644 --- a/synthtool/gcp/templates/python_notebooks/.cloud-build/execute_notebook.py +++ b/synthtool/gcp/templates/python_notebooks/.cloud-build/execute_notebook.py @@ -46,10 +46,7 @@ def execute_notebook( execute_preprocessor = ExecutePreprocessor(timeout=-1, kernel_name="python3") # Use no-execute preprocessor - ( - nb, - resources, - ) = remove_no_execute_cells_preprocessor.preprocess(nb) + (nb, resources,) = remove_no_execute_cells_preprocessor.preprocess(nb) (nb, resources) = update_variables_preprocessor.preprocess(nb, resources) @@ -86,4 +83,4 @@ def execute_notebook( print(f"Error executing the notebook: {notebook_file_path}.\n\n") has_error = True - raise \ No newline at end of file + raise diff --git a/synthtool/gcp/templates/python_notebooks/.cloud-build/notebook_processors.py b/synthtool/gcp/templates/python_notebooks/.cloud-build/notebook_processors.py index 44c628f31..2fbe9fe63 100644 --- a/synthtool/gcp/templates/python_notebooks/.cloud-build/notebook_processors.py +++ b/synthtool/gcp/templates/python_notebooks/.cloud-build/notebook_processors.py @@ -53,10 +53,9 @@ def preprocess(self, notebook, resources=None): for cell in notebook.cells: if cell.cell_type == "code": cell.source = self.update_variables( - content=cell.source, - replacement_map=self._replacement_map, + content=cell.source, replacement_map=self._replacement_map, ) executable_cells.append(cell) notebook.cells = executable_cells - return notebook, resources \ No newline at end of file + return notebook, resources diff --git a/synthtool/gcp/templates/python_notebooks/.cloud-build/update_notebook_variables.py b/synthtool/gcp/templates/python_notebooks/.cloud-build/update_notebook_variables.py index 9c74627e5..b357d7854 100644 --- a/synthtool/gcp/templates/python_notebooks/.cloud-build/update_notebook_variables.py +++ b/synthtool/gcp/templates/python_notebooks/.cloud-build/update_notebook_variables.py @@ -71,4 +71,4 @@ def test_region(): variable_name="REGION", variable_value="us-central1", ) - assert new_content == 'REGION = "us-central1" # @param {type:"string"}' \ No newline at end of file + assert new_content == 'REGION = "us-central1" # @param {type:"string"}' From 790c29821a7812191d1730aad8f11abbf42686c8 Mon Sep 17 00:00:00 2001 From: Lo Ferris Date: Wed, 25 Aug 2021 17:35:56 -0700 Subject: [PATCH 04/19] updating .cloud-build to recent versions --- .../.cloud-build/check_python_version.py | 4 ++-- .../notebook-execution-test-cloudbuild.yaml | 10 +++++++--- .../python_notebooks/.cloud-build/requirements.txt | 12 +++++++----- .../python_notebooks/.cloud-build/test_folders.txt | 0 4 files changed, 16 insertions(+), 10 deletions(-) delete mode 100644 synthtool/gcp/templates/python_notebooks/.cloud-build/test_folders.txt diff --git a/synthtool/gcp/templates/python_notebooks/.cloud-build/check_python_version.py b/synthtool/gcp/templates/python_notebooks/.cloud-build/check_python_version.py index a40348a3b..2d86e77b0 100644 --- a/synthtool/gcp/templates/python_notebooks/.cloud-build/check_python_version.py +++ b/synthtool/gcp/templates/python_notebooks/.cloud-build/check_python_version.py @@ -15,13 +15,13 @@ import sys MINIMUM_MAJOR_VERSION = 3 -MINIMUM_MINOR_VERSION = 3 +MINIMUM_MINOR_VERSION = 5 if ( sys.version_info.major < MINIMUM_MAJOR_VERSION and sys.version_info.minor < MINIMUM_MINOR_VERSION ): - print("Error: Python version less than 3.5") + print(f"Error: Python version less than {MINIMUM_MAJOR_VERSION}.{MINIMUM_MINOR_VERSION}") exit(1) else: print(f"Python version acceptable: {sys.version}") diff --git a/synthtool/gcp/templates/python_notebooks/.cloud-build/notebook-execution-test-cloudbuild.yaml b/synthtool/gcp/templates/python_notebooks/.cloud-build/notebook-execution-test-cloudbuild.yaml index de036e877..ee07264c3 100644 --- a/synthtool/gcp/templates/python_notebooks/.cloud-build/notebook-execution-test-cloudbuild.yaml +++ b/synthtool/gcp/templates/python_notebooks/.cloud-build/notebook-execution-test-cloudbuild.yaml @@ -10,19 +10,23 @@ steps: entrypoint: /bin/sh args: - -c - - 'python3 .cloud-build/check_python_version.py' + - 'python3 .cloud-build/CheckPythonVersion.py' # Fetch base branch if required - name: ${_PYTHON_IMAGE} entrypoint: /bin/sh args: - -c - 'if [ -n "${_BASE_BRANCH}" ]; then git fetch origin "${_BASE_BRANCH}":refs/remotes/origin/"${_BASE_BRANCH}"; else echo "Skipping fetch."; fi' + # Install Python dependencies + - name: ${_PYTHON_IMAGE} + entrypoint: pip + args: ['install', '--upgrade', '--user', '--requirement', '.cloud-build/requirements.txt'] # Install Python dependencies and run testing script - name: ${_PYTHON_IMAGE} entrypoint: /bin/sh args: - -c - - 'python3 -m pip install -U -r .cloud-build/requirements.txt && python3 -m pip freeze && python3 .cloud-build/execute_changed_notebooks.py --test_paths_file .cloud-build/test_folders.txt --base_branch "${_FORCED_BASE_BRANCH}" --output_folder ${BUILD_ID} --variable_project_id ${PROJECT_ID} --variable_region ${_GCP_REGION}' + - 'python3 -m pip freeze && python3 .cloud-build/ExecuteChangedNotebooks.py --test_paths_file "${_TEST_PATHS_FILE}" --base_branch "${_FORCED_BASE_BRANCH}" --output_folder ${BUILD_ID} --variable_project_id ${PROJECT_ID} --variable_region ${_GCP_REGION}' env: - 'IS_TESTING=1' # Manually copy artifacts to GCS @@ -30,7 +34,7 @@ steps: entrypoint: /bin/sh args: - -c - - 'if [ $(ls -pR "/workspace/${BUILD_ID}" | grep -v / | grep -v ^$ | wc -l) -ne 0 ]; then gsutil rsync -r "/workspace/${BUILD_ID}" "gs://${_GCS_ARTIFACTS_BUCKET}/test-artifacts/PR_${_PR_NUMBER}/BUILD_${BUILD_ID}/"; else echo "No artifacts to copy."; fi' + - 'if [ $(ls -pR "/workspace/${BUILD_ID}" | grep -v / | grep -v ^$ | wc -l) -ne 0 ]; then gsutil -m -q rsync -r "/workspace/${BUILD_ID}" "gs://${_GCS_ARTIFACTS_BUCKET}/test-artifacts/PR_${_PR_NUMBER}/BUILD_${BUILD_ID}/"; else echo "No artifacts to copy."; fi' # Fail if there is anything in the failure folder - name: ${_PYTHON_IMAGE} entrypoint: /bin/sh diff --git a/synthtool/gcp/templates/python_notebooks/.cloud-build/requirements.txt b/synthtool/gcp/templates/python_notebooks/.cloud-build/requirements.txt index c6a9a18ac..e6546f198 100644 --- a/synthtool/gcp/templates/python_notebooks/.cloud-build/requirements.txt +++ b/synthtool/gcp/templates/python_notebooks/.cloud-build/requirements.txt @@ -1,6 +1,8 @@ -ipython -jupyter>=1.0 +ipython>=7.0 +jupyter>=1.0 nbconvert>=6.0 -papermill -numpy -pandas \ No newline at end of file +papermill>=2.3 +numpy>=1.19 +pandas>=1.2 +matplotlib>=3.4 +tabulate \ No newline at end of file diff --git a/synthtool/gcp/templates/python_notebooks/.cloud-build/test_folders.txt b/synthtool/gcp/templates/python_notebooks/.cloud-build/test_folders.txt deleted file mode 100644 index e69de29bb..000000000 From 7aa53a708c2a4f3a4cd33b773e014d03d61f1501 Mon Sep 17 00:00:00 2001 From: Lo Ferris Date: Wed, 25 Aug 2021 17:45:24 -0700 Subject: [PATCH 05/19] addressing kokoro fail --- .../python_notebooks/.cloud-build/check_python_version.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/synthtool/gcp/templates/python_notebooks/.cloud-build/check_python_version.py b/synthtool/gcp/templates/python_notebooks/.cloud-build/check_python_version.py index 2d86e77b0..ced621898 100644 --- a/synthtool/gcp/templates/python_notebooks/.cloud-build/check_python_version.py +++ b/synthtool/gcp/templates/python_notebooks/.cloud-build/check_python_version.py @@ -21,7 +21,9 @@ sys.version_info.major < MINIMUM_MAJOR_VERSION and sys.version_info.minor < MINIMUM_MINOR_VERSION ): - print(f"Error: Python version less than {MINIMUM_MAJOR_VERSION}.{MINIMUM_MINOR_VERSION}") + print( + f"Error: Python version less than {MINIMUM_MAJOR_VERSION}.{MINIMUM_MINOR_VERSION}" + ) exit(1) else: print(f"Python version acceptable: {sys.version}") From 373043ebbee81a9504c49fb1f890b0ff583a1690 Mon Sep 17 00:00:00 2001 From: Lo Ferris Date: Fri, 27 Aug 2021 12:46:22 -0700 Subject: [PATCH 06/19] streamlining purpose of PR to cover only our shared testing pipeline --- .../gcp/templates/python_notebooks/.gitignore | 152 ------------- .../gcp/templates/python_notebooks/CODEOWNERS | 4 - .../gcp/templates/python_notebooks/LICENSE | 202 ------------------ .../gcp/templates/python_notebooks/README.md | 20 -- .../python_notebooks/docs/code-of-conduct.md | 93 -------- .../python_notebooks/docs/contributing.md | 28 --- .../notebooks/community/README.md | 3 - .../notebooks/official/README.md | 3 - .../.cloud-build/check_python_version.py | 0 .../cleanup/cleanup-cloudbuild.yaml | 0 .../cleanup/cleanup-requirements.txt | 0 .../.cloud-build/cleanup/cleanup.py | 0 .../cleanup/resource_cleanup_manager.py | 0 .../.cloud-build/execute_changed_notebooks.py | 0 .../.cloud-build/execute_notebook.py | 0 .../notebook-execution-test-cloudbuild.yaml | 0 .../.cloud-build/notebook_processors.py | 0 .../.cloud-build/requirements.txt | 0 .../.cloud-build/update_notebook_variables.py | 0 .../.github/ISSUE_TEMPLATE/bug_report.md | 0 .../ISSUE_TEMPLATE/sample_feature_request.md | 0 .../.github/pull-request-template.md | 0 .../.github/snippet-bot.yml | 0 .../.github/workflows/ci.yaml | 0 .../.github/workflows/linter/requirements.txt | 0 .../.github/workflows/linter/run_linter.sh | 0 26 files changed, 505 deletions(-) delete mode 100644 synthtool/gcp/templates/python_notebooks/.gitignore delete mode 100644 synthtool/gcp/templates/python_notebooks/CODEOWNERS delete mode 100644 synthtool/gcp/templates/python_notebooks/LICENSE delete mode 100644 synthtool/gcp/templates/python_notebooks/README.md delete mode 100644 synthtool/gcp/templates/python_notebooks/docs/code-of-conduct.md delete mode 100644 synthtool/gcp/templates/python_notebooks/docs/contributing.md delete mode 100644 synthtool/gcp/templates/python_notebooks/notebooks/community/README.md delete mode 100644 synthtool/gcp/templates/python_notebooks/notebooks/official/README.md rename synthtool/gcp/templates/{python_notebooks => python_notebooks_testing_pipeline}/.cloud-build/check_python_version.py (100%) rename synthtool/gcp/templates/{python_notebooks => python_notebooks_testing_pipeline}/.cloud-build/cleanup/cleanup-cloudbuild.yaml (100%) rename synthtool/gcp/templates/{python_notebooks => python_notebooks_testing_pipeline}/.cloud-build/cleanup/cleanup-requirements.txt (100%) rename synthtool/gcp/templates/{python_notebooks => python_notebooks_testing_pipeline}/.cloud-build/cleanup/cleanup.py (100%) rename synthtool/gcp/templates/{python_notebooks => python_notebooks_testing_pipeline}/.cloud-build/cleanup/resource_cleanup_manager.py (100%) rename synthtool/gcp/templates/{python_notebooks => python_notebooks_testing_pipeline}/.cloud-build/execute_changed_notebooks.py (100%) rename synthtool/gcp/templates/{python_notebooks => python_notebooks_testing_pipeline}/.cloud-build/execute_notebook.py (100%) rename synthtool/gcp/templates/{python_notebooks => python_notebooks_testing_pipeline}/.cloud-build/notebook-execution-test-cloudbuild.yaml (100%) rename synthtool/gcp/templates/{python_notebooks => python_notebooks_testing_pipeline}/.cloud-build/notebook_processors.py (100%) rename synthtool/gcp/templates/{python_notebooks => python_notebooks_testing_pipeline}/.cloud-build/requirements.txt (100%) rename synthtool/gcp/templates/{python_notebooks => python_notebooks_testing_pipeline}/.cloud-build/update_notebook_variables.py (100%) rename synthtool/gcp/templates/{python_notebooks => python_notebooks_testing_pipeline}/.github/ISSUE_TEMPLATE/bug_report.md (100%) rename synthtool/gcp/templates/{python_notebooks => python_notebooks_testing_pipeline}/.github/ISSUE_TEMPLATE/sample_feature_request.md (100%) rename synthtool/gcp/templates/{python_notebooks => python_notebooks_testing_pipeline}/.github/pull-request-template.md (100%) rename synthtool/gcp/templates/{python_notebooks => python_notebooks_testing_pipeline}/.github/snippet-bot.yml (100%) rename synthtool/gcp/templates/{python_notebooks => python_notebooks_testing_pipeline}/.github/workflows/ci.yaml (100%) rename synthtool/gcp/templates/{python_notebooks => python_notebooks_testing_pipeline}/.github/workflows/linter/requirements.txt (100%) rename synthtool/gcp/templates/{python_notebooks => python_notebooks_testing_pipeline}/.github/workflows/linter/run_linter.sh (100%) diff --git a/synthtool/gcp/templates/python_notebooks/.gitignore b/synthtool/gcp/templates/python_notebooks/.gitignore deleted file mode 100644 index d6703f5f8..000000000 --- a/synthtool/gcp/templates/python_notebooks/.gitignore +++ /dev/null @@ -1,152 +0,0 @@ -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class - -# C extensions -*.so - -# Distribution / packaging -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -share/python-wheels/ -*.egg-info/ -.installed.cfg -*.egg -MANIFEST - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.nox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -*.py,cover -.hypothesis/ -.pytest_cache/ -cover/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py -db.sqlite3 -db.sqlite3-journal - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -.pybuilder/ -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# IPython -profile_default/ -ipython_config.py - -# pyenv -# For a library or package, you might want to ignore these files since the code is -# intended to run in multiple environments; otherwise, check them in: -# .python-version - -# pipenv -# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. -# However, in case of collaboration, if having platform-specific dependencies or dependencies -# having no cross-platform support, pipenv may install dependencies that don't work, or not -# install all needed dependencies. -#Pipfile.lock - -# PEP 582; used by e.g. github.com/David-OConnor/pyflow -__pypackages__/ - -# Celery stuff -celerybeat-schedule -celerybeat.pid - -# SageMath parsed files -*.sage.py - -# Environments -.env -.venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ -.dmypy.json -dmypy.json - -# Pyre type checker -.pyre/ - -# pytype static type analyzer -.pytype/ - -# Cython debug symbols -cython_debug/ - -# OS junk files -[Tt]humbs.db -*.DS_Store - -# PyCharm -.idea/* - -# VS Code -.vscode/* - -*.log -*.err -*.pyc \ No newline at end of file diff --git a/synthtool/gcp/templates/python_notebooks/CODEOWNERS b/synthtool/gcp/templates/python_notebooks/CODEOWNERS deleted file mode 100644 index 3ceb126e2..000000000 --- a/synthtool/gcp/templates/python_notebooks/CODEOWNERS +++ /dev/null @@ -1,4 +0,0 @@ -# These owners will be the default owners for everything in -# the repo. Unless a later match takes precedence, -# @global-owner1 and @global-owner2 will be requested for -# review when someone opens a pull request. \ No newline at end of file diff --git a/synthtool/gcp/templates/python_notebooks/LICENSE b/synthtool/gcp/templates/python_notebooks/LICENSE deleted file mode 100644 index 7a4a3ea24..000000000 --- a/synthtool/gcp/templates/python_notebooks/LICENSE +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. \ No newline at end of file diff --git a/synthtool/gcp/templates/python_notebooks/README.md b/synthtool/gcp/templates/python_notebooks/README.md deleted file mode 100644 index f74eb61ed..000000000 --- a/synthtool/gcp/templates/python_notebooks/README.md +++ /dev/null @@ -1,20 +0,0 @@ -# Google Cloud [PRODUCT] Notebooks - -[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](LICENSE) - -Welcome to the Google Cloud [PRODUCT](#) Notebooks sample repository. - -## Overview - -The repository contains [Notebooks](#) - -## Contributing - -Contributions welcome! See the [Contributing Guide](#). - -## Getting help - -Please use the [issues page](#) to provide feedback or submit a bug report. - -## Disclaimer -This is not an officially supported Google product. The code in this repository is for demonstrative purposes only. \ No newline at end of file diff --git a/synthtool/gcp/templates/python_notebooks/docs/code-of-conduct.md b/synthtool/gcp/templates/python_notebooks/docs/code-of-conduct.md deleted file mode 100644 index 8ba5767ed..000000000 --- a/synthtool/gcp/templates/python_notebooks/docs/code-of-conduct.md +++ /dev/null @@ -1,93 +0,0 @@ -# Code of Conduct - -## Our Pledge - -In the interest of fostering an open and welcoming environment, we as -contributors and maintainers pledge to making participation in our project and -our community a harassment-free experience for everyone, regardless of age, body -size, disability, ethnicity, gender identity and expression, level of -experience, education, socio-economic status, nationality, personal appearance, -race, religion, or sexual identity and orientation. - -## Our Standards - -Examples of behavior that contributes to creating a positive environment -include: - -* Using welcoming and inclusive language -* Being respectful of differing viewpoints and experiences -* Gracefully accepting constructive criticism -* Focusing on what is best for the community -* Showing empathy towards other community members - -Examples of unacceptable behavior by participants include: - -* The use of sexualized language or imagery and unwelcome sexual attention or - advances -* Trolling, insulting/derogatory comments, and personal or political attacks -* Public or private harassment -* Publishing others' private information, such as a physical or electronic - address, without explicit permission -* Other conduct which could reasonably be considered inappropriate in a - professional setting - -## Our Responsibilities - -Project maintainers are responsible for clarifying the standards of acceptable -behavior and are expected to take appropriate and fair corrective action in -response to any instances of unacceptable behavior. - -Project maintainers have the right and responsibility to remove, edit, or reject -comments, commits, code, wiki edits, issues, and other contributions that are -not aligned to this Code of Conduct, or to ban temporarily or permanently any -contributor for other behaviors that they deem inappropriate, threatening, -offensive, or harmful. - -## Scope - -This Code of Conduct applies both within project spaces and in public spaces -when an individual is representing the project or its community. Examples of -representing a project or community include using an official project e-mail -address, posting via an official social media account, or acting as an appointed -representative at an online or offline event. Representation of a project may be -further defined and clarified by project maintainers. - -This Code of Conduct also applies outside the project spaces when the Project -Steward has a reasonable belief that an individual's behavior may have a -negative impact on the project or its community. - -## Conflict Resolution - -We do not believe that all conflict is bad; healthy debate and disagreement -often yield positive results. However, it is never okay to be disrespectful or -to engage in behavior that violates the project’s code of conduct. - -If you see someone violating the code of conduct, you are encouraged to address -the behavior directly with those involved. Many issues can be resolved quickly -and easily, and this gives people more control over the outcome of their -dispute. If you are unable to resolve the matter for any reason, or if the -behavior is threatening or harassing, report it. We are dedicated to providing -an environment where participants feel welcome and safe. - -Reports should be directed to *[PROJECT STEWARD NAME(s) AND EMAIL(s)]*, the -Project Steward(s) for *[PROJECT NAME]*. It is the Project Steward’s duty to -receive and address reported violations of the code of conduct. They will then -work with a committee consisting of representatives from the Open Source -Programs Office and the Google Open Source Strategy team. If for any reason you -are uncomfortable reaching out to the Project Steward, please email -opensource@google.com. - -We will investigate every complaint, but you may not receive a direct response. -We will use our discretion in determining when and how to follow up on reported -incidents, which may range from not taking action to permanent expulsion from -the project and project-sponsored spaces. We will notify the accused of the -report and provide them an opportunity to discuss it before any action is taken. -The identity of the reporter will be omitted from the details of the report -supplied to the accused. In potentially harmful situations, such as ongoing -harassment or threats to anyone's safety, we may take action without notice. - -## Attribution - -This Code of Conduct is adapted from the Contributor Covenant, version 1.4, -available at -https://www.contributor-covenant.org/version/1/4/code-of-conduct.html \ No newline at end of file diff --git a/synthtool/gcp/templates/python_notebooks/docs/contributing.md b/synthtool/gcp/templates/python_notebooks/docs/contributing.md deleted file mode 100644 index 3abd08301..000000000 --- a/synthtool/gcp/templates/python_notebooks/docs/contributing.md +++ /dev/null @@ -1,28 +0,0 @@ -# How to Contribute - -We'd love to accept your patches and contributions to this project. There are -just a few small guidelines you need to follow. - -## Contributor License Agreement - -Contributions to this project must be accompanied by a Contributor License -Agreement. You (or your employer) retain the copyright to your contribution; -this simply gives us permission to use and redistribute your contributions as -part of the project. Head over to to see -your current agreements on file or to sign a new one. - -You generally only need to submit a CLA once, so if you've already submitted one -(even if it was for a different project), you probably don't need to do it -again. - -## Code Reviews - -All submissions, including submissions by project members, require review. We -use GitHub pull requests for this purpose. Consult -[GitHub Help](https://help.github.com/articles/about-pull-requests/) for more -information on using pull requests. - -## Community Guidelines - -This project follows [Google's Open Source Community -Guidelines](https://opensource.google/conduct/). \ No newline at end of file diff --git a/synthtool/gcp/templates/python_notebooks/notebooks/community/README.md b/synthtool/gcp/templates/python_notebooks/notebooks/community/README.md deleted file mode 100644 index c148606fc..000000000 --- a/synthtool/gcp/templates/python_notebooks/notebooks/community/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# Google Cloud [PRODUCT] Official Notebooks - -The official tutorials are organized by Google Cloud [PRODUCT] products. \ No newline at end of file diff --git a/synthtool/gcp/templates/python_notebooks/notebooks/official/README.md b/synthtool/gcp/templates/python_notebooks/notebooks/official/README.md deleted file mode 100644 index 50071b817..000000000 --- a/synthtool/gcp/templates/python_notebooks/notebooks/official/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# Google Cloud [PRODUCT] Community Notebooks - -Community tutorials are organized by Google Cloud [PRODUCT] products and not officially maintined. \ No newline at end of file diff --git a/synthtool/gcp/templates/python_notebooks/.cloud-build/check_python_version.py b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/check_python_version.py similarity index 100% rename from synthtool/gcp/templates/python_notebooks/.cloud-build/check_python_version.py rename to synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/check_python_version.py diff --git a/synthtool/gcp/templates/python_notebooks/.cloud-build/cleanup/cleanup-cloudbuild.yaml b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/cleanup/cleanup-cloudbuild.yaml similarity index 100% rename from synthtool/gcp/templates/python_notebooks/.cloud-build/cleanup/cleanup-cloudbuild.yaml rename to synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/cleanup/cleanup-cloudbuild.yaml diff --git a/synthtool/gcp/templates/python_notebooks/.cloud-build/cleanup/cleanup-requirements.txt b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/cleanup/cleanup-requirements.txt similarity index 100% rename from synthtool/gcp/templates/python_notebooks/.cloud-build/cleanup/cleanup-requirements.txt rename to synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/cleanup/cleanup-requirements.txt diff --git a/synthtool/gcp/templates/python_notebooks/.cloud-build/cleanup/cleanup.py b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/cleanup/cleanup.py similarity index 100% rename from synthtool/gcp/templates/python_notebooks/.cloud-build/cleanup/cleanup.py rename to synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/cleanup/cleanup.py diff --git a/synthtool/gcp/templates/python_notebooks/.cloud-build/cleanup/resource_cleanup_manager.py b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/cleanup/resource_cleanup_manager.py similarity index 100% rename from synthtool/gcp/templates/python_notebooks/.cloud-build/cleanup/resource_cleanup_manager.py rename to synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/cleanup/resource_cleanup_manager.py diff --git a/synthtool/gcp/templates/python_notebooks/.cloud-build/execute_changed_notebooks.py b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/execute_changed_notebooks.py similarity index 100% rename from synthtool/gcp/templates/python_notebooks/.cloud-build/execute_changed_notebooks.py rename to synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/execute_changed_notebooks.py diff --git a/synthtool/gcp/templates/python_notebooks/.cloud-build/execute_notebook.py b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/execute_notebook.py similarity index 100% rename from synthtool/gcp/templates/python_notebooks/.cloud-build/execute_notebook.py rename to synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/execute_notebook.py diff --git a/synthtool/gcp/templates/python_notebooks/.cloud-build/notebook-execution-test-cloudbuild.yaml b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/notebook-execution-test-cloudbuild.yaml similarity index 100% rename from synthtool/gcp/templates/python_notebooks/.cloud-build/notebook-execution-test-cloudbuild.yaml rename to synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/notebook-execution-test-cloudbuild.yaml diff --git a/synthtool/gcp/templates/python_notebooks/.cloud-build/notebook_processors.py b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/notebook_processors.py similarity index 100% rename from synthtool/gcp/templates/python_notebooks/.cloud-build/notebook_processors.py rename to synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/notebook_processors.py diff --git a/synthtool/gcp/templates/python_notebooks/.cloud-build/requirements.txt b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/requirements.txt similarity index 100% rename from synthtool/gcp/templates/python_notebooks/.cloud-build/requirements.txt rename to synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/requirements.txt diff --git a/synthtool/gcp/templates/python_notebooks/.cloud-build/update_notebook_variables.py b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/update_notebook_variables.py similarity index 100% rename from synthtool/gcp/templates/python_notebooks/.cloud-build/update_notebook_variables.py rename to synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/update_notebook_variables.py diff --git a/synthtool/gcp/templates/python_notebooks/.github/ISSUE_TEMPLATE/bug_report.md b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.github/ISSUE_TEMPLATE/bug_report.md similarity index 100% rename from synthtool/gcp/templates/python_notebooks/.github/ISSUE_TEMPLATE/bug_report.md rename to synthtool/gcp/templates/python_notebooks_testing_pipeline/.github/ISSUE_TEMPLATE/bug_report.md diff --git a/synthtool/gcp/templates/python_notebooks/.github/ISSUE_TEMPLATE/sample_feature_request.md b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.github/ISSUE_TEMPLATE/sample_feature_request.md similarity index 100% rename from synthtool/gcp/templates/python_notebooks/.github/ISSUE_TEMPLATE/sample_feature_request.md rename to synthtool/gcp/templates/python_notebooks_testing_pipeline/.github/ISSUE_TEMPLATE/sample_feature_request.md diff --git a/synthtool/gcp/templates/python_notebooks/.github/pull-request-template.md b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.github/pull-request-template.md similarity index 100% rename from synthtool/gcp/templates/python_notebooks/.github/pull-request-template.md rename to synthtool/gcp/templates/python_notebooks_testing_pipeline/.github/pull-request-template.md diff --git a/synthtool/gcp/templates/python_notebooks/.github/snippet-bot.yml b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.github/snippet-bot.yml similarity index 100% rename from synthtool/gcp/templates/python_notebooks/.github/snippet-bot.yml rename to synthtool/gcp/templates/python_notebooks_testing_pipeline/.github/snippet-bot.yml diff --git a/synthtool/gcp/templates/python_notebooks/.github/workflows/ci.yaml b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.github/workflows/ci.yaml similarity index 100% rename from synthtool/gcp/templates/python_notebooks/.github/workflows/ci.yaml rename to synthtool/gcp/templates/python_notebooks_testing_pipeline/.github/workflows/ci.yaml diff --git a/synthtool/gcp/templates/python_notebooks/.github/workflows/linter/requirements.txt b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.github/workflows/linter/requirements.txt similarity index 100% rename from synthtool/gcp/templates/python_notebooks/.github/workflows/linter/requirements.txt rename to synthtool/gcp/templates/python_notebooks_testing_pipeline/.github/workflows/linter/requirements.txt diff --git a/synthtool/gcp/templates/python_notebooks/.github/workflows/linter/run_linter.sh b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.github/workflows/linter/run_linter.sh similarity index 100% rename from synthtool/gcp/templates/python_notebooks/.github/workflows/linter/run_linter.sh rename to synthtool/gcp/templates/python_notebooks_testing_pipeline/.github/workflows/linter/run_linter.sh From 7e9b33e3899bd6b69e8c7e1c04b90290bad2cd16 Mon Sep 17 00:00:00 2001 From: Lo Ferris Date: Fri, 27 Aug 2021 12:55:02 -0700 Subject: [PATCH 07/19] updating check for Python version and adding a licence to cloud-build yaml --- .../.cloud-build/check_python_version.py | 10 +++++----- .../notebook-execution-test-cloudbuild.yaml | 14 ++++++++++++++ 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/check_python_version.py b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/check_python_version.py index ced621898..06fc6e4ae 100644 --- a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/check_python_version.py +++ b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/check_python_version.py @@ -18,13 +18,13 @@ MINIMUM_MINOR_VERSION = 5 if ( - sys.version_info.major < MINIMUM_MAJOR_VERSION - and sys.version_info.minor < MINIMUM_MINOR_VERSION + sys.version_info.major >= MINIMUM_MAJOR_VERSION + or sys.version_info.minor >= MINIMUM_MINOR_VERSION ): + print(f"Python version acceptable: {sys.version}") + exit(0) +else: print( f"Error: Python version less than {MINIMUM_MAJOR_VERSION}.{MINIMUM_MINOR_VERSION}" ) exit(1) -else: - print(f"Python version acceptable: {sys.version}") - exit(0) diff --git a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/notebook-execution-test-cloudbuild.yaml b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/notebook-execution-test-cloudbuild.yaml index ee07264c3..ab362348a 100644 --- a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/notebook-execution-test-cloudbuild.yaml +++ b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/notebook-execution-test-cloudbuild.yaml @@ -1,3 +1,17 @@ +#!/usr/bin/env python +# # Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. steps: # Show the gcloud info and check if gcloud exists - name: ${_PYTHON_IMAGE} From a2d758173d810b6ea678f79fe3de744d2c14e25a Mon Sep 17 00:00:00 2001 From: Lo Ferris Date: Wed, 1 Sep 2021 16:07:33 -0700 Subject: [PATCH 08/19] removing PR templates --- .../.github/ISSUE_TEMPLATE/bug_report.md | 31 ------------------- .../ISSUE_TEMPLATE/sample_feature_request.md | 20 ------------ .../.github/pull-request-template.md | 12 ------- 3 files changed, 63 deletions(-) delete mode 100644 synthtool/gcp/templates/python_notebooks_testing_pipeline/.github/ISSUE_TEMPLATE/bug_report.md delete mode 100644 synthtool/gcp/templates/python_notebooks_testing_pipeline/.github/ISSUE_TEMPLATE/sample_feature_request.md delete mode 100644 synthtool/gcp/templates/python_notebooks_testing_pipeline/.github/pull-request-template.md diff --git a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.github/ISSUE_TEMPLATE/bug_report.md b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.github/ISSUE_TEMPLATE/bug_report.md deleted file mode 100644 index ebb13c156..000000000 --- a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.github/ISSUE_TEMPLATE/bug_report.md +++ /dev/null @@ -1,31 +0,0 @@ ---- -name: Bug report -about: For problems running the sample code please provide the following information. - ---- - -**Describe the bug** -A clear and concise description of what the bug is. Be sure to convey here whether it occurred locally or on the server (BigQuery, AI Platform, Google Dataflow) - -**What sample is this bug related to?** - -**Source code / logs** -Include any logs or source code that would be helpful to diagnose the problem. If including tracebacks, please include the full traceback. Large logs and files should be attached. Try to provide a reproducible test case that is the bare minimum necessary to generate the problem. - -**To Reproduce** -Steps to reproduce the behavior: -1. Go to '...' -2. Click on '....' -3. Scroll down to '....' -4. See error - -**Expected behavior** -A clear and concise description of what you expected to happen. - -**System Information** -- **OS Platform and Distribution (e.g., Linux Ubuntu 16.04)**: -- **Python version**: -- **Exact command to reproduce**: - -**Additional context** -Add any other context about the problem here. \ No newline at end of file diff --git a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.github/ISSUE_TEMPLATE/sample_feature_request.md b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.github/ISSUE_TEMPLATE/sample_feature_request.md deleted file mode 100644 index a825dfbdf..000000000 --- a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.github/ISSUE_TEMPLATE/sample_feature_request.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -name: Sample/Feature request -about: Suggest an idea for this project - ---- - -**Is your feature request related to a problem? Please describe.** -A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] - -**Describe the solution you'd like** -A clear and concise description of what you want to happen. - -**Key technologies to include** -* AI Platform specfic features (CPU, GPU, HP Tuning, TPU): -* Framework (BigQuery ML, AutoML, Tensorflow, Keras, scikit-learn, XGBoost, ...): -* Model: -* Dataset: - -**Additional context** -Add any other context or screenshots about the feature request here. \ No newline at end of file diff --git a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.github/pull-request-template.md b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.github/pull-request-template.md deleted file mode 100644 index 7fac8f48f..000000000 --- a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.github/pull-request-template.md +++ /dev/null @@ -1,12 +0,0 @@ -Fixes # - -Before submitting a Jupyter notebook, follow this mandatory checklist: - -- [ ] Use the [notebook template](#) as a starting point. -- [ ] Double check that all links, including the Colab and Github links to the notebook, are valid. -- [ ] Follow the style and grammar rules outlined in the above notebook template. -- [ ] Verify the notebook runs successfully in Colab since the automated tests cannot guarantee this even when it passes. -- [ ] Passes all the required automated checks -- [ ] You have consulted with a tech writer to see if tech writer review is necessary. If so, the notebook has been reviewed by a tech writer, and they have approved it. -- [ ] This notebook has been added to the CODEOWNERS file, pointing to the author or the author's team. If the CODEOWNERS file doesn't exist, create one in the nearest folder that makes sense. -- [ ] The Jupyter notebook cleans up any artifacts it has created (datasets, ML models, endpoints, etc) so as not to eat up unnecessary resources. \ No newline at end of file From 6b7deab9d1f718c84b383b97f942464ad8e4b63c Mon Sep 17 00:00:00 2001 From: Lo Ferris Date: Thu, 9 Sep 2021 13:19:35 -0700 Subject: [PATCH 09/19] updating nbQA version --- .../.github/workflows/linter/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.github/workflows/linter/requirements.txt b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.github/workflows/linter/requirements.txt index 02f8896b7..079bb4016 100644 --- a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.github/workflows/linter/requirements.txt +++ b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.github/workflows/linter/requirements.txt @@ -6,4 +6,4 @@ black==20.8b1 pyupgrade==2.7.3 isort==5.6.4 flake8==3.9.0 -nbqa==0.6.0 \ No newline at end of file +nbqa==1.1.0 \ No newline at end of file From 672ee731cca095e4b4ab598b1ae1ecc6b9bae1c6 Mon Sep 17 00:00:00 2001 From: Lo Ferris Date: Tue, 14 Sep 2021 18:38:18 -0700 Subject: [PATCH 10/19] upgrading nbqa and black, running black directly --- .../.github/workflows/linter/requirements.txt | 2 +- .../.github/workflows/linter/run_linter.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.github/workflows/linter/requirements.txt b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.github/workflows/linter/requirements.txt index 079bb4016..2f011649c 100644 --- a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.github/workflows/linter/requirements.txt +++ b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.github/workflows/linter/requirements.txt @@ -2,7 +2,7 @@ git+https://github.com/tensorflow/docs ipython jupyter nbconvert -black==20.8b1 +black==21.8b0 pyupgrade==2.7.3 isort==5.6.4 flake8==3.9.0 diff --git a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.github/workflows/linter/run_linter.sh b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.github/workflows/linter/run_linter.sh index 4ba3a8b06..7b7cadb22 100644 --- a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.github/workflows/linter/run_linter.sh +++ b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.github/workflows/linter/run_linter.sh @@ -86,7 +86,7 @@ if [ ${#notebooks[@]} -gt 0 ]; then FLAKE8_RTN=$? else echo "Running black..." - python3 -m nbqa black "$notebook" --nbqa-mutate + python3 -m black "$notebook" BLACK_RTN=$? echo "Running pyupgrade..." python3 -m nbqa pyupgrade "$notebook" --nbqa-mutate From 1812656eb6a68c5a20d55c35ce5361496a1b47eb Mon Sep 17 00:00:00 2001 From: Lo Ferris Date: Wed, 15 Sep 2021 15:45:28 -0700 Subject: [PATCH 11/19] revising cloud-build files to fit with kokoro --- .../.cloud-build/execute_changed_notebooks.py | 6 +++--- .../.cloud-build/execute_notebook.py | 7 +++---- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/execute_changed_notebooks.py b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/execute_changed_notebooks.py index b3af75362..9f2dcf61b 100644 --- a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/execute_changed_notebooks.py +++ b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/execute_changed_notebooks.py @@ -16,7 +16,7 @@ import pathlib import subprocess from pathlib import Path -from typing import Dict, List, Optional +from typing import List, Optional import execute_notebook @@ -68,7 +68,7 @@ def run_changed_notebooks( + test_paths ) else: - print(f"Looking for all notebooks.") + print("Looking for all notebooks.") notebooks = subprocess.check_output(["git", "ls-files"] + test_paths) notebooks = notebooks.decode("utf-8").split("\n") @@ -101,7 +101,7 @@ def run_changed_notebooks( "REGION": variable_region, }, ) - print(f"Notebook finished successfully.") + print("Notebook finished successfully.") passed_notebooks.append(notebook) except Exception as error: print(f"Notebook finished with failure: {error}") diff --git a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/execute_notebook.py b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/execute_notebook.py index 83ee38c77..03678a3a4 100644 --- a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/execute_notebook.py +++ b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/execute_notebook.py @@ -14,7 +14,7 @@ # limitations under the License. import sys import nbformat -from nbconvert.preprocessors import ExecutePreprocessor, CellExecutionError +from nbconvert.preprocessors import ExecutePreprocessor import os import errno from notebook_processors import RemoveNoExecuteCells, UpdateVariablesPreprocessor @@ -43,7 +43,7 @@ def execute_notebook( update_variables_preprocessor = UpdateVariablesPreprocessor( replacement_map=replacement_map ) - execute_preprocessor = ExecutePreprocessor(timeout=-1, kernel_name="python3") + ExecutePreprocessor(timeout=-1, kernel_name="python3") # Use no-execute preprocessor (nb, resources,) = remove_no_execute_cells_preprocessor.preprocess(nb) @@ -78,8 +78,7 @@ def execute_notebook( stderr_file=sys.stderr, ) - except Exception as error: - out = None + except Exception: print(f"Error executing the notebook: {notebook_file_path}.\n\n") has_error = True From ff9c271d3cac775556fead20457ab9f3777f6d66 Mon Sep 17 00:00:00 2001 From: Lo Ferris Date: Fri, 17 Sep 2021 16:06:06 -0700 Subject: [PATCH 12/19] updating versions of cloud-build suite --- ...ython_version.py => CheckPythonVersion.py} | 0 .../.cloud-build/ExecuteChangedNotebooks.py | 305 ++++++++++++++++++ .../.cloud-build/ExecuteNotebook.py | 171 ++++++++++ ...ok_processors.py => NotebookProcessors.py} | 4 +- ...ariables.py => UpdateNotebookVariables.py} | 0 .../.cloud-build/execute_changed_notebooks.py | 160 --------- .../.cloud-build/execute_notebook.py | 85 ----- 7 files changed, 478 insertions(+), 247 deletions(-) rename synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/{check_python_version.py => CheckPythonVersion.py} (100%) create mode 100644 synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/ExecuteChangedNotebooks.py create mode 100644 synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/ExecuteNotebook.py rename synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/{notebook_processors.py => NotebookProcessors.py} (95%) rename synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/{update_notebook_variables.py => UpdateNotebookVariables.py} (100%) delete mode 100644 synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/execute_changed_notebooks.py delete mode 100644 synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/execute_notebook.py diff --git a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/check_python_version.py b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/CheckPythonVersion.py similarity index 100% rename from synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/check_python_version.py rename to synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/CheckPythonVersion.py diff --git a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/ExecuteChangedNotebooks.py b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/ExecuteChangedNotebooks.py new file mode 100644 index 000000000..d8a00b660 --- /dev/null +++ b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/ExecuteChangedNotebooks.py @@ -0,0 +1,305 @@ +#!/usr/bin/env python +# # Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import argparse +import dataclasses +import datetime +import functools +import pathlib +import os +import subprocess +from pathlib import Path +from typing import List, Optional +import concurrent +from tabulate import tabulate + +import ExecuteNotebook + + +def str2bool(v): + if isinstance(v, bool): + return v + if v.lower() in ("yes", "true", "t", "y", "1"): + return True + elif v.lower() in ("no", "false", "f", "n", "0"): + return False + else: + raise argparse.ArgumentTypeError("Boolean value expected.") + + +def format_timedelta(delta: datetime.timedelta) -> str: + """Formats a timedelta duration to [N days] %H:%M:%S format""" + seconds = int(delta.total_seconds()) + + secs_in_a_day = 86400 + secs_in_a_hour = 3600 + secs_in_a_min = 60 + + days, seconds = divmod(seconds, secs_in_a_day) + hours, seconds = divmod(seconds, secs_in_a_hour) + minutes, seconds = divmod(seconds, secs_in_a_min) + + time_fmt = f"{hours:02d}:{minutes:02d}:{seconds:02d}" + + if days > 0: + suffix = "s" if days > 1 else "" + return f"{days} day{suffix} {time_fmt}" + + return time_fmt + + +@dataclasses.dataclass +class NotebookExecutionResult: + notebook: str + duration: datetime.timedelta + is_pass: bool + error_message: Optional[str] + + +def execute_notebook( + artifacts_path: str, + variable_project_id: str, + variable_region: str, + should_log_output: bool, + should_use_new_kernel: bool, + notebook: str, +) -> NotebookExecutionResult: + print(f"Running notebook: {notebook}") + + result = NotebookExecutionResult( + notebook=notebook, + duration=datetime.timedelta(seconds=0), + is_pass=False, + error_message=None, + ) + + # TODO: Handle cases where multiple notebooks have the same name + time_start = datetime.datetime.now() + try: + ExecuteNotebook.execute_notebook( + notebook_file_path=notebook, + output_file_folder=artifacts_path, + replacement_map={ + "PROJECT_ID": variable_project_id, + "REGION": variable_region, + }, + should_log_output=should_log_output, + should_use_new_kernel=should_use_new_kernel, + ) + result.duration = datetime.datetime.now() - time_start + result.is_pass = True + print(f"{notebook} PASSED in {format_timedelta(result.duration)}.") + except Exception as error: + result.duration = datetime.datetime.now() - time_start + result.is_pass = False + result.error_message = str(error) + print( + f"{notebook} FAILED in {format_timedelta(result.duration)}: {result.error_message}" + ) + + return result + + +def run_changed_notebooks( + test_paths_file: str, + base_branch: Optional[str], + output_folder: str, + variable_project_id: str, + variable_region: str, + should_parallelize: bool, + should_use_separate_kernels: bool, +): + """ + Run the notebooks that exist under the folders defined in the test_paths_file. + It only runs notebooks that have differences from the Git base_branch. + The executed notebooks are saved in the output_folder. + Variables are also injected into the notebooks such as the variable_project_id and variable_region. + Args: + test_paths_file (str): + Required. The new-line delimited file to folders and files that need checking. + Folders are checked recursively. + base_branch (str): + Optional. If provided, only the files that have changed from the base_branch will be checked. + If not provided, all files will be checked. + output_folder (str): + Required. The folder to write executed notebooks to. + variable_project_id (str): + Required. The value for PROJECT_ID to inject into notebooks. + variable_region (str): + Required. The value for REGION to inject into notebooks. + should_parallelize (bool): + Required. Should run notebooks in parallel using a thread pool as opposed to in sequence. + should_use_separate_kernels (bool): + Note: Dependencies don't install correctly when this is set to True + See https://github.com/nteract/papermill/issues/625 + Required. Should run each notebook in a separate and independent virtual environment. + """ + + test_paths = [] + with open(test_paths_file) as file: + lines = [line.strip() for line in file.readlines()] + lines = [line for line in lines if len(line) > 0] + test_paths = [line for line in lines] + + if len(test_paths) == 0: + raise RuntimeError("No test folders found.") + + print(f"Checking folders: {test_paths}") + + # Find notebooks + notebooks = [] + if base_branch: + print(f"Looking for notebooks that changed from branch: {base_branch}") + notebooks = subprocess.check_output( + ["git", "diff", "--name-only", f"origin/{base_branch}..."] + test_paths + ) + else: + print("Looking for all notebooks.") + notebooks = subprocess.check_output(["git", "ls-files"] + test_paths) + + notebooks = notebooks.decode("utf-8").split("\n") + notebooks = [notebook for notebook in notebooks if notebook.endswith(".ipynb")] + notebooks = [notebook for notebook in notebooks if len(notebook) > 0] + notebooks = [notebook for notebook in notebooks if Path(notebook).exists()] + + # Create paths + artifacts_path = Path(output_folder) + artifacts_path.mkdir(parents=True, exist_ok=True) + artifacts_path.joinpath("success").mkdir(parents=True, exist_ok=True) + artifacts_path.joinpath("failure").mkdir(parents=True, exist_ok=True) + + notebook_execution_results: List[NotebookExecutionResult] = [] + + if len(notebooks) > 0: + print(f"Found {len(notebooks)} modified notebooks: {notebooks}") + + if should_parallelize and len(notebooks) > 1: + print( + "Running notebooks in parallel, so no logs will be displayed. Please wait..." + ) + with concurrent.futures.ThreadPoolExecutor(max_workers=None) as executor: + notebook_execution_results = list( + executor.map( + functools.partial( + execute_notebook, + artifacts_path, + variable_project_id, + variable_region, + False, + should_use_separate_kernels, + ), + notebooks, + ) + ) + else: + notebook_execution_results = [ + execute_notebook( + artifacts_path=artifacts_path, + variable_project_id=variable_project_id, + variable_region=variable_region, + notebook=notebook, + should_log_output=True, + should_use_new_kernel=should_use_separate_kernels, + ) + for notebook in notebooks + ] + else: + print("No notebooks modified in this pull request.") + + print("\n=== RESULTS ===\n") + + notebooks_sorted = sorted( + notebook_execution_results, key=lambda result: result.is_pass, reverse=True, + ) + # Print results + print( + tabulate( + [ + [ + os.path.basename(os.path.normpath(result.notebook)), + "PASSED" if result.is_pass else "FAILED", + format_timedelta(result.duration), + result.error_message or "--", + ] + for result in notebooks_sorted + ], + headers=["file", "status", "duration", "error"], + ) + ) + + print("\n=== END RESULTS===\n") + + +parser = argparse.ArgumentParser(description="Run changed notebooks.") +parser.add_argument( + "--test_paths_file", + type=pathlib.Path, + help="The path to the file that has newline-limited folders of notebooks that should be tested.", + required=True, +) +parser.add_argument( + "--base_branch", + help="The base git branch to diff against to find changed files.", + required=False, +) +parser.add_argument( + "--output_folder", + type=pathlib.Path, + help="The path to the folder to store executed notebooks.", + required=True, +) +parser.add_argument( + "--variable_project_id", + type=str, + help="The GCP project id. This is used to inject a variable value into the notebook before running.", + required=True, +) +parser.add_argument( + "--variable_region", + type=str, + help="The GCP region. This is used to inject a variable value into the notebook before running.", + required=True, +) + +# Note: Dependencies don't install correctly when this is set to True +parser.add_argument( + "--should_parallelize", + type=str2bool, + nargs="?", + const=True, + default=False, + help="Should run notebooks in parallel.", +) + +# Note: This isn't guaranteed to work correctly due to existing Papermill issue +# See https://github.com/nteract/papermill/issues/625 +parser.add_argument( + "--should_use_separate_kernels", + type=str2bool, + nargs="?", + const=True, + default=False, + help="(Experimental) Should run each notebook in a separate and independent virtual environment.", +) + +args = parser.parse_args() +run_changed_notebooks( + test_paths_file=args.test_paths_file, + base_branch=args.base_branch, + output_folder=args.output_folder, + variable_project_id=args.variable_project_id, + variable_region=args.variable_region, + should_parallelize=args.should_parallelize, + should_use_separate_kernels=args.should_use_separate_kernels, +) diff --git a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/ExecuteNotebook.py b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/ExecuteNotebook.py new file mode 100644 index 000000000..f9fdbb857 --- /dev/null +++ b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/ExecuteNotebook.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python +# # Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import json +import sys +import nbformat +import os +import errno +from NotebookProcessors import RemoveNoExecuteCells, UpdateVariablesPreprocessor +from typing import Dict, Tuple +import papermill as pm +import shutil +import virtualenv +import uuid +from jupyter_client.kernelspecapp import KernelSpecManager + +# This script is used to execute a notebook and write out the output notebook. +# The replaces calling the nbconvert via command-line, which doesn't write the output notebook correctly when there are errors during execution. + +STAGING_FOLDER = "staging" +ENVIRONMENTS_PATH = "environments" +KERNELS_SPECS_PATH = "kernel_specs" + + +def create_and_install_kernel() -> Tuple[str, str]: + # Create environment + kernel_name = str(uuid.uuid4()) + env_name = f"{ENVIRONMENTS_PATH}/{kernel_name}" + # venv.create(env_name, system_site_packages=True, with_pip=True) + virtualenv.cli_run([env_name, "--system-site-packages"]) + + # Create kernel spec + kernel_spec = { + "argv": [ + f"{env_name}/bin/python", + "-m", + "ipykernel_launcher", + "-f", + "{connection_file}", + ], + "display_name": "Python 3", + "language": "python", + } + kernel_spec_folder = os.path.join(KERNELS_SPECS_PATH, kernel_name) + kernel_spec_file = os.path.join(kernel_spec_folder, "kernel.json") + + # Create kernel spec folder + if not os.path.exists(os.path.dirname(kernel_spec_file)): + try: + os.makedirs(os.path.dirname(kernel_spec_file)) + except OSError as exc: # Guard against race condition + if exc.errno != errno.EEXIST: + raise + + with open(kernel_spec_file, mode="w", encoding="utf-8") as f: + json.dump(kernel_spec, f) + + # Install kernel + kernel_spec_manager = KernelSpecManager() + kernel_spec_manager.install_kernel_spec( + source_dir=kernel_spec_folder, kernel_name=kernel_name + ) + + return kernel_name, env_name + + +def execute_notebook( + notebook_file_path: str, + output_file_folder: str, + replacement_map: Dict[str, str], + should_log_output: bool, + should_use_new_kernel: bool, +): + # Create staging directory if it doesn't exist + staging_file_path = f"{STAGING_FOLDER}/{notebook_file_path}" + if not os.path.exists(os.path.dirname(staging_file_path)): + try: + os.makedirs(os.path.dirname(staging_file_path)) + except OSError as exc: # Guard against race condition + if exc.errno != errno.EEXIST: + raise + + file_name = os.path.basename(os.path.normpath(notebook_file_path)) + + # Create environments folder + if not os.path.exists(ENVIRONMENTS_PATH): + try: + os.makedirs(ENVIRONMENTS_PATH) + except OSError as exc: # Guard against race condition + if exc.errno != errno.EEXIST: + raise + + # Create and install kernel + kernel_name = next( + iter(KernelSpecManager().find_kernel_specs().keys()), None + ) # Find first existing kernel and use as default + env_name = None + if should_use_new_kernel: + kernel_name, env_name = create_and_install_kernel() + + # Read notebook + with open(notebook_file_path) as f: + nb = nbformat.read(f, as_version=4) + + has_error = False + + # Execute notebook + try: + # Create preprocessors + remove_no_execute_cells_preprocessor = RemoveNoExecuteCells() + update_variables_preprocessor = UpdateVariablesPreprocessor( + replacement_map=replacement_map + ) + + # Use no-execute preprocessor + (nb, resources,) = remove_no_execute_cells_preprocessor.preprocess(nb) + + (nb, resources) = update_variables_preprocessor.preprocess(nb, resources) + + # print(f"Staging modified notebook to: {staging_file_path}") + with open(staging_file_path, mode="w", encoding="utf-8") as f: + nbformat.write(nb, f) + + # Execute notebook + pm.execute_notebook( + input_path=staging_file_path, + output_path=staging_file_path, + kernel_name=kernel_name, + progress_bar=should_log_output, + request_save_on_cell_execute=should_log_output, + log_output=should_log_output, + stdout_file=sys.stdout if should_log_output else None, + stderr_file=sys.stderr if should_log_output else None, + ) + except Exception: + # print(f"Error executing the notebook: {notebook_file_path}.\n\n") + has_error = True + + raise + + finally: + # Clear env + if env_name is not None: + shutil.rmtree(path=env_name) + + # Copy execute notebook + output_file_path = os.path.join( + output_file_folder, "failure" if has_error else "success", file_name + ) + + # Create directories if they don't exist + if not os.path.exists(os.path.dirname(output_file_path)): + try: + os.makedirs(os.path.dirname(output_file_path)) + except OSError as exc: # Guard against race condition + if exc.errno != errno.EEXIST: + raise + + # print(f"Writing output to: {output_file_path}") + shutil.move(staging_file_path, output_file_path) diff --git a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/notebook_processors.py b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/NotebookProcessors.py similarity index 95% rename from synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/notebook_processors.py rename to synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/NotebookProcessors.py index 2fbe9fe63..90a61a51c 100644 --- a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/notebook_processors.py +++ b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/NotebookProcessors.py @@ -14,7 +14,7 @@ # limitations under the License. from nbconvert.preprocessors import Preprocessor from typing import Dict -import update_notebook_variables +import UpdateNotebookVariables class RemoveNoExecuteCells(Preprocessor): @@ -40,7 +40,7 @@ def update_variables(content: str, replacement_map: Dict[str, str]): # VARIABLE_NAME = '[description]' for variable_name, variable_value in replacement_map.items(): - content = update_notebook_variables.get_updated_value( + content = UpdateNotebookVariables.get_updated_value( content=content, variable_name=variable_name, variable_value=variable_value, diff --git a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/update_notebook_variables.py b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/UpdateNotebookVariables.py similarity index 100% rename from synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/update_notebook_variables.py rename to synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/UpdateNotebookVariables.py diff --git a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/execute_changed_notebooks.py b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/execute_changed_notebooks.py deleted file mode 100644 index 9f2dcf61b..000000000 --- a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/execute_changed_notebooks.py +++ /dev/null @@ -1,160 +0,0 @@ -#!/usr/bin/env python -# # Copyright 2021 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import argparse -import pathlib -import subprocess -from pathlib import Path -from typing import List, Optional - -import execute_notebook - - -def run_changed_notebooks( - test_paths_file: str, - output_folder: str, - variable_project_id: str, - variable_region: str, - base_branch: Optional[str], -): - """ - Run the notebooks that exist under the folders defined in the test_paths_file. - It only runs notebooks that have differences from the Git base_branch. - The executed notebooks are saved in the output_folder. - Variables are also injected into the notebooks such as the variable_project_id and variable_region. - Args: - test_paths_file (str): - Required. The new-line delimited file to folders and files that need checking. - Folders are checked recursively. - base_branch (str): - Optional. If provided, only the files that have changed from the base_branch will be checked. - If not provided, all files will be checked. - output_folder (str): - Required. The folder to write executed notebooks to. - variable_project_id (str): - Required. The value for PROJECT_ID to inject into notebooks. - variable_region (str): - Required. The value for REGION to inject into notebooks. - """ - - test_paths = [] - with open(test_paths_file) as file: - lines = [line.strip() for line in file.readlines()] - lines = [line for line in lines if len(line) > 0] - test_paths = [line for line in lines] - - if len(test_paths) == 0: - raise RuntimeError("No test folders found.") - - print(f"Checking folders: {test_paths}") - - # Find notebooks - notebooks = [] - if base_branch: - print(f"Looking for notebooks that changed from branch: {base_branch}") - notebooks = subprocess.check_output( - ["git", "diff", "--name-only", f"origin/{base_branch}...", "--"] - + test_paths - ) - else: - print("Looking for all notebooks.") - notebooks = subprocess.check_output(["git", "ls-files"] + test_paths) - - notebooks = notebooks.decode("utf-8").split("\n") - notebooks = [notebook for notebook in notebooks if notebook.endswith(".ipynb")] - notebooks = [notebook for notebook in notebooks if len(notebook) > 0] - notebooks = [notebook for notebook in notebooks if Path(notebook).exists()] - - # Create paths - artifacts_path = Path(output_folder) - artifacts_path.mkdir(parents=True, exist_ok=True) - artifacts_path.joinpath("success").mkdir(parents=True, exist_ok=True) - artifacts_path.joinpath("failure").mkdir(parents=True, exist_ok=True) - - passed_notebooks: List[str] = [] - failed_notebooks: List[str] = [] - - if len(notebooks) > 0: - print(f"Found {len(notebooks)} modified notebooks: {notebooks}") - - for notebook in notebooks: - print(f"Running notebook: {notebook}") - - # TODO: Handle cases where multiple notebooks have the same name - try: - execute_notebook.execute_notebook( - notebook_file_path=notebook, - output_file_folder=artifacts_path, - replacement_map={ - "PROJECT_ID": variable_project_id, - "REGION": variable_region, - }, - ) - print("Notebook finished successfully.") - passed_notebooks.append(notebook) - except Exception as error: - print(f"Notebook finished with failure: {error}") - failed_notebooks.append(notebook) - else: - print("No notebooks modified in this pull request.") - - if len(failed_notebooks) > 0: - print(f"{len(failed_notebooks)} notebooks failed:") - print(failed_notebooks) - print(f"{len(passed_notebooks)} notebooks passed:") - print(passed_notebooks) - elif len(passed_notebooks) > 0: - print("All notebooks executed successfully:") - print(passed_notebooks) - - -parser = argparse.ArgumentParser(description="Run changed notebooks.") -parser.add_argument( - "--test_paths_file", - type=pathlib.Path, - help="The path to the file that has newline-limited folders of notebooks that should be tested.", - required=True, -) -parser.add_argument( - "--base_branch", - help="The base git branch to diff against to find changed files.", - required=False, -) -parser.add_argument( - "--output_folder", - type=pathlib.Path, - help="The path to the folder to store executed notebooks.", - required=True, -) -parser.add_argument( - "--variable_project_id", - type=str, - help="The GCP project id. This is used to inject a variable value into the notebook before running.", - required=True, -) -parser.add_argument( - "--variable_region", - type=str, - help="The GCP region. This is used to inject a variable value into the notebook before running.", - required=True, -) - -args = parser.parse_args() -run_changed_notebooks( - test_paths_file=args.test_paths_file, - base_branch=args.base_branch, - output_folder=args.output_folder, - variable_project_id=args.variable_project_id, - variable_region=args.variable_region, -) diff --git a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/execute_notebook.py b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/execute_notebook.py deleted file mode 100644 index 03678a3a4..000000000 --- a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/execute_notebook.py +++ /dev/null @@ -1,85 +0,0 @@ -#!/usr/bin/env python -# # Copyright 2021 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import sys -import nbformat -from nbconvert.preprocessors import ExecutePreprocessor -import os -import errno -from notebook_processors import RemoveNoExecuteCells, UpdateVariablesPreprocessor -from typing import Dict -import papermill as pm - -# This script is used to execute a notebook and write out the output notebook. -# The replaces calling the nbconvert via command-line, which doesn't write the output notebook correctly when there are errors during execution. - - -def execute_notebook( - notebook_file_path: str, output_file_folder: str, replacement_map: Dict[str, str] -): - file_name = os.path.basename(os.path.normpath(notebook_file_path)) - - # Read notebook - with open(notebook_file_path) as f: - nb = nbformat.read(f, as_version=4) - - has_error = False - - # Execute notebook - try: - # Create preprocessors - remove_no_execute_cells_preprocessor = RemoveNoExecuteCells() - update_variables_preprocessor = UpdateVariablesPreprocessor( - replacement_map=replacement_map - ) - ExecutePreprocessor(timeout=-1, kernel_name="python3") - - # Use no-execute preprocessor - (nb, resources,) = remove_no_execute_cells_preprocessor.preprocess(nb) - - (nb, resources) = update_variables_preprocessor.preprocess(nb, resources) - - # Execute notebook - # out = execute_preprocessor.preprocess(nb, resources) - output_file_path = os.path.join( - output_file_folder, "failure" if has_error else "success", file_name - ) - - # Create directories if they don't exist - if not os.path.exists(os.path.dirname(output_file_path)): - try: - os.makedirs(os.path.dirname(output_file_path)) - except OSError as exc: # Guard against race condition - if exc.errno != errno.EEXIST: - raise - - print(f"Writing modified notebook to: {output_file_path}") - with open(output_file_path, mode="w", encoding="utf-8") as f: - nbformat.write(nb, f) - - pm.execute_notebook( - input_path=output_file_path, - output_path=output_file_path, - progress_bar=True, - request_save_on_cell_execute=True, - log_output=True, - stdout_file=sys.stdout, - stderr_file=sys.stderr, - ) - - except Exception: - print(f"Error executing the notebook: {notebook_file_path}.\n\n") - has_error = True - - raise From 990627082c635ca8d62e04931cb3a15b64c272a0 Mon Sep 17 00:00:00 2001 From: Lo Ferris Date: Wed, 22 Sep 2021 15:19:31 -0700 Subject: [PATCH 13/19] addressed comments on cleanup setup --- .../.cloud-build/ExecuteChangedNotebooks.py | 1 - .../.cloud-build/cleanup/cleanup.py | 12 ++++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/ExecuteChangedNotebooks.py b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/ExecuteChangedNotebooks.py index d8a00b660..53974f26a 100644 --- a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/ExecuteChangedNotebooks.py +++ b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/ExecuteChangedNotebooks.py @@ -84,7 +84,6 @@ def execute_notebook( error_message=None, ) - # TODO: Handle cases where multiple notebooks have the same name time_start = datetime.datetime.now() try: ExecuteNotebook.execute_notebook( diff --git a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/cleanup/cleanup.py b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/cleanup/cleanup.py index f3976bf61..35ec9dd20 100644 --- a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/cleanup/cleanup.py +++ b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/cleanup/cleanup.py @@ -38,13 +38,13 @@ def run_cleanup_managers(managers: List[ResourceCleanupManager], is_dry_run: boo else: manager.delete(resource) - print("") - -is_dry_run = False - -if is_dry_run: +def set_dry_run(dry_run_status: bool): + if dry_run_status is True: + return True print("Starting cleanup in dry run mode...") + return False + # List of all cleanup managers managers = [ @@ -53,4 +53,4 @@ def run_cleanup_managers(managers: List[ResourceCleanupManager], is_dry_run: boo ModelResourceCleanupManager(), ] -run_cleanup_managers(managers=managers, is_dry_run=is_dry_run) +run_cleanup_managers(managers=managers, is_dry_run=set_dry_run(False)) From f03fb8995878778900fe45c665a2d3a8e50a9d5e Mon Sep 17 00:00:00 2001 From: Lo Ferris Date: Thu, 30 Sep 2021 17:40:43 -0700 Subject: [PATCH 14/19] make requirements more compatible with renovatebot --- .../.cloud-build/cleanup/cleanup-requirements.txt | 1 - .../.cloud-build/cleanup/requirements.txt | 1 + .../.cloud-build/requirements.txt | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) delete mode 100644 synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/cleanup/cleanup-requirements.txt create mode 100644 synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/cleanup/requirements.txt diff --git a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/cleanup/cleanup-requirements.txt b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/cleanup/cleanup-requirements.txt deleted file mode 100644 index 983ca6b4c..000000000 --- a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/cleanup/cleanup-requirements.txt +++ /dev/null @@ -1 +0,0 @@ -google-cloud-aiplatform \ No newline at end of file diff --git a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/cleanup/requirements.txt b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/cleanup/requirements.txt new file mode 100644 index 000000000..8a69bdc70 --- /dev/null +++ b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/cleanup/requirements.txt @@ -0,0 +1 @@ +google-cloud-aiplatform>=1.4.3 \ No newline at end of file diff --git a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/requirements.txt b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/requirements.txt index e6546f198..8268567cd 100644 --- a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/requirements.txt +++ b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/requirements.txt @@ -5,4 +5,4 @@ papermill>=2.3 numpy>=1.19 pandas>=1.2 matplotlib>=3.4 -tabulate \ No newline at end of file +tabulate>=0.8.9 \ No newline at end of file From 00cdb6867bfca9154ad5b26297ac47e3a9a363f2 Mon Sep 17 00:00:00 2001 From: Lo Ferris Date: Thu, 30 Sep 2021 17:43:50 -0700 Subject: [PATCH 15/19] pinning versions --- .../.cloud-build/cleanup/requirements.txt | 2 +- .../.cloud-build/requirements.txt | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/cleanup/requirements.txt b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/cleanup/requirements.txt index 8a69bdc70..87be9a131 100644 --- a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/cleanup/requirements.txt +++ b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/cleanup/requirements.txt @@ -1 +1 @@ -google-cloud-aiplatform>=1.4.3 \ No newline at end of file +google-cloud-aiplatform==1.4.3 \ No newline at end of file diff --git a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/requirements.txt b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/requirements.txt index 8268567cd..81a3079b8 100644 --- a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/requirements.txt +++ b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/requirements.txt @@ -1,8 +1,8 @@ -ipython>=7.0 -jupyter>=1.0 -nbconvert>=6.0 -papermill>=2.3 -numpy>=1.19 -pandas>=1.2 -matplotlib>=3.4 -tabulate>=0.8.9 \ No newline at end of file +ipython==7.0 +jupyter==1.0 +nbconvert==6.0 +papermill==2.3 +numpy==1.19 +pandas==1.2 +matplotlib==3.4 +tabulate==0.8.9 \ No newline at end of file From c3ee69a77a9f1d7f7c03d4b2bee1bbc488771c7c Mon Sep 17 00:00:00 2001 From: Lo Ferris Date: Thu, 30 Sep 2021 17:57:51 -0700 Subject: [PATCH 16/19] used default artifacts pattern instead of manual copying of artifacts --- .../notebook-execution-test-cloudbuild.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/notebook-execution-test-cloudbuild.yaml b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/notebook-execution-test-cloudbuild.yaml index ab362348a..7e68b3413 100644 --- a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/notebook-execution-test-cloudbuild.yaml +++ b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/notebook-execution-test-cloudbuild.yaml @@ -43,16 +43,16 @@ steps: - 'python3 -m pip freeze && python3 .cloud-build/ExecuteChangedNotebooks.py --test_paths_file "${_TEST_PATHS_FILE}" --base_branch "${_FORCED_BASE_BRANCH}" --output_folder ${BUILD_ID} --variable_project_id ${PROJECT_ID} --variable_region ${_GCP_REGION}' env: - 'IS_TESTING=1' - # Manually copy artifacts to GCS - - name: gcr.io/cloud-builders/gsutil - entrypoint: /bin/sh - args: - - -c - - 'if [ $(ls -pR "/workspace/${BUILD_ID}" | grep -v / | grep -v ^$ | wc -l) -ne 0 ]; then gsutil -m -q rsync -r "/workspace/${BUILD_ID}" "gs://${_GCS_ARTIFACTS_BUCKET}/test-artifacts/PR_${_PR_NUMBER}/BUILD_${BUILD_ID}/"; else echo "No artifacts to copy."; fi' # Fail if there is anything in the failure folder - name: ${_PYTHON_IMAGE} entrypoint: /bin/sh args: - -c - 'echo "Download executed notebooks with this command: \"mkdir -p artifacts && gsutil rsync -r gs://${_GCS_ARTIFACTS_BUCKET}/test-artifacts/PR_${_PR_NUMBER}/BUILD_${BUILD_ID} artifacts/\"" && if [ "$(ls -A /workspace/${BUILD_ID}/failure | wc -l)" -ne 0 ]; then exit 1; else exit 0; fi' -timeout: 86400s \ No newline at end of file + timeout: 86400s + # Copy artifacts to Google Storage + artifacts: + objects: + location: gs://${_GCS_ARTIFACTS_BUCKET}/test-artifacts/PR_${_PR_NUMBER}/BUILD_${BUILD_ID}/ + paths: + - /workspace/${BUILD_ID} \ No newline at end of file From f86f6ac76990812523972da647a30e3c436ea7fc Mon Sep 17 00:00:00 2001 From: Lo Ferris Date: Fri, 15 Oct 2021 15:11:50 -0700 Subject: [PATCH 17/19] reformatting commands --- .../.cloud-build/notebook-execution-test-cloudbuild.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/notebook-execution-test-cloudbuild.yaml b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/notebook-execution-test-cloudbuild.yaml index 7e68b3413..c48543df4 100644 --- a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/notebook-execution-test-cloudbuild.yaml +++ b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/notebook-execution-test-cloudbuild.yaml @@ -39,8 +39,10 @@ steps: - name: ${_PYTHON_IMAGE} entrypoint: /bin/sh args: - - -c - - 'python3 -m pip freeze && python3 .cloud-build/ExecuteChangedNotebooks.py --test_paths_file "${_TEST_PATHS_FILE}" --base_branch "${_FORCED_BASE_BRANCH}" --output_folder ${BUILD_ID} --variable_project_id ${PROJECT_ID} --variable_region ${_GCP_REGION}' + - '-c' + - | + python3 -m pip freeze && python3 .cloud-build/ExecuteChangedNotebooks.py \ + --test_paths_file "${_TEST_PATHS_FILE}" --base_branch "${_FORCED_BASE_BRANCH}" --output_folder ${BUILD_ID} --variable_project_id ${PROJECT_ID} --variable_region ${_GCP_REGION} env: - 'IS_TESTING=1' # Fail if there is anything in the failure folder From 3c2b2259ecc2122749c7b0d15ff3ce76b3169d01 Mon Sep 17 00:00:00 2001 From: Lo Ferris Date: Fri, 15 Oct 2021 15:22:14 -0700 Subject: [PATCH 18/19] adding defaults --- .../notebook-execution-test-cloudbuild.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/notebook-execution-test-cloudbuild.yaml b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/notebook-execution-test-cloudbuild.yaml index c48543df4..95c759c5e 100644 --- a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/notebook-execution-test-cloudbuild.yaml +++ b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/notebook-execution-test-cloudbuild.yaml @@ -14,29 +14,29 @@ # limitations under the License. steps: # Show the gcloud info and check if gcloud exists - - name: ${_PYTHON_IMAGE} + - name: 'gcr.io/cloud-devrel-public-resources/python-samples-testing-docker:latest' entrypoint: /bin/sh args: - -c - 'gcloud config list' # Check the Python version - - name: ${_PYTHON_IMAGE} + - name: 'gcr.io/cloud-devrel-public-resources/python-samples-testing-docker:latest' entrypoint: /bin/sh args: - -c - 'python3 .cloud-build/CheckPythonVersion.py' # Fetch base branch if required - - name: ${_PYTHON_IMAGE} + - name: 'gcr.io/cloud-devrel-public-resources/python-samples-testing-docker:latest' entrypoint: /bin/sh args: - -c - 'if [ -n "${_BASE_BRANCH}" ]; then git fetch origin "${_BASE_BRANCH}":refs/remotes/origin/"${_BASE_BRANCH}"; else echo "Skipping fetch."; fi' # Install Python dependencies - - name: ${_PYTHON_IMAGE} + - name: 'gcr.io/cloud-devrel-public-resources/python-samples-testing-docker:latest' entrypoint: pip args: ['install', '--upgrade', '--user', '--requirement', '.cloud-build/requirements.txt'] # Install Python dependencies and run testing script - - name: ${_PYTHON_IMAGE} + - name: 'gcr.io/cloud-devrel-public-resources/python-samples-testing-docker:latest' entrypoint: /bin/sh args: - '-c' @@ -46,7 +46,7 @@ steps: env: - 'IS_TESTING=1' # Fail if there is anything in the failure folder - - name: ${_PYTHON_IMAGE} + - name: 'gcr.io/cloud-devrel-public-resources/python-samples-testing-docker:latest' entrypoint: /bin/sh args: - -c From cbc422072f1129e7d10685e7d9ad59bd8b0aec65 Mon Sep 17 00:00:00 2001 From: Lo Ferris Date: Tue, 19 Oct 2021 15:59:08 -0700 Subject: [PATCH 19/19] reverting to previous cloud-build.yaml file --- .../notebook-execution-test-cloudbuild.yaml | 34 +++++-------------- 1 file changed, 9 insertions(+), 25 deletions(-) diff --git a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/notebook-execution-test-cloudbuild.yaml b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/notebook-execution-test-cloudbuild.yaml index 95c759c5e..15d1d8283 100644 --- a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/notebook-execution-test-cloudbuild.yaml +++ b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/notebook-execution-test-cloudbuild.yaml @@ -1,17 +1,3 @@ -#!/usr/bin/env python -# # Copyright 2021 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. steps: # Show the gcloud info and check if gcloud exists - name: 'gcr.io/cloud-devrel-public-resources/python-samples-testing-docker:latest' @@ -39,22 +25,20 @@ steps: - name: 'gcr.io/cloud-devrel-public-resources/python-samples-testing-docker:latest' entrypoint: /bin/sh args: - - '-c' - - | - python3 -m pip freeze && python3 .cloud-build/ExecuteChangedNotebooks.py \ - --test_paths_file "${_TEST_PATHS_FILE}" --base_branch "${_FORCED_BASE_BRANCH}" --output_folder ${BUILD_ID} --variable_project_id ${PROJECT_ID} --variable_region ${_GCP_REGION} + - -c + - 'python3 -m pip freeze && python3 .cloud-build/ExecuteChangedNotebooks.py --test_paths_file "${_TEST_PATHS_FILE}" --base_branch "${_FORCED_BASE_BRANCH}" --output_folder ${BUILD_ID} --variable_project_id ${PROJECT_ID} --variable_region ${_GCP_REGION}' env: - 'IS_TESTING=1' + # Manually copy artifacts to GCS + - name: gcr.io/cloud-builders/gsutil + entrypoint: /bin/sh + args: + - -c + - 'if [ $(ls -pR "/workspace/${BUILD_ID}" | grep -v / | grep -v ^$ | wc -l) -ne 0 ]; then gsutil -m -q rsync -r "/workspace/${BUILD_ID}" "gs://${_GCS_ARTIFACTS_BUCKET}/test-artifacts/PR_${_PR_NUMBER}/BUILD_${BUILD_ID}/"; else echo "No artifacts to copy."; fi' # Fail if there is anything in the failure folder - name: 'gcr.io/cloud-devrel-public-resources/python-samples-testing-docker:latest' entrypoint: /bin/sh args: - -c - 'echo "Download executed notebooks with this command: \"mkdir -p artifacts && gsutil rsync -r gs://${_GCS_ARTIFACTS_BUCKET}/test-artifacts/PR_${_PR_NUMBER}/BUILD_${BUILD_ID} artifacts/\"" && if [ "$(ls -A /workspace/${BUILD_ID}/failure | wc -l)" -ne 0 ]; then exit 1; else exit 0; fi' - timeout: 86400s - # Copy artifacts to Google Storage - artifacts: - objects: - location: gs://${_GCS_ARTIFACTS_BUCKET}/test-artifacts/PR_${_PR_NUMBER}/BUILD_${BUILD_ID}/ - paths: - - /workspace/${BUILD_ID} \ No newline at end of file +timeout: 86400s \ No newline at end of file