From 4333d1a3031c58e75099827f86999b507079f5fc Mon Sep 17 00:00:00 2001 From: Jirka Date: Sat, 12 Nov 2022 13:10:51 +0100 Subject: [PATCH 1/4] fix include package --- .actions/assistant.py | 49 ++++++++++++++++++++++++++++--------------- 1 file changed, 32 insertions(+), 17 deletions(-) diff --git a/.actions/assistant.py b/.actions/assistant.py index 0c6b567885a2c..d789d7f4a1567 100644 --- a/.actions/assistant.py +++ b/.actions/assistant.py @@ -1,12 +1,15 @@ +import logging import os import re +import shutil from itertools import chain +from os.path import dirname, isfile from pathlib import Path -from pprint import pprint from typing import Dict, List, Optional, Sequence, Tuple import pkg_resources +_PATH_ROOT = dirname((dirname(__file__))) REQUIREMENT_FILES = { "pytorch": ( "requirements/pytorch/base.txt", @@ -65,7 +68,11 @@ def _replace_imports(lines: List[str], mapping: List[Tuple[str, str]]) -> List[s def copy_replace_imports( source_dir: str, source_imports: List[str], target_imports: List[str], target_dir: Optional[str] = None ) -> None: - print(f"Replacing imports: {locals()}") + """Copy package content with import adjustments. + + >>> _ = copy_replace_imports(os.path.join(_PATH_ROOT, "src"), ["lightning_app"], ["lightning.app"], os.path.join(_PATH_ROOT, "src", "lightning")) + """ + logging.info(f"Replacing imports: {locals()}") assert len(source_imports) == len(target_imports), ( "source and target imports must have the same length, " f"source: {len(source_imports)}, target: {len(target_imports)}" @@ -75,19 +82,27 @@ def copy_replace_imports( ls = _retrieve_files(source_dir) for fp in ls: - if fp.endswith(".py") or not fp.endswith(".pyc"): - with open(fp, encoding="utf-8") as fo: - try: - lines = fo.readlines() - except UnicodeDecodeError: - # a binary file, skip - print(f"Skipped replacing imports for {fp}") - continue - lines = _replace_imports(lines, list(zip(source_imports, target_imports))) - fp_new = fp.replace(source_dir, target_dir) - os.makedirs(os.path.dirname(fp_new), exist_ok=True) - with open(fp_new, "w", encoding="utf-8") as fo: - fo.writelines(lines) + fp_new = fp.replace(source_dir, target_dir) + _, ext = os.path.splitext(fp) + if ext in (".png", ".jpg", ".ico"): + os.makedirs(dirname(fp_new), exist_ok=True) + if not isfile(fp_new): + shutil.copy(fp, fp_new) + continue + elif ext in (".pyc", ): + continue + # Try to parse everything else + with open(fp, encoding="utf-8") as fo: + try: + lines = fo.readlines() + except UnicodeDecodeError: + # a binary file, skip + logging.warning(f"Skipped replacing imports for {fp}") + continue + lines = _replace_imports(lines, list(zip(source_imports, target_imports))) + os.makedirs(os.path.dirname(fp_new), exist_ok=True) + with open(fp_new, "w", encoding="utf-8") as fo: + fo.writelines(lines) def create_mirror_package(source_dir: str, package_mapping: Dict[str, str]) -> None: @@ -129,7 +144,7 @@ def _prune_packages(req_file: str, packages: Sequence[str]) -> None: req = list(pkg_resources.parse_requirements(ln_))[0] if req.name not in packages: final.append(line) - pprint(final) + logging.info(final) path.write_text("\n".join(final)) @staticmethod @@ -155,5 +170,5 @@ def copy_replace_imports( if __name__ == "__main__": import jsonargparse - + logging.basicConfig(level=logging.INFO) jsonargparse.CLI(AssistantCLI, as_positional=False) From 2dca63bd3234d347669b98e950599a6d1d0d1def Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 12 Nov 2022 12:14:50 +0000 Subject: [PATCH 2/4] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .actions/assistant.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.actions/assistant.py b/.actions/assistant.py index d789d7f4a1567..23c8a5ada7821 100644 --- a/.actions/assistant.py +++ b/.actions/assistant.py @@ -9,7 +9,7 @@ import pkg_resources -_PATH_ROOT = dirname((dirname(__file__))) +_PATH_ROOT = dirname(dirname(__file__)) REQUIREMENT_FILES = { "pytorch": ( "requirements/pytorch/base.txt", @@ -89,7 +89,7 @@ def copy_replace_imports( if not isfile(fp_new): shutil.copy(fp, fp_new) continue - elif ext in (".pyc", ): + elif ext in (".pyc",): continue # Try to parse everything else with open(fp, encoding="utf-8") as fo: @@ -170,5 +170,6 @@ def copy_replace_imports( if __name__ == "__main__": import jsonargparse + logging.basicConfig(level=logging.INFO) jsonargparse.CLI(AssistantCLI, as_positional=False) From a5969a5d79c0daf5457bda56b41925dcc0f6aa0f Mon Sep 17 00:00:00 2001 From: Jirka Date: Sat, 12 Nov 2022 13:15:31 +0100 Subject: [PATCH 3/4] clean --- .actions/assistant.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.actions/assistant.py b/.actions/assistant.py index d789d7f4a1567..ba67f76e35cd4 100644 --- a/.actions/assistant.py +++ b/.actions/assistant.py @@ -9,7 +9,7 @@ import pkg_resources -_PATH_ROOT = dirname((dirname(__file__))) +_PATH_ROOT = dirname(dirname(__file__)) REQUIREMENT_FILES = { "pytorch": ( "requirements/pytorch/base.txt", @@ -70,7 +70,8 @@ def copy_replace_imports( ) -> None: """Copy package content with import adjustments. - >>> _ = copy_replace_imports(os.path.join(_PATH_ROOT, "src"), ["lightning_app"], ["lightning.app"], os.path.join(_PATH_ROOT, "src", "lightning")) + >>> _ = copy_replace_imports(os.path.join( + ... _PATH_ROOT, "src"), ["lightning_app"], ["lightning.app"], os.path.join(_PATH_ROOT, "src", "lightning")) """ logging.info(f"Replacing imports: {locals()}") assert len(source_imports) == len(target_imports), ( @@ -89,7 +90,7 @@ def copy_replace_imports( if not isfile(fp_new): shutil.copy(fp, fp_new) continue - elif ext in (".pyc", ): + elif ext in (".pyc",): continue # Try to parse everything else with open(fp, encoding="utf-8") as fo: @@ -170,5 +171,6 @@ def copy_replace_imports( if __name__ == "__main__": import jsonargparse + logging.basicConfig(level=logging.INFO) jsonargparse.CLI(AssistantCLI, as_positional=False) From c98ba0195827b3fd2812f1f16ecb05badc59f856 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Mochol=C3=AD?= Date: Sat, 12 Nov 2022 19:10:26 +0100 Subject: [PATCH 4/4] Self suggested changes --- .actions/assistant.py | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/.actions/assistant.py b/.actions/assistant.py index ba67f76e35cd4..ce253a96c4c21 100644 --- a/.actions/assistant.py +++ b/.actions/assistant.py @@ -1,4 +1,3 @@ -import logging import os import re import shutil @@ -9,7 +8,6 @@ import pkg_resources -_PATH_ROOT = dirname(dirname(__file__)) REQUIREMENT_FILES = { "pytorch": ( "requirements/pytorch/base.txt", @@ -68,12 +66,8 @@ def _replace_imports(lines: List[str], mapping: List[Tuple[str, str]]) -> List[s def copy_replace_imports( source_dir: str, source_imports: List[str], target_imports: List[str], target_dir: Optional[str] = None ) -> None: - """Copy package content with import adjustments. - - >>> _ = copy_replace_imports(os.path.join( - ... _PATH_ROOT, "src"), ["lightning_app"], ["lightning.app"], os.path.join(_PATH_ROOT, "src", "lightning")) - """ - logging.info(f"Replacing imports: {locals()}") + """Copy package content with import adjustments.""" + print(f"Replacing imports: {locals()}") assert len(source_imports) == len(target_imports), ( "source and target imports must have the same length, " f"source: {len(source_imports)}, target: {len(target_imports)}" @@ -98,7 +92,7 @@ def copy_replace_imports( lines = fo.readlines() except UnicodeDecodeError: # a binary file, skip - logging.warning(f"Skipped replacing imports for {fp}") + print(f"Skipped replacing imports for {fp}") continue lines = _replace_imports(lines, list(zip(source_imports, target_imports))) os.makedirs(os.path.dirname(fp_new), exist_ok=True) @@ -145,7 +139,7 @@ def _prune_packages(req_file: str, packages: Sequence[str]) -> None: req = list(pkg_resources.parse_requirements(ln_))[0] if req.name not in packages: final.append(line) - logging.info(final) + print(final) path.write_text("\n".join(final)) @staticmethod @@ -163,7 +157,7 @@ def replace_oldest_ver(requirement_fnames: Sequence[str] = REQUIREMENT_FILES_ALL def copy_replace_imports( source_dir: str, source_import: str, target_import: str, target_dir: Optional[str] = None ) -> None: - """Recursively replace imports in given folder.""" + """Copy package content with import adjustments.""" source_imports = source_import.strip().split(",") target_imports = target_import.strip().split(",") copy_replace_imports(source_dir, source_imports, target_imports, target_dir=target_dir) @@ -172,5 +166,4 @@ def copy_replace_imports( if __name__ == "__main__": import jsonargparse - logging.basicConfig(level=logging.INFO) jsonargparse.CLI(AssistantCLI, as_positional=False)