From f27888b53812970f7fd32c1dc4efa92accfda975 Mon Sep 17 00:00:00 2001 From: dmitryduev Date: Mon, 18 Jul 2022 23:32:43 -0700 Subject: [PATCH 1/7] wip --- mypy.ini | 6 +++++- tox.ini | 11 ++++++++--- wandb/docker/__init__.py | 2 +- wandb/sdk/internal/internal_api.py | 4 ++-- wandb/sdk/internal/update.py | 4 ++-- wandb/sdk/lib/apikey.py | 2 +- wandb/sdk/verify/verify.py | 2 +- wandb/sdk/wandb_artifacts.py | 3 ++- wandb/sdk/wandb_settings.py | 2 +- wandb/util.py | 4 ++-- 10 files changed, 25 insertions(+), 15 deletions(-) diff --git a/mypy.ini b/mypy.ini index 3433a37e0f1..eb4e5cd9415 100644 --- a/mypy.ini +++ b/mypy.ini @@ -4,6 +4,9 @@ [mypy] warn_redundant_casts = True +exclude = (?x)( + wandb/vendor/. + ) [mypy-wandb.*] ignore_errors = True @@ -400,7 +403,8 @@ ignore_errors = True # ----------------------------- [mypy-wandb.proto.*] -ignore_errors = False +;ignore_errors = False +ignore_missing_imports = True # ----------------------------- # wandb/docker/ diff --git a/tox.ini b/tox.ini index 94d87383131..3fe99eb0890 100644 --- a/tox.ini +++ b/tox.ini @@ -199,13 +199,17 @@ commands= basepython=python3 skip_install = true deps= - mypy==0.812 +; types-protobuf==3.19.22 +; types-PyYAML==6.0.9 +; types-six==1.16.17 +; types-requests==2.28.0 + mypy==0.961 lxml grpcio==1.40.0 setenv = MYPYPATH = {toxinidir} commands= - mypy --show-error-codes --config-file {toxinidir}/mypy.ini -p wandb --html-report mypy-results/ --cobertura-xml-report mypy-results/ --exclude wandb/vendor/ + mypy --install-types --show-error-codes --config-file {toxinidir}/mypy.ini -p wandb --html-report mypy-results/ --cobertura-xml-report mypy-results/ --lineprecision-report mypy-results/ [testenv:mypy-report] basepython=python3 @@ -213,7 +217,8 @@ skip_install = true deps= pycobertura commands= - pycobertura show mypy-results/cobertura.xml + pycobertura show --format markdown mypy-results/cobertura.xml +; pycobertura show --format markdown mypy-results/index.html [black] deps= diff --git a/wandb/docker/__init__.py b/wandb/docker/__init__.py index a040e180699..5344abdc6a4 100644 --- a/wandb/docker/__init__.py +++ b/wandb/docker/__init__.py @@ -141,7 +141,7 @@ def auth_token(registry: str, repo: str) -> Dict[str, str]: auth_info = auth_config.resolve_authconfig(registry) if auth_info: normalized = {k.lower(): v for k, v in auth_info.items()} - normalized_auth_info: Optional[Tuple] = ( + normalized_auth_info: Optional[Tuple[str, str]] = ( normalized.get("username"), normalized.get("password"), ) diff --git a/wandb/sdk/internal/internal_api.py b/wandb/sdk/internal/internal_api.py index 29afaea2dad..e597d65716b 100644 --- a/wandb/sdk/internal/internal_api.py +++ b/wandb/sdk/internal/internal_api.py @@ -34,7 +34,7 @@ import socket import sys -import click +import click # type: ignore import yaml import wandb @@ -2379,7 +2379,7 @@ def push( else: length = os.fstat(open_file.fileno()).st_size with click.progressbar( - file=progress, # type: ignore + file=progress, length=length, label=f"Uploading file: {file_name}", fill_char=click.style("&", fg="green"), diff --git a/wandb/sdk/internal/update.py b/wandb/sdk/internal/update.py index 2881599ce69..4f4d5ec9f48 100644 --- a/wandb/sdk/internal/update.py +++ b/wandb/sdk/internal/update.py @@ -49,13 +49,13 @@ def _find_available( if parse_version(latest_version) <= parsed_current_version: # pre-releases are not included in latest_version # so if we are currently running a pre-release we check more - if not parsed_current_version.is_prerelease: # type: ignore + if not parsed_current_version.is_prerelease: return None # Candidates are pre-releases with the same base_version release_list = map(parse_version, release_list) release_list = filter(lambda v: v.is_prerelease, release_list) release_list = filter( - lambda v: v.base_version == parsed_current_version.base_version, # type: ignore + lambda v: v.base_version == parsed_current_version.base_version, release_list, ) release_list = sorted(release_list) diff --git a/wandb/sdk/lib/apikey.py b/wandb/sdk/lib/apikey.py index 3923154f4e9..d1e2f758ac2 100644 --- a/wandb/sdk/lib/apikey.py +++ b/wandb/sdk/lib/apikey.py @@ -9,7 +9,7 @@ import textwrap from urllib.parse import urlparse -import click +import click # type: ignore import requests import wandb from wandb.apis import InternalApi diff --git a/wandb/sdk/verify/verify.py b/wandb/sdk/verify/verify.py index 525aa21400a..ccc3e5d53d9 100644 --- a/wandb/sdk/verify/verify.py +++ b/wandb/sdk/verify/verify.py @@ -15,7 +15,7 @@ Union, ) -import click +import click # type: ignore from pkg_resources import parse_version import requests import wandb diff --git a/wandb/sdk/wandb_artifacts.py b/wandb/sdk/wandb_artifacts.py index 815a09d4cad..a1bfb5da3ca 100644 --- a/wandb/sdk/wandb_artifacts.py +++ b/wandb/sdk/wandb_artifacts.py @@ -23,6 +23,7 @@ from urllib.parse import parse_qsl, quote, urlparse import requests +import urllib3 import wandb from wandb import env from wandb import util @@ -57,7 +58,7 @@ # This makes the first sleep 1s, and then doubles it up to total times, # which makes for ~18 hours. -_REQUEST_RETRY_STRATEGY = requests.packages.urllib3.util.retry.Retry( +_REQUEST_RETRY_STRATEGY = urllib3.util.retry.Retry( backoff_factor=1, total=16, status_forcelist=(308, 408, 409, 429, 500, 502, 503, 504), diff --git a/wandb/sdk/wandb_settings.py b/wandb/sdk/wandb_settings.py index d44f634aa23..1dd23c94742 100644 --- a/wandb/sdk/wandb_settings.py +++ b/wandb/sdk/wandb_settings.py @@ -122,7 +122,7 @@ def _get_program() -> Optional[Any]: if program is not None: return program try: - import __main__ # type: ignore + import __main__ if __main__.__spec__ is None: return __main__.__file__ diff --git a/wandb/util.py b/wandb/util.py index fcb04da5018..2beff0c5311 100644 --- a/wandb/util.py +++ b/wandb/util.py @@ -1420,7 +1420,7 @@ def _has_internet() -> bool: def rand_alphanumeric(length: int = 8, rand: Optional[ModuleType] = None) -> str: rand = rand or random - return "".join(rand.choice("0123456789ABCDEF") for _ in range(length)) # type: ignore + return "".join(rand.choice("0123456789ABCDEF") for _ in range(length)) @contextlib.contextmanager @@ -1464,7 +1464,7 @@ def _is_databricks() -> bool: if "dbutils" in sys.modules: dbutils = sys.modules["dbutils"] if hasattr(dbutils, "shell"): - shell = dbutils.shell # type: ignore + shell = dbutils.shell if hasattr(shell, "sc"): sc = shell.sc if hasattr(sc, "appName"): From 53c3bac42b9eaafa43a98339ea4cc6415bf4140b Mon Sep 17 00:00:00 2001 From: Dmitry Duev Date: Fri, 12 Aug 2022 01:10:10 -0700 Subject: [PATCH 2/7] get mypy to pass; move parse_version --- wandb/apis/public.py | 4 +++- wandb/docker/__init__.py | 2 +- wandb/docker/auth.py | 2 +- wandb/sdk/data_types/base_types/wb_value.py | 6 ++++-- wandb/sdk/data_types/image.py | 13 ++++++++++--- wandb/sdk/internal/handler.py | 4 ++-- wandb/sdk/internal/internal_api.py | 6 +++--- wandb/sdk/internal/profiler.py | 5 +++-- wandb/sdk/internal/sender.py | 5 ++++- wandb/sdk/internal/stats.py | 4 ++-- wandb/sdk/internal/update.py | 3 ++- wandb/sdk/lib/exit_hooks.py | 2 +- wandb/sdk/lib/printer.py | 17 +++++++++++------ wandb/sdk/service/server_sock.py | 2 +- wandb/util.py | 2 +- 15 files changed, 49 insertions(+), 28 deletions(-) diff --git a/wandb/apis/public.py b/wandb/apis/public.py index 4ffe7a1497c..2ce570c936e 100644 --- a/wandb/apis/public.py +++ b/wandb/apis/public.py @@ -25,7 +25,6 @@ from typing import List, Optional import urllib -from pkg_resources import parse_version import requests import wandb from wandb import __version__, env, util @@ -2944,6 +2943,9 @@ def mongo_to_filter(self, filter): class PythonMongoishQueryGenerator: + + from pkg_resources import parse_version + def __init__(self, run_set): self.run_set = run_set self.panel_metrics_helper = PanelMetricsHelper() diff --git a/wandb/docker/__init__.py b/wandb/docker/__init__.py index 5344abdc6a4..aa47b649666 100644 --- a/wandb/docker/__init__.py +++ b/wandb/docker/__init__.py @@ -141,7 +141,7 @@ def auth_token(registry: str, repo: str) -> Dict[str, str]: auth_info = auth_config.resolve_authconfig(registry) if auth_info: normalized = {k.lower(): v for k, v in auth_info.items()} - normalized_auth_info: Optional[Tuple[str, str]] = ( + normalized_auth_info: Optional[Tuple[str, str]] = ( # type: ignore normalized.get("username"), normalized.get("password"), ) diff --git a/wandb/docker/auth.py b/wandb/docker/auth.py index a4271fcf4d8..c25e08ddca8 100644 --- a/wandb/docker/auth.py +++ b/wandb/docker/auth.py @@ -162,7 +162,7 @@ def parse_auth( conf = {} for registry, entry in entries.items(): if not isinstance(entry, dict): - log.debug(f"Config entry for key {registry} is not auth config") + log.debug(f"Config entry for key {registry} is not auth config") # type: ignore # We sometimes fall back to parsing the whole config as if it # was the auth config by itself, for legacy purposes. In that # case, we fail silently and return an empty conf if any of the diff --git a/wandb/sdk/data_types/base_types/wb_value.py b/wandb/sdk/data_types/base_types/wb_value.py index 6f38fcd6892..2b4354bfb3d 100644 --- a/wandb/sdk/data_types/base_types/wb_value.py +++ b/wandb/sdk/data_types/base_types/wb_value.py @@ -1,6 +1,5 @@ from typing import Any, ClassVar, Dict, List, Optional, Type, TYPE_CHECKING, Union -from pkg_resources import parse_version from wandb import util if TYPE_CHECKING: # pragma: no cover @@ -13,6 +12,8 @@ def _server_accepts_client_ids() -> bool: + from pkg_resources import parse_version + # First, if we are offline, assume the backend server cannot # accept client IDs. Unfortunately, this is the best we can do # until we are sure that all local versions are > "0.11.0" max_cli_version. @@ -30,7 +31,8 @@ def _server_accepts_client_ids() -> bool: max_cli_version = util._get_max_cli_version() if max_cli_version is None: return False - return parse_version("0.11.0") <= parse_version(max_cli_version) + accepts_client_ids: bool = parse_version("0.11.0") <= parse_version(max_cli_version) + return accepts_client_ids class _WBValueArtifactSource: diff --git a/wandb/sdk/data_types/image.py b/wandb/sdk/data_types/image.py index 75d3d5d3ca1..d88b6bca9eb 100644 --- a/wandb/sdk/data_types/image.py +++ b/wandb/sdk/data_types/image.py @@ -4,7 +4,6 @@ import os from typing import Any, cast, Dict, List, Optional, Sequence, Type, TYPE_CHECKING, Union -from pkg_resources import parse_version import wandb from wandb import util @@ -32,20 +31,28 @@ def _server_accepts_image_filenames() -> bool: + from pkg_resources import parse_version + # Newer versions of wandb accept large image filenames arrays # but older versions would have issues with this. max_cli_version = util._get_max_cli_version() if max_cli_version is None: return False - return parse_version("0.12.10") <= parse_version(max_cli_version) + accepts_image_filenames: bool = parse_version("0.12.10") <= parse_version( + max_cli_version + ) + return accepts_image_filenames def _server_accepts_artifact_path() -> bool: + from pkg_resources import parse_version + target_version = "0.12.14" max_cli_version = util._get_max_cli_version() if not util._is_offline() else None - return max_cli_version is not None and parse_version( + accepts_artifact_path: bool = max_cli_version is not None and parse_version( target_version ) <= parse_version(max_cli_version) + return accepts_artifact_path class Image(BatchableMedia): diff --git a/wandb/sdk/internal/handler.py b/wandb/sdk/internal/handler.py index a77a2e7c4f4..0bf52cd5aee 100644 --- a/wandb/sdk/internal/handler.py +++ b/wandb/sdk/internal/handler.py @@ -128,7 +128,7 @@ def handle(self, record: Record) -> None: record_type = record.WhichOneof("record_type") assert record_type handler_str = "handle_" + record_type - handler: Callable[[Record], None] = getattr(self, handler_str, None) + handler: Callable[[Record], None] = getattr(self, handler_str, None) # type: ignore assert handler, f"unknown handle: {handler_str}" handler(record) @@ -136,7 +136,7 @@ def handle_request(self, record: Record) -> None: request_type = record.request.WhichOneof("request_type") assert request_type handler_str = "handle_request_" + request_type - handler: Callable[[Record], None] = getattr(self, handler_str, None) + handler: Callable[[Record], None] = getattr(self, handler_str, None) # type: ignore if request_type != "network_status": logger.debug(f"handle_request: {request_type}") assert handler, f"unknown handle: {handler_str}" diff --git a/wandb/sdk/internal/internal_api.py b/wandb/sdk/internal/internal_api.py index 52568e6a6e4..da6b75beec5 100644 --- a/wandb/sdk/internal/internal_api.py +++ b/wandb/sdk/internal/internal_api.py @@ -24,10 +24,8 @@ import base64 from copy import deepcopy import datetime -from io import BytesIO import json import os -from pkg_resources import parse_version import re import requests import logging @@ -1767,7 +1765,7 @@ def download_file(self, url: str) -> Tuple[int, requests.Response]: Returns: A tuple of the content length and the streaming response """ - response = requests.get(url, auth=("user", self.api_key), stream=True) + response = requests.get(url, auth=("user", self.api_key), stream=True) # type: ignore response.raise_for_status() return int(response.headers.get("content-length", 0)), response @@ -2535,6 +2533,8 @@ def create_artifact( enable_digest_deduplication: Optional[bool] = False, history_step: Optional[int] = None, ) -> Tuple[Dict, Dict]: + from pkg_resources import parse_version + _, server_info = self.viewer_server_info() max_cli_version = server_info.get("cliVersionInfo", {}).get( "max_cli_version", None diff --git a/wandb/sdk/internal/profiler.py b/wandb/sdk/internal/profiler.py index c035faa85e4..bc9dbce0742 100644 --- a/wandb/sdk/internal/profiler.py +++ b/wandb/sdk/internal/profiler.py @@ -1,7 +1,6 @@ """Integration with pytorch profiler.""" import os -from pkg_resources import parse_version import wandb from wandb.errors import Error, UsageError from wandb.sdk.lib import telemetry @@ -15,7 +14,7 @@ def torch_trace_handler(): Provide as an argument to `torch.profiler.profile`: ```python - torch.profiler.profile(..., on_trace_ready = wandb.profiler.torch_trace_handler()) + torch.profiler.profile(..., on_trace_ready=wandb.profiler.torch_trace_handler()) ``` Calling this function ensures that profiler charts & tables can be viewed in your run dashboard @@ -53,6 +52,8 @@ def torch_trace_handler(): prof.step() ``` """ + from pkg_resources import parse_version + torch = wandb.util.get_module(PYTORCH_MODULE, required=True) torch_profiler = wandb.util.get_module(PYTORCH_PROFILER_MODULE, required=True) diff --git a/wandb/sdk/internal/sender.py b/wandb/sdk/internal/sender.py index 2b0551b4f3a..c598b8ce3c9 100644 --- a/wandb/sdk/internal/sender.py +++ b/wandb/sdk/internal/sender.py @@ -24,7 +24,6 @@ TYPE_CHECKING, ) -from pkg_resources import parse_version import requests import wandb from wandb import util @@ -1251,6 +1250,8 @@ def send_artifact(self, record: "Record") -> None: def _send_artifact( self, artifact: "ArtifactRecord", history_step: Optional[int] = None ) -> Optional[Dict]: + from pkg_resources import parse_version + assert self._pusher saver = artifacts.ArtifactSaver( api=self._api, @@ -1288,6 +1289,8 @@ def _send_artifact( ) def send_alert(self, record: "Record") -> None: + from pkg_resources import parse_version + alert = record.alert max_cli_version = self._max_cli_version() if max_cli_version is None or parse_version(max_cli_version) < parse_version( diff --git a/wandb/sdk/internal/stats.py b/wandb/sdk/internal/stats.py index 69a84ef55eb..6edb7b0de63 100644 --- a/wandb/sdk/internal/stats.py +++ b/wandb/sdk/internal/stats.py @@ -78,7 +78,7 @@ def __init__(self, settings: SettingsStatic, interface: InterfaceQueue) -> None: self._interface = interface self.sampler = {} self.samples = 0 - self._shutdown = False + self._shutdown: bool = False self._telem = telemetry.TelemetryRecord() if psutil: net = psutil.net_io_counters() @@ -150,7 +150,7 @@ def _thread_body(self) -> None: time.sleep(0.1) seconds += 0.1 if self._shutdown: - self.flush() + self.flush() # type: ignore return def shutdown(self) -> None: diff --git a/wandb/sdk/internal/update.py b/wandb/sdk/internal/update.py index 4f4d5ec9f48..c7016d4dbc1 100644 --- a/wandb/sdk/internal/update.py +++ b/wandb/sdk/internal/update.py @@ -1,6 +1,5 @@ from typing import Dict, Optional, Tuple -from pkg_resources import parse_version import requests import wandb @@ -8,6 +7,8 @@ def _find_available( current_version: str, ) -> Optional[Tuple[str, bool, bool, bool, Optional[str]]]: + from pkg_resources import parse_version + pypi_url = f"https://pypi.org/pypi/{wandb._wandb_module}/json" yanked_dict = {} diff --git a/wandb/sdk/lib/exit_hooks.py b/wandb/sdk/lib/exit_hooks.py index f278940b2d9..bf6d4349eac 100644 --- a/wandb/sdk/lib/exit_hooks.py +++ b/wandb/sdk/lib/exit_hooks.py @@ -28,7 +28,7 @@ def hook(self) -> None: != sys.__excepthook__ # respect hooks by other libraries like pdb else None ) - sys.excepthook = self.exc_handler + sys.excepthook = self.exc_handler # type: ignore def exit(self, code: object = 0) -> "NoReturn": orig_code = code diff --git a/wandb/sdk/lib/printer.py b/wandb/sdk/lib/printer.py index ce28b27b9da..da316f2e846 100644 --- a/wandb/sdk/lib/printer.py +++ b/wandb/sdk/lib/printer.py @@ -6,7 +6,7 @@ import sys from typing import Callable, List, Optional, Tuple, Union -import click +import click # type: ignore import wandb from . import ipython, sparkline @@ -174,13 +174,16 @@ def progress_close(self) -> None: wandb.termlog(" " * 79) def code(self, text: str) -> str: - return click.style(text, bold=True) + ret: str = click.style(text, bold=True) + return ret def name(self, text: str) -> str: - return click.style(text, fg="yellow") + ret: str = click.style(text, fg="yellow") + return ret def link(self, link: str, text: Optional[str] = None) -> str: - return click.style(link, fg="blue", underline=True) + ret: str = click.style(link, fg="blue", underline=True) + return ret def emoji(self, name: str) -> str: emojis = dict() @@ -191,10 +194,12 @@ def emoji(self, name: str) -> str: def status(self, text: str, failure: Optional[bool] = None) -> str: color = "red" if failure else "green" - return click.style(text, fg=color) + ret: str = click.style(text, fg=color) + return ret def files(self, text: str) -> str: - return click.style(text, fg="magenta", bold=True) + ret: str = click.style(text, fg="magenta", bold=True) + return ret def grid(self, rows: List[List[str]], title: Optional[str] = None) -> str: max_len = max(len(row[0]) for row in rows) diff --git a/wandb/sdk/service/server_sock.py b/wandb/sdk/service/server_sock.py index 84d65da83c0..765b99da41b 100644 --- a/wandb/sdk/service/server_sock.py +++ b/wandb/sdk/service/server_sock.py @@ -106,7 +106,7 @@ def run(self) -> None: assert sreq, "read_server_request should never timeout" sreq_type = sreq.WhichOneof("server_request_type") shandler_str = "server_" + sreq_type - shandler: "Callable[[spb.ServerRequest], None]" = getattr( + shandler: "Callable[[spb.ServerRequest], None]" = getattr( # type: ignore self, shandler_str, None ) assert shandler, f"unknown handle: {shandler_str}" diff --git a/wandb/util.py b/wandb/util.py index 3439ff819c8..2783f367df8 100644 --- a/wandb/util.py +++ b/wandb/util.py @@ -1261,7 +1261,7 @@ def guess_data_type(shape: Sequence[int], risky: bool = False) -> Optional[str]: def download_file_from_url( dest_path: str, source_url: str, api_key: Optional[str] = None ) -> None: - response = requests.get(source_url, auth=("api", api_key), stream=True, timeout=5) + response = requests.get(source_url, auth=("api", api_key), stream=True, timeout=5) # type: ignore response.raise_for_status() if os.sep in dest_path: From 429f869adaedbd3f0f771a5c13732201f63e3bef Mon Sep 17 00:00:00 2001 From: Dmitry Duev Date: Fri, 12 Aug 2022 01:11:57 -0700 Subject: [PATCH 3/7] get mypy to pass; move parse_version --- wandb/apis/public.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/wandb/apis/public.py b/wandb/apis/public.py index 2ce570c936e..a484c2576f1 100644 --- a/wandb/apis/public.py +++ b/wandb/apis/public.py @@ -240,6 +240,8 @@ def server_info(self): return self._server_info def version_supported(self, min_version): + from pkg_resources import parse_version + return parse_version(min_version) <= parse_version( self.server_info["cliVersionInfo"]["max_cli_version"] ) From 590cd4f4a8d8b12a0ab6e1556e79e3ec873d920a Mon Sep 17 00:00:00 2001 From: Dmitry Duev Date: Mon, 15 Aug 2022 11:37:07 -0700 Subject: [PATCH 4/7] mypy 0.971 --- tox.ini | 9 +++------ wandb/sdk/internal/internal_api.py | 4 ++-- wandb/sdk/lib/apikey.py | 2 +- wandb/sdk/lib/printer.py | 2 +- wandb/sdk/verify/verify.py | 2 +- wandb/sdk/wandb_run.py | 8 ++++---- 6 files changed, 12 insertions(+), 15 deletions(-) diff --git a/tox.ini b/tox.ini index 83b682ab788..0f2423b7af2 100644 --- a/tox.ini +++ b/tox.ini @@ -202,17 +202,14 @@ commands= basepython=python3 skip_install = true deps= -; types-protobuf==3.19.22 -; types-PyYAML==6.0.9 -; types-six==1.16.17 -; types-requests==2.28.0 - mypy==0.961 + types-click==7.1.8 + mypy==0.971 lxml grpcio==1.40.0 setenv = MYPYPATH = {toxinidir} commands= - mypy --install-types --show-error-codes --config-file {toxinidir}/mypy.ini -p wandb --html-report mypy-results/ --cobertura-xml-report mypy-results/ --lineprecision-report mypy-results/ + mypy --install-types --non-interactive --show-error-codes --config-file {toxinidir}/mypy.ini -p wandb --html-report mypy-results/ --cobertura-xml-report mypy-results/ --lineprecision-report mypy-results/ [testenv:mypy-report] basepython=python3 diff --git a/wandb/sdk/internal/internal_api.py b/wandb/sdk/internal/internal_api.py index da6b75beec5..0e5da1d72d0 100644 --- a/wandb/sdk/internal/internal_api.py +++ b/wandb/sdk/internal/internal_api.py @@ -32,7 +32,7 @@ import socket import sys -import click # type: ignore +import click import yaml import wandb @@ -2331,7 +2331,7 @@ def push( else: length = os.fstat(open_file.fileno()).st_size with click.progressbar( - file=progress, + file=progress, # type: ignore length=length, label=f"Uploading file: {file_name}", fill_char=click.style("&", fg="green"), diff --git a/wandb/sdk/lib/apikey.py b/wandb/sdk/lib/apikey.py index d1e2f758ac2..3923154f4e9 100644 --- a/wandb/sdk/lib/apikey.py +++ b/wandb/sdk/lib/apikey.py @@ -9,7 +9,7 @@ import textwrap from urllib.parse import urlparse -import click # type: ignore +import click import requests import wandb from wandb.apis import InternalApi diff --git a/wandb/sdk/lib/printer.py b/wandb/sdk/lib/printer.py index da316f2e846..30ee2d0b685 100644 --- a/wandb/sdk/lib/printer.py +++ b/wandb/sdk/lib/printer.py @@ -6,7 +6,7 @@ import sys from typing import Callable, List, Optional, Tuple, Union -import click # type: ignore +import click import wandb from . import ipython, sparkline diff --git a/wandb/sdk/verify/verify.py b/wandb/sdk/verify/verify.py index ccc3e5d53d9..525aa21400a 100644 --- a/wandb/sdk/verify/verify.py +++ b/wandb/sdk/verify/verify.py @@ -15,7 +15,7 @@ Union, ) -import click # type: ignore +import click from pkg_resources import parse_version import requests import wandb diff --git a/wandb/sdk/wandb_run.py b/wandb/sdk/wandb_run.py index 1686c11f0fc..5452bae08be 100644 --- a/wandb/sdk/wandb_run.py +++ b/wandb/sdk/wandb_run.py @@ -3189,7 +3189,7 @@ def _footer_multiple_runs_file_pusher_status_info( return megabyte = wandb.util.POW_2_BYTES[2][1] - total_files = sum( + total_files: int = sum( sum( [ response.file_counts.wandb_count, @@ -3199,17 +3199,17 @@ def _footer_multiple_runs_file_pusher_status_info( ] ) for response in poll_exit_responses - if response and response.file_counts + if response is not None and response.file_counts is not None ) uploaded = sum( response.pusher_stats.uploaded_bytes for response in poll_exit_responses - if response and response.pusher_stats + if response is not None and response.pusher_stats is not None ) total = sum( response.pusher_stats.total_bytes for response in poll_exit_responses - if response and response.pusher_stats + if response is not None and response.pusher_stats is not None ) line = f"Processing {len(poll_exit_responses)} runs with {total_files} files ({uploaded/megabyte :.2f} MB/{total/megabyte :.2f} MB)\r" From 02a19a542937946954e3698e16b13e72741c1426 Mon Sep 17 00:00:00 2001 From: Dmitry Duev Date: Mon, 15 Aug 2022 11:58:29 -0700 Subject: [PATCH 5/7] fix mypy-report & lint wandb_run.py --- tox.ini | 3 +-- wandb/sdk/wandb_run.py | 47 +++++++++++++++++++++++++++++++----------- 2 files changed, 36 insertions(+), 14 deletions(-) diff --git a/tox.ini b/tox.ini index 537a295c7dc..1817cdfaf9b 100644 --- a/tox.ini +++ b/tox.ini @@ -217,8 +217,7 @@ skip_install = true deps= pycobertura commands= - pycobertura show --format markdown mypy-results/cobertura.xml -; pycobertura show --format markdown mypy-results/index.html + pycobertura show --format text mypy-results/cobertura.xml [black] deps= diff --git a/wandb/sdk/wandb_run.py b/wandb/sdk/wandb_run.py index 5452bae08be..38c28793c64 100644 --- a/wandb/sdk/wandb_run.py +++ b/wandb/sdk/wandb_run.py @@ -241,7 +241,10 @@ def wrapper(self: Type["Run"], *args: Any, **kwargs: Any) -> Any: ): if cls._is_attaching: - message = f"Trying to attach `{func.__name__}` while in the middle of attaching `{cls._is_attaching}`" + message = ( + f"Trying to attach `{func.__name__}` " + f"while in the middle of attaching `{cls._is_attaching}`" + ) raise RuntimeError(message) cls._is_attaching = func.__name__ try: @@ -261,10 +264,12 @@ def _noop(cls, func: Callable) -> Callable: @functools.wraps(func) def wrapper(self: Type["Run"], *args: Any, **kwargs: Any) -> Any: # `_attach_id` is only assigned in service hence for all service cases - # it will be a passthrough. We don't pickle non-service so again a way to see that we are in non-service case + # it will be a passthrough. We don't pickle non-service so again a way + # to see that we are in non-service case if getattr(self, "_attach_id", None) is None: # `_init_pid` is only assigned in __init__ (this will be constant check for mp): - # - for non-fork case the object is shared through pickling and we don't pickle non-service so will be None + # - for non-fork case the object is shared through pickling, + # and we don't pickle non-service so will be None # - for fork case the new process share mem space hence the value would be of parent process. _init_pid = getattr(self, "_init_pid", None) if _init_pid != os.getpid(): @@ -274,7 +279,8 @@ def wrapper(self: Type["Run"], *args: Any, **kwargs: Any) -> Any: _init_pid, wburls.get("multiprocess"), ) - # - if this process was pickled in non-service case, we ignore the attributes (since pickle is not supported) + # - if this process was pickled in non-service case, + # we ignore the attributes (since pickle is not supported) # - for fork case will use the settings of the parent process # - only point of inconsistent behavior from forked and non-forked cases settings = getattr(self, "_settings", None) @@ -1374,7 +1380,8 @@ def _log( if step is not None: if os.getpid() != self._init_pid or self._is_attached: wandb.termwarn( - "Note that setting step in multiprocessing can result in data loss. Please log your step values as a metric such as 'global_step'", + "Note that setting step in multiprocessing can result in data loss. " + "Please log your step values as a metric such as 'global_step'", repeat=False, ) # if step is passed in when tensorboard_sync is used we honor the step passed @@ -1382,7 +1389,8 @@ def _log( # this history later on in publish_history() if len(wandb.patched["tensorboard"]) > 0: wandb.termwarn( - "Step cannot be set when using syncing with tensorboard. Please log your step values as a metric such as 'global_step'", + "Step cannot be set when using syncing with tensorboard. " + "Please log your step values as a metric such as 'global_step'", repeat=False, ) if step > self._step: @@ -1983,9 +1991,11 @@ def _on_init(self) -> None: logger.info(f"got version response {self._check_version}") def _on_start(self) -> None: - # would like to move _set_global to _on_ready to unify _on_start and _on_attach (we want to do the set globals after attach) + # would like to move _set_global to _on_ready to unify _on_start and _on_attach + # (we want to do the set globals after attach) # TODO(console) However _console_start calls Redirect that uses `wandb.run` hence breaks - # TODO(jupyter) However _header calls _header_run_info that uses wandb.jupyter that uses `wandb.run` and hence breaks + # TODO(jupyter) However _header calls _header_run_info that uses wandb.jupyter that uses + # `wandb.run` and hence breaks self._set_globals() self._header( self._check_version, settings=self._settings, printer=self._printer @@ -2363,7 +2373,15 @@ def _define_metric( # TODO(jhr): annotate this @_run_decorator._attach - def watch(self, models, criterion=None, log="gradients", log_freq=100, idx=None, log_graph=False) -> None: # type: ignore + def watch( # type: ignore + self, + models, + criterion=None, + log="gradients", + log_freq=100, + idx=None, + log_graph=False, + ) -> None: wandb.watch(models, criterion, log, log_freq, idx, log_graph) # TODO(jhr): annotate this @@ -2387,7 +2405,8 @@ def _swap_artifact_name(self, artifact_name: str, use_as: Optional[str]) -> str: return f"{entity}/{project}/{new_name}" elif replacement_artifact_info is None and use_as is None: wandb.termwarn( - f"Could not find {artifact_name} in launch artifact mapping. Searching for unique artifacts with sequence name: {artifact_name}" + f"Could not find {artifact_name} in launch artifact mapping. " + f"Searching for unique artifacts with sequence name: {artifact_name}" ) sequence_name = artifact_name.split(":")[0].split("/")[-1] unique_artifact_replacement_info = ( @@ -2432,7 +2451,8 @@ def link_artifact( artifact: the (public or local) artifact which will be linked target_path: `str` - takes the following forms: {portfolio}, {project}/{portfolio}, or {entity}/{project}/{portfolio} - aliases: `List[str]` - optional alias(es) that will only be applied on this linked artifact inside the portfolio. + aliases: `List[str]` - optional alias(es) that will only be applied on this linked artifact + inside the portfolio. The alias "latest" will always be applied to the latest version of an artifact that is linked. Returns: @@ -3212,7 +3232,10 @@ def _footer_multiple_runs_file_pusher_status_info( if response is not None and response.pusher_stats is not None ) - line = f"Processing {len(poll_exit_responses)} runs with {total_files} files ({uploaded/megabyte :.2f} MB/{total/megabyte :.2f} MB)\r" + line = ( + f"Processing {len(poll_exit_responses)} runs with {total_files} files " + f"({uploaded/megabyte :.2f} MB/{total/megabyte :.2f} MB)\r" + ) # line = "{}{:<{max_len}}\r".format(line, " ", max_len=(80 - len(line))) printer.progress_update(line) # type: ignore [call-arg] From faaf1cd0b6572e30fd22541b3772100af425fbb7 Mon Sep 17 00:00:00 2001 From: Dmitry Duev Date: Mon, 15 Aug 2022 13:24:24 -0700 Subject: [PATCH 6/7] do not install broken sweeps on py310 --- requirements_sweeps.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_sweeps.txt b/requirements_sweeps.txt index d1a3cf25f90..deebb6dd4bb 100644 --- a/requirements_sweeps.txt +++ b/requirements_sweeps.txt @@ -1 +1 @@ -sweeps>=0.1.0 +sweeps>=0.1.0; python_version < '3.10' From 2cdbeeb5d1a51e9ef9eb043684639292bb721b8f Mon Sep 17 00:00:00 2001 From: Dmitry Duev Date: Mon, 15 Aug 2022 14:14:18 -0700 Subject: [PATCH 7/7] pin sweeps>=0.2 --- requirements_sweeps.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_sweeps.txt b/requirements_sweeps.txt index deebb6dd4bb..3883d61dc12 100644 --- a/requirements_sweeps.txt +++ b/requirements_sweeps.txt @@ -1 +1 @@ -sweeps>=0.1.0; python_version < '3.10' +sweeps>=0.2.0