diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index afbce896af..55aa61e030 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -30,6 +30,7 @@ jobs: - name: Install requirements run: | pip install --upgrade pip setuptools wheel + pip install collective.checkdocs==0.2 pip install ".[dev]" pre-commit - name: Check README run: python setup.py checkdocs @@ -74,7 +75,7 @@ jobs: --cov-report=xml --cov-report=term ${{ env.extra_test_args }} - name: upload coverage report - uses: codecov/codecov-action@v2.1.0 + uses: codecov/codecov-action@v3 with: file: ./coverage.xml fail_ci_if_error: false diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8bbb4cf4a1..2983bad28a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -6,7 +6,7 @@ repos: - id: black language_version: python3 repo: https://github.com/ambv/black - rev: 22.1.0 + rev: 22.3.0 - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.1.0 hooks: diff --git a/dvc/api.py b/dvc/api.py index b1ac0a0f2e..e31d4506a7 100644 --- a/dvc/api.py +++ b/dvc/api.py @@ -28,6 +28,10 @@ def get_url(path, repo=None, rev=None, remote=None): cloud = info["repo"].cloud dvc_path = _repo.fs.path.relpath(fs_path, info["repo"].root_dir) + + if not os.path.isabs(path): + dvc_path = dvc_path.replace("\\", "/") + md5 = info["repo"].dvcfs.info(dvc_path)["md5"] return cloud.get_url_for(remote, checksum=md5) diff --git a/dvc/cli/__init__.py b/dvc/cli/__init__.py index 78dd13819c..3910ebf731 100644 --- a/dvc/cli/__init__.py +++ b/dvc/cli/__init__.py @@ -1,6 +1,7 @@ """This module provides an entrypoint to the dvc cli and parsing utils.""" import logging +import sys # Workaround for CPython bug. See [1] and [2] for more info. # [1] https://github.com/aws/aws-cli/blob/1.16.277/awscli/clidriver.py#L55 @@ -48,6 +49,16 @@ def main(argv=None): # noqa: C901 from dvc.exceptions import DvcException, NotDvcRepoError from dvc.logger import FOOTER, disable_other_loggers + # NOTE: stderr/stdout may be closed if we are running from dvc.daemon. + # On Linux we directly call cli.main after double forking and closing + # the copied parent's standard file descriptors. If we make any logging + # calls in this state it will cause an exception due to writing to a closed + # file descriptor. + if sys.stderr.closed: # pylint: disable=using-constant-test + logging.disable() + elif sys.stdout.closed: # pylint: disable=using-constant-test + logging.disable(logging.INFO) + args = None disable_other_loggers() @@ -68,7 +79,7 @@ def main(argv=None): # noqa: C901 logger.trace(args) - if not args.quiet: + if not sys.stdout.closed and not args.quiet: from dvc.ui import ui ui.enable() diff --git a/dvc/commands/experiments/show.py b/dvc/commands/experiments/show.py index c00df5a4fe..28580b876c 100644 --- a/dvc/commands/experiments/show.py +++ b/dvc/commands/experiments/show.py @@ -426,11 +426,12 @@ def show_experiments( subset=subset, ) td.drop_duplicates("rows", subset=subset) - td.column("Experiment")[:] = [ - # remove tree characters - str(x).encode("ascii", "ignore").strip().decode() - for x in td.column("Experiment") - ] + if "Experiment" in td: + td.column("Experiment")[:] = [ + # remove tree characters + str(x).encode("ascii", "ignore").strip().decode() + for x in td.column("Experiment") + ] out = kwargs.get("out") or "dvc_plots" output_file = os.path.join(out, "index.html") ui.write( diff --git a/dvc/commands/plots.py b/dvc/commands/plots.py index fb6bbd07ca..3534d92fb4 100644 --- a/dvc/commands/plots.py +++ b/dvc/commands/plots.py @@ -21,8 +21,7 @@ def _show_json(renderers, split=False): result = { renderer.name: to_json(renderer, split) for renderer in renderers } - if result: - ui.write_json(result) + ui.write_json(result) class CmdPlots(CmdBase): diff --git a/dvc/commands/remote.py b/dvc/commands/remote.py index 16a4498b62..cd249cd5e6 100644 --- a/dvc/commands/remote.py +++ b/dvc/commands/remote.py @@ -113,8 +113,8 @@ def run(self): class CmdRemoteList(CmdRemote): def run(self): conf = self.config.read(self.args.level) - for name, conf_val in conf["remote"].items(): - ui.write(name, conf_val["url"], sep="\t") + for name, remote_conf in conf["remote"].items(): + ui.write(name, remote_conf["url"], sep="\t") return 0 diff --git a/dvc/daemon.py b/dvc/daemon.py index 09b094713c..c0b84e5e03 100644 --- a/dvc/daemon.py +++ b/dvc/daemon.py @@ -106,8 +106,9 @@ def daemon(args): cmd = ["daemon", "-q"] + args env = fix_env() - file_path = os.path.abspath(inspect.stack()[0][1]) - env["PYTHONPATH"] = os.path.dirname(os.path.dirname(file_path)) + if not is_binary(): + file_path = os.path.abspath(inspect.stack()[0][1]) + env["PYTHONPATH"] = os.path.dirname(os.path.dirname(file_path)) env[DVC_DAEMON] = "1" _spawn(cmd, env) diff --git a/dvc/data/tree.py b/dvc/data/tree.py index 25e29d8121..5996a74903 100644 --- a/dvc/data/tree.py +++ b/dvc/data/tree.py @@ -19,6 +19,10 @@ logger = logging.getLogger(__name__) +class TreeError(Exception): + pass + + def _try_load( odbs: Iterable["ObjectDB"], hash_info: "HashInfo", @@ -197,6 +201,27 @@ def get(self, odb, prefix: Tuple[str]) -> Optional[HashFile]: tree.digest() return tree + def ls(self, prefix=None): + kwargs = {} + if prefix: + kwargs["prefix"] = prefix + + meta, hash_info = self._trie.get(prefix, (None, None)) + if hash_info and hash_info.isdir and meta and not meta.obj: + raise TreeError + + ret = [] + + def node_factory(_, key, children, *args): + if key == prefix: + list(children) + else: + ret.append(key[-1]) + + self._trie.traverse(node_factory, **kwargs) + + return ret + def du(odb, tree): try: diff --git a/dvc/dependency/repo.py b/dvc/dependency/repo.py index 87fd386630..6737b4f317 100644 --- a/dvc/dependency/repo.py +++ b/dvc/dependency/repo.py @@ -99,7 +99,7 @@ def _get_used_and_obj( ) -> Tuple[Dict[Optional["ObjectDB"], Set["HashInfo"]], "HashFile"]: from dvc.config import NoRemoteError from dvc.data.stage import stage - from dvc.data.tree import Tree + from dvc.data.tree import Tree, TreeError from dvc.exceptions import NoOutputOrStageError, PathMissingError local_odb = self.repo.odb.local @@ -136,7 +136,7 @@ def _get_used_and_obj( repo.repo_fs, local_odb.fs.PARAM_CHECKSUM, ) - except FileNotFoundError as exc: + except (FileNotFoundError, TreeError) as exc: raise PathMissingError( self.def_path, self.def_repo[self.PARAM_URL] ) from exc diff --git a/dvc/fs/__init__.py b/dvc/fs/__init__.py index 4797ec2028..6d289f2cf7 100644 --- a/dvc/fs/__init__.py +++ b/dvc/fs/__init__.py @@ -114,7 +114,12 @@ def get_cloud_fs(repo, **kwargs): remote_conf["gdrive_credentials_tmp_dir"] = repo.tmp_dir url = remote_conf.pop("url") - fs_path = cls._strip_protocol(url) # pylint:disable=protected-access + if issubclass(cls, WebDAVFileSystem): + # For WebDAVFileSystem, provided url is the base path itself, so it + # should be treated as being a root path. + fs_path = cls.root_marker + else: + fs_path = cls._strip_protocol(url) # pylint:disable=protected-access extras = cls._get_kwargs_from_urls(url) # pylint:disable=protected-access conf = {**extras, **remote_conf} # remote config takes priority diff --git a/dvc/fs/dvc.py b/dvc/fs/dvc.py index cbe457139f..be76ed6d5d 100644 --- a/dvc/fs/dvc.py +++ b/dvc/fs/dvc.py @@ -1,9 +1,13 @@ import logging import os +import threading import typing +from fsspec import AbstractFileSystem +from funcy import cached_property, wrap_prop + from ._callback import DEFAULT_CALLBACK -from .base import FileSystem +from .fsspec_wrapper import FSSpecWrapper if typing.TYPE_CHECKING: from dvc.types import AnyPath @@ -11,18 +15,13 @@ logger = logging.getLogger(__name__) -class DvcFileSystem(FileSystem): # pylint:disable=abstract-method +class _DvcFileSystem(AbstractFileSystem): # pylint:disable=abstract-method """DVC repo fs. Args: repo: DVC repo. """ - sep = os.sep - - scheme = "local" - PARAM_CHECKSUM = "md5" - def __init__(self, **kwargs): super().__init__(**kwargs) self.repo = kwargs["repo"] @@ -43,8 +42,8 @@ def _get_key(self, path): return (cls.scheme, *fs.path.parts(fs_path)) fs_key = "repo" - key = self.path.parts(path) - if key == (".",) or key == ("",): + key = path.split(self.sep) + if key == ["."] or key == [""]: key = () return (fs_key, *key) @@ -74,81 +73,28 @@ def _get_fs_path(self, path: "AnyPath", remote=None): def open( # type: ignore self, path: str, mode="r", encoding=None, **kwargs - ): # pylint: disable=arguments-renamed + ): # pylint: disable=arguments-renamed, arguments-differ fs, fspath = self._get_fs_path(path, **kwargs) return fs.open(fspath, mode=mode, encoding=encoding) - def exists(self, path): # pylint: disable=arguments-renamed - try: - self.info(path) - return True - except FileNotFoundError: - return False - - def isdir(self, path): # pylint: disable=arguments-renamed - try: - return self.info(path)["type"] == "directory" - except FileNotFoundError: - return False - - def isfile(self, path): # pylint: disable=arguments-renamed - try: - return self.info(path)["type"] == "file" - except FileNotFoundError: - return False - - def _walk(self, root, topdown=True, **kwargs): - dirs = set() - files = [] - - root_parts = self._get_key(root) - root_len = len(root_parts) - try: - for key, (meta, hash_info) in self.repo.index.tree.iteritems( - prefix=root_parts - ): # noqa: B301 - if hash_info and hash_info.isdir and meta and not meta.obj: - raise FileNotFoundError - - if key == root_parts: - continue - - if hash_info.isdir: - continue - - name = key[root_len] - if len(key) > root_len + 1: - dirs.add(name) - continue - - files.append(name) - except KeyError: - pass - - assert topdown - dirs = list(dirs) - yield root, dirs, files - - for dname in dirs: - yield from self._walk(self.path.join(root, dname)) + def ls(self, path, detail=True, **kwargs): + info = self.info(path) + if info["type"] != "directory": + return [info] if detail else [path] - def walk(self, top, topdown=True, **kwargs): - assert topdown + root_key = self._get_key(path) try: - info = self.info(top) - except FileNotFoundError: - return - - if info["type"] != "directory": - return + entries = [ + self.sep.join((path, name)) if path else name + for name in self.repo.index.tree.ls(prefix=root_key) + ] + except KeyError as exc: + raise FileNotFoundError from exc - yield from self._walk(top, topdown=topdown, **kwargs) + if not detail: + return entries - def find(self, path, prefix=None): - for root, _, files in self.walk(path): - for fname in files: - # NOTE: os.path.join is ~5.5 times slower - yield f"{root}{os.sep}{fname}" + return [self.info(epath) for epath in entries] def isdvc(self, path, recursive=False, strict=True): try: @@ -159,7 +105,7 @@ def isdvc(self, path, recursive=False, strict=True): recurse = recursive or not strict return bool(info.get("outs") if recurse else info.get("isout")) - def info(self, path): + def info(self, path, **kwargs): from dvc.data.meta import Meta key = self._get_key(path) @@ -175,6 +121,7 @@ def info(self, path): "isexec": False, "isdvc": False, "outs": outs, + "name": path, } if len(outs) > 1 and outs[0][0] != key: @@ -206,12 +153,10 @@ def info(self, path): ret["type"] = "directory" return ret - def get_file( - self, from_info, to_file, callback=DEFAULT_CALLBACK, **kwargs - ): - fs, path = self._get_fs_path(from_info) + def get_file(self, rpath, lpath, callback=DEFAULT_CALLBACK, **kwargs): + fs, path = self._get_fs_path(rpath) fs.get_file( # pylint: disable=protected-access - path, to_file, callback=callback, **kwargs + path, lpath, callback=callback, **kwargs ) def checksum(self, path): @@ -220,3 +165,24 @@ def checksum(self, path): if md5: return md5 raise NotImplementedError + + +class DvcFileSystem(FSSpecWrapper): + scheme = "local" + + PARAM_CHECKSUM = "md5" + + def _prepare_credentials(self, **config): + return config + + @wrap_prop(threading.Lock()) + @cached_property + def fs(self): + return _DvcFileSystem(**self.fs_args) + + def isdvc(self, path, **kwargs): + return self.fs.isdvc(path, **kwargs) + + @property + def repo(self): + return self.fs.repo diff --git a/dvc/fs/repo.py b/dvc/fs/repo.py index ba4258a6d7..822104dcdf 100644 --- a/dvc/fs/repo.py +++ b/dvc/fs/repo.py @@ -1,6 +1,7 @@ import logging import os import threading +from contextlib import suppress from itertools import takewhile from typing import TYPE_CHECKING, Callable, Optional, Tuple, Type, Union @@ -23,6 +24,21 @@ def _wrap_walk(dvc_fs, *args, **kwargs): yield dvc_fs.path.join(dvc_fs.repo.root_dir, root), dnames, fnames +def _ls(fs, path): + dnames = [] + fnames = [] + + with suppress(FileNotFoundError): + for entry in fs.ls(path, detail=True): + name = fs.path.name(entry["name"]) + if entry["type"] == "directory": + dnames.append(name) + else: + fnames.append(name) + + return dnames, fnames + + class RepoFileSystem(FileSystem): # pylint:disable=abstract-method """DVC + git-tracked files fs. @@ -212,6 +228,8 @@ def _get_fs_pair( if path.startswith(repo.root_dir): dvc_path = path[len(repo.root_dir) + 1 :] + + dvc_path = dvc_path.replace("\\", "/") else: dvc_path = path @@ -326,15 +344,6 @@ def isfile(self, path): # pylint: disable=arguments-renamed return info["type"] == "file" - def _dvc_walk(self, walk): - try: - root, dirs, files = next(walk) - except StopIteration: - return - yield root, dirs, files - for _ in dirs: - yield from self._dvc_walk(walk) - def _subrepo_walk(self, dir_path, **kwargs): """Walk into a new repo. @@ -343,21 +352,17 @@ def _subrepo_walk(self, dir_path, **kwargs): """ fs, dvc_fs, dvc_path = self._get_fs_pair(dir_path) fs_walk = fs.walk(dir_path, topdown=True) - if dvc_fs: - dvc_walk = _wrap_walk(dvc_fs, dvc_path, topdown=True, **kwargs) - else: - dvc_walk = None - yield from self._walk(fs_walk, dvc_walk, **kwargs) + yield from self._walk(fs_walk, dvc_fs, dvc_path, **kwargs) - def _walk(self, repo_walk, dvc_walk=None, dvcfiles=False): + def _walk(self, repo_walk, dvc_fs, dvc_path, dvcfiles=False): from dvc.dvcfile import is_valid_filename from dvc.ignore import DvcIgnore assert repo_walk + + dvc_dirs, dvc_fnames = _ls(dvc_fs, dvc_path) if dvc_fs else ([], []) + try: - _, dvc_dirs, dvc_fnames = ( - next(dvc_walk) if dvc_walk else (None, [], []) - ) repo_root, repo_dirs, repo_fnames = next(repo_walk) except StopIteration: return @@ -398,11 +403,18 @@ def is_dvc_repo(d): dir_path = os.path.join(repo_root, dirname) yield from self._subrepo_walk(dir_path, dvcfiles=dvcfiles) elif dirname in shared: - yield from self._walk(repo_walk, dvc_walk, dvcfiles=dvcfiles) + yield from self._walk( + repo_walk, + dvc_fs, + dvc_fs.path.join(dvc_path, dirname), + dvcfiles=dvcfiles, + ) elif dirname in dvc_set: - yield from self._dvc_walk(dvc_walk) + yield from _wrap_walk( + dvc_fs, dvc_fs.path.join(dvc_path, dirname) + ) elif dirname in repo_set: - yield from self._walk(repo_walk, None, dvcfiles=dvcfiles) + yield from self._walk(repo_walk, None, None, dvcfiles=dvcfiles) def walk(self, top, topdown=True, **kwargs): """Walk and merge both DVC and repo fss. @@ -433,17 +445,13 @@ def walk(self, top, topdown=True, **kwargs): repo_walk = repo.dvcignore.walk(fs, top, topdown=topdown, **kwargs) if not dvc_fs or (repo_exists and dvc_fs.isdvc(dvc_path)): - yield from self._walk(repo_walk, None, dvcfiles=dvcfiles) + yield from self._walk(repo_walk, None, None, dvcfiles=dvcfiles) return if not repo_exists: yield from _wrap_walk(dvc_fs, dvc_path, topdown=topdown, **kwargs) - dvc_walk = None - if dvc_fs.exists(dvc_path): - dvc_walk = _wrap_walk(dvc_fs, dvc_path, topdown=topdown, **kwargs) - - yield from self._walk(repo_walk, dvc_walk, dvcfiles=dvcfiles) + yield from self._walk(repo_walk, dvc_fs, dvc_path, dvcfiles=dvcfiles) def find(self, path, prefix=None): for root, _, files in self.walk(path): diff --git a/dvc/fs/webdav.py b/dvc/fs/webdav.py index 0eac678673..4fb8a766b2 100644 --- a/dvc/fs/webdav.py +++ b/dvc/fs/webdav.py @@ -13,6 +13,7 @@ class WebDAVFileSystem(FSSpecWrapper): # pylint:disable=abstract-method scheme = Schemes.WEBDAV + root_marker = "" CAN_TRAVERSE = True TRAVERSE_PREFIX_LEN = 2 REQUIRES = {"webdav4": "webdav4"} @@ -33,14 +34,6 @@ def __init__(self, **config): } ) - @classmethod - def _strip_protocol(cls, path: str) -> str: - """ - For WebDAVFileSystem, provided url is the base path itself, so it - should be treated as being a root path. - """ - return "" - def unstrip_protocol(self, path: str) -> str: return self.fs_args["base_url"] + "/" + path diff --git a/dvc/render/convert.py b/dvc/render/convert.py index de78728c6e..84b330ab84 100644 --- a/dvc/render/convert.py +++ b/dvc/render/convert.py @@ -60,7 +60,7 @@ def to_json(renderer, split: bool = False) -> List[Dict]: return [ { TYPE_KEY: renderer.TYPE, - REVISIONS_KEY: datapoint.get(REVISION_FIELD), + REVISIONS_KEY: [datapoint.get(REVISION_FIELD)], "url": datapoint.get(SRC_FIELD), } for datapoint in renderer.datapoints diff --git a/dvc/repo/index.py b/dvc/repo/index.py index ccf667b0bb..f896c93756 100644 --- a/dvc/repo/index.py +++ b/dvc/repo/index.py @@ -91,6 +91,13 @@ def __contains__(self, stage: "Stage") -> bool: def __iter__(self) -> Iterator["Stage"]: yield from self.stages + def __getitem__(self, item: str) -> "Stage": + """Get a stage by its addressing attribute.""" + for stage in self: + if stage.addressing == item: + return stage + raise KeyError(f"{item} - available stages are {self.stages}") + def filter(self, filter_fn: Callable[["Stage"], bool]) -> "Index": stages_it = filter(filter_fn, self) return Index(self.repo, self.fs, stages=list(stages_it)) diff --git a/pyproject.toml b/pyproject.toml index 8c125efc1e..284868d51b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["setuptools>=48", "wheel", "setuptools_scm[toml]>=6.3.1"] +requires = ["setuptools>=48", "setuptools_scm[toml]>=6.3.1", "setuptools_scm_git_archive==1.1"] build-backend = "setuptools.build_meta" [tool.setuptools_scm] diff --git a/setup.cfg b/setup.cfg index 205012b34a..460316e8c6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -35,12 +35,9 @@ zip_safe = False packages = find: include_package_data = True install_requires = - # See https://github.com/pyinstaller/pyinstaller/issues/1945 - ply>=3.9 colorama>=0.3.9 configobj>=5.0.6 nanotime>=0.5.2 - pyasn1>=0.4.1 voluptuous>=0.11.7 requests>=2.22.0 # See https://github.com/bdcht/grandalf/issues/26 @@ -70,13 +67,13 @@ install_requires = dictdiffer>=0.8.1 python-benedict>=0.24.2 pyparsing>=2.4.7 - typing_extensions>=3.7.4 + typing-extensions>=3.7.4 fsspec[http]>=2021.10.1 aiohttp-retry>=2.4.5 diskcache>=5.2.1 jaraco.windows>=5.7.0; python_version < '3.8' and sys_platform == 'win32' scmrepo==0.0.16 - dvc_render==0.0.4 + dvc-render==0.0.4 [options.extras_require] all = @@ -124,7 +121,6 @@ webdhfs_kerberos = requests-kerberos==0.14.0 terraform = tpi[ssh]>=2.1.0 tests = %(terraform)s - wheel==0.37.1 dvc_ssh==0.0.1a0 # Test requirements pytest==7.1.1 @@ -137,19 +133,7 @@ tests = flaky==3.7.0 mock==4.0.3 pytest-timeout==2.1.0 - rangehttpserver==1.2.0 - mock-ssh-server==0.9.1 - wget==3.2 filelock==3.6.0 - wsgidav==4.0.1 - crc32c==2.2.post0 - xmltodict==0.12.0 - google-compute-engine==2.8.13 - google-cloud-storage==2.2.1 - dvclive[image]==0.6.0 - hdfs==2.7.0 - collective.checkdocs==0.2 - pydocstyle==6.1.1 # pylint requirements pylint==2.12.2 # we use this to suppress pytest-related false positives in our tests. @@ -159,11 +143,12 @@ tests = pylint-plugin-utils==0.7 # type-checking mypy==0.942 - types-requests==2.27.15 - types-tabulate==0.8.6 - types-toml==0.10.4 + types-requests>=2.27.15 + types-tabulate>=0.8.6 + types-toml>=0.10.4 # optional dependencies pywin32>=225; sys_platform == 'win32' + dvclive[image]==0.6.0 [options.packages.find] exclude = diff --git a/tests/func/experiments/test_show.py b/tests/func/experiments/test_show.py index a83b70ee32..efa7490e1e 100644 --- a/tests/func/experiments/test_show.py +++ b/tests/func/experiments/test_show.py @@ -621,6 +621,10 @@ def test_show_parallel_coordinates(tmp_dir, dvc, scm, mocker, capsys): assert '"label": "Created"' not in html_text assert '"label": "foobar"' not in html_text + assert main(["exp", "show", "--pcp", "--drop", "Experiment"]) == 0 + html_text = (tmp_dir / "dvc_plots" / "index.html").read_text() + assert '"label": "Experiment"' not in html_text + def test_show_outs(tmp_dir, dvc, scm): tmp_dir.gen("copy.py", COPY_SCRIPT) diff --git a/tests/func/test_live.py b/tests/func/test_live.py index e5aca88594..b3186ba970 100644 --- a/tests/func/test_live.py +++ b/tests/func/test_live.py @@ -8,6 +8,9 @@ from dvc import stage as stage_module from dvc.render.match import get_files +pytest.importorskip("dvclive", reason="no dvclive") + + LIVE_SCRIPT = dedent( """ from dvclive import Live @@ -60,11 +63,6 @@ def dump(value, path): @pytest.fixture def live_stage(tmp_dir, scm, dvc, mocker): - try: - import dvclive # noqa, pylint:disable=unused-import - except ImportError: - pytest.skip("no dvclive") - mocker.patch("dvc.stage.run.Monitor.AWAIT", 0.01) def make( @@ -156,11 +154,6 @@ def test_live_html(tmp_dir, dvc, live_stage, html): @pytest.fixture def live_checkpoint_stage(tmp_dir, scm, dvc, mocker): - try: - import dvclive # noqa, pylint:disable=unused-import - except ImportError: - pytest.skip("no dvclive") - mocker.patch("dvc.stage.run.Monitor.AWAIT", 0.01) def make(live=None, live_no_cache=None): diff --git a/tests/func/test_repo_index.py b/tests/func/test_repo_index.py index 22826a78c0..b3290750fd 100644 --- a/tests/func/test_repo_index.py +++ b/tests/func/test_repo_index.py @@ -291,3 +291,15 @@ def test_used_objs(tmp_dir, scm, dvc, run_copy, rev): assert index.used_objs("copy-foo-bar", with_deps=True) == { None: {expected_objs[0]} } + + +def test_getitem(tmp_dir, dvc, run_copy): + (stage1,) = tmp_dir.dvc_gen("foo", "foo") + stage2 = run_copy("foo", "bar", name="copy-foo-bar") + + index = Index(dvc) + assert index["foo.dvc"] == stage1 + assert index["copy-foo-bar"] == stage2 + + with pytest.raises(KeyError): + _ = index["no-valid-stage-name"] diff --git a/tests/unit/command/test_plots.py b/tests/unit/command/test_plots.py index b740b033e8..a1526a80d9 100644 --- a/tests/unit/command/test_plots.py +++ b/tests/unit/command/test_plots.py @@ -336,3 +336,30 @@ def test_plots_templates_choices(tmp_dir, dvc): assert CmdPlotsTemplates.TEMPLATES_CHOICES == list( pluck_attr("DEFAULT_NAME", TEMPLATES) ) + + +@pytest.mark.parametrize("split", (True, False)) +def test_show_json(split, mocker, capsys): + import dvc.commands.plots + + renderer = mocker.MagicMock() + renderer.name = "rname" + to_json_mock = mocker.patch( + "dvc.render.convert.to_json", return_value={"renderer": "json"} + ) + + dvc.commands.plots._show_json([renderer], split) + + to_json_mock.assert_called_once_with(renderer, split) + + out, _ = capsys.readouterr() + assert json.dumps({"rname": {"renderer": "json"}}) in out + + +def test_show_json_no_renderers(capsys): + import dvc.commands.plots + + dvc.commands.plots._show_json([]) + + out, _ = capsys.readouterr() + assert json.dumps({}) in out diff --git a/tests/unit/fs/test_dvc.py b/tests/unit/fs/test_dvc.py index 4dd7a9c0ca..45fda5e63f 100644 --- a/tests/unit/fs/test_dvc.py +++ b/tests/unit/fs/test_dvc.py @@ -1,4 +1,4 @@ -import os +import posixpath import shutil import pytest @@ -16,12 +16,12 @@ ("", ("repo",)), (".", ("repo",)), ("foo", ("repo", "foo")), - (os.path.join("dir", "foo"), ("repo", "dir", "foo")), + ("dir/foo", ("repo", "dir", "foo")), ], ) def test_get_key(tmp_dir, dvc, path, key): fs = DvcFileSystem(repo=dvc) - assert fs._get_key(path) == key + assert fs.fs._get_key(path) == key def test_exists(tmp_dir, dvc): @@ -143,19 +143,19 @@ def test_walk(tmp_dir, dvc): fs = DvcFileSystem(repo=dvc) expected = [ - os.path.join("dir", "subdir1"), - os.path.join("dir", "subdir2"), - os.path.join("dir", "subdir1", "foo1"), - os.path.join("dir", "subdir1", "bar1"), - os.path.join("dir", "subdir2", "foo2"), - os.path.join("dir", "foo"), - os.path.join("dir", "bar"), + "dir/subdir1", + "dir/subdir2", + "dir/subdir1/foo1", + "dir/subdir1/bar1", + "dir/subdir2/foo2", + "dir/foo", + "dir/bar", ] actual = [] for root, dirs, files in fs.walk("dir"): for entry in dirs + files: - actual.append(os.path.join(root, entry)) + actual.append(posixpath.join(root, entry)) assert set(actual) == set(expected) assert len(actual) == len(expected) @@ -177,19 +177,19 @@ def test_walk_dir(tmp_dir, dvc): fs = DvcFileSystem(repo=dvc) expected = [ - os.path.join("dir", "subdir1"), - os.path.join("dir", "subdir2"), - os.path.join("dir", "subdir1", "foo1"), - os.path.join("dir", "subdir1", "bar1"), - os.path.join("dir", "subdir2", "foo2"), - os.path.join("dir", "foo"), - os.path.join("dir", "bar"), + "dir/subdir1", + "dir/subdir2", + "dir/subdir1/foo1", + "dir/subdir1/bar1", + "dir/subdir2/foo2", + "dir/foo", + "dir/bar", ] actual = [] for root, dirs, files in fs.walk("dir"): for entry in dirs + files: - actual.append(os.path.join(root, entry)) + actual.append(posixpath.join(root, entry)) assert set(actual) == set(expected) assert len(actual) == len(expected) @@ -241,13 +241,13 @@ def test_get_hash_granular(tmp_dir, dvc): {"dir": {"foo": "foo", "bar": "bar", "subdir": {"data": "data"}}} ) fs = DvcFileSystem(repo=dvc) - subdir = os.path.join("dir", "subdir") + subdir = "dir/subdir" assert fs.info(subdir).get("md5") is None _, _, obj = stage(dvc.odb.local, subdir, fs, "md5", dry_run=True) assert obj.hash_info == HashInfo( "md5", "af314506f1622d107e0ed3f14ec1a3b5.dir" ) - data = os.path.join(subdir, "data") + data = posixpath.join(subdir, "data") assert fs.info(data)["md5"] == "8d777f385d3dfec8815d20f7496026dc" _, _, obj = stage(dvc.odb.local, data, fs, "md5", dry_run=True) assert obj.hash_info == HashInfo("md5", "8d777f385d3dfec8815d20f7496026dc") diff --git a/tests/unit/render/test_convert.py b/tests/unit/render/test_convert.py index 077a86c7ca..8f91d6d1fb 100644 --- a/tests/unit/render/test_convert.py +++ b/tests/unit/render/test_convert.py @@ -214,6 +214,6 @@ def test_to_json_image(mocker): result = to_json(image_renderer) assert result[0] == { "url": image_renderer.datapoints[0].get(SRC_FIELD), - REVISIONS_KEY: image_renderer.datapoints[0].get(REVISION_FIELD), + REVISIONS_KEY: [image_renderer.datapoints[0].get(REVISION_FIELD)], TYPE_KEY: image_renderer.TYPE, }