diff --git a/src/neptune/new/cli/__main__.py b/src/neptune/new/cli/__main__.py index 2b0809c24..20e42c29a 100644 --- a/src/neptune/new/cli/__main__.py +++ b/src/neptune/new/cli/__main__.py @@ -18,6 +18,7 @@ import pkg_resources from neptune.new.cli.commands import ( + clear, status, sync, ) @@ -30,6 +31,7 @@ def main(): main.add_command(sync) main.add_command(status) +main.add_command(clear) plugins = {entry_point.name: entry_point for entry_point in pkg_resources.iter_entry_points("neptune.plugins")} diff --git a/src/neptune/new/cli/clear.py b/src/neptune/new/cli/clear.py new file mode 100644 index 000000000..3a31a0963 --- /dev/null +++ b/src/neptune/new/cli/clear.py @@ -0,0 +1,80 @@ +# +# Copyright (c) 2022, Neptune Labs Sp. z o.o. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +__all__ = ["ClearRunner"] + +import shutil +from pathlib import Path +from typing import Sequence + +import click + +from neptune.new.cli.abstract_backend_runner import AbstractBackendRunner +from neptune.new.cli.container_manager import ContainersManager +from neptune.new.cli.status import StatusRunner +from neptune.new.cli.utils import get_offline_dirs +from neptune.new.internal.backends.api_model import ApiExperiment +from neptune.new.internal.id_formats import UniqueId +from neptune.new.internal.utils.logger import logger + + +class ClearRunner(AbstractBackendRunner): + def clear(self, path: Path, force: bool = False): + container_manager = ContainersManager(self._backend, path) + synced_containers, unsynced_containers, not_found = container_manager.partition_containers_and_clean_junk(path) + + offline_containers = get_offline_dirs(path) + + ClearRunner.remove_containers(not_found) + + if offline_containers or unsynced_containers: + self.log_junk_metadata(offline_containers, unsynced_containers) + + if force or click.confirm("\nDo you want to delete the listed metadata?"): + self.remove_data(container_manager, offline_containers, unsynced_containers) + + @staticmethod + def log_junk_metadata(offline_containers, unsynced_containers): + if unsynced_containers: + logger.info("") + StatusRunner.log_unsync_objects(unsynced_containers=unsynced_containers) + if offline_containers: + logger.info("") + StatusRunner.log_offline_objects(offline_dirs=offline_containers, info=False) + + @staticmethod + def remove_data( + container_manager: ContainersManager, + offline_containers: Sequence[UniqueId], + unsynced_containers: Sequence[ApiExperiment], + ): + + offline_containers_paths = [container_manager.resolve_offline_container_dir(x) for x in offline_containers] + unsynced_containers_paths = [ + container_manager.resolve_async_path(container) for container in unsynced_containers + ] + + ClearRunner.remove_containers(offline_containers_paths) + ClearRunner.remove_containers(unsynced_containers_paths) + + @staticmethod + def remove_containers(paths): + for path in paths: + try: + shutil.rmtree(path) + logger.info(f"Deleted: {path}") + except OSError: + logger.warn(f"Cannot remove directory: {path}") diff --git a/src/neptune/new/cli/commands.py b/src/neptune/new/cli/commands.py index 02c92b626..92d26fc26 100644 --- a/src/neptune/new/cli/commands.py +++ b/src/neptune/new/cli/commands.py @@ -14,7 +14,7 @@ # limitations under the License. # -__all__ = ["status", "sync"] +__all__ = ["status", "sync", "clear"] from pathlib import Path from typing import ( @@ -25,9 +25,10 @@ import click from neptune.common.exceptions import NeptuneException # noqa: F401 +from neptune.new.cli.clear import ClearRunner +from neptune.new.cli.path_option import path_option from neptune.new.cli.status import StatusRunner from neptune.new.cli.sync import SyncRunner -from neptune.new.constants import NEPTUNE_DATA_DIRECTORY from neptune.new.exceptions import ( # noqa: F401 CannotSynchronizeOfflineRunsWithoutProject, NeptuneConnectionLostException, @@ -46,27 +47,6 @@ from neptune.new.internal.utils.logger import logger -def get_neptune_path(ctx, param, path: str) -> Path: - # check if path exists and contains a '.neptune' folder - path = Path(path) - if (path / NEPTUNE_DATA_DIRECTORY).is_dir(): - return path / NEPTUNE_DATA_DIRECTORY - elif path.name == NEPTUNE_DATA_DIRECTORY and path.is_dir(): - return path - else: - raise click.BadParameter("Path {} does not contain a '{}' folder.".format(path, NEPTUNE_DATA_DIRECTORY)) - - -path_option = click.option( - "--path", - type=click.Path(exists=True, file_okay=False, resolve_path=True), - default=Path.cwd(), - callback=get_neptune_path, - metavar="", - help="path to a directory containing a '.neptune' folder with stored objects", -) - - @click.command() @path_option def status(path: Path) -> None: @@ -190,3 +170,27 @@ def sync( sync_runner.sync_selected_containers(path, project_name, object_names) else: sync_runner.sync_all_containers(path, project_name) + + +@click.command() +@path_option +def clear(path: Path): + """ + Clears metadata that has been synchronized or trashed, but is still present in local storage. + + Lists objects and data to be cleared before deleting the data. + + Examples: + + \b + # Clear junk metadata from local storage + neptune clear + + \b + # Clear junk metadata from directory "foo/bar" + neptune clear --path foo/bar + """ + backend = HostedNeptuneBackend(Credentials.from_token()) + clear_runner = ClearRunner(backend=backend) + + clear_runner.clear(path) diff --git a/src/neptune/new/cli/container_manager.py b/src/neptune/new/cli/container_manager.py new file mode 100644 index 000000000..14dc1b40f --- /dev/null +++ b/src/neptune/new/cli/container_manager.py @@ -0,0 +1,83 @@ +# +# Copyright (c) 2022, Neptune Labs Sp. z o.o. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +__all__ = ["ContainersManager"] + +import abc +from pathlib import Path +from typing import ( + List, + Tuple, +) + +from neptune.new.cli.utils import ( + get_metadata_container, + is_container_synced_and_remove_junk, + iterate_containers, +) +from neptune.new.constants import ( + ASYNC_DIRECTORY, + OFFLINE_DIRECTORY, + SYNC_DIRECTORY, +) +from neptune.new.internal.backends.api_model import ApiExperiment +from neptune.new.internal.backends.neptune_backend import NeptuneBackend +from neptune.new.internal.id_formats import UniqueId + + +class ContainersManager(abc.ABC): + _backend: NeptuneBackend + + def __init__(self, backend: NeptuneBackend, base_path: Path): + self._backend = backend + self._base_path = base_path + + def partition_containers_and_clean_junk( + self, + base_path: Path, + ) -> Tuple[List[ApiExperiment], List[ApiExperiment], List[Path]]: + synced_containers = [] + unsynced_containers = [] + not_found = [] + async_path = base_path / ASYNC_DIRECTORY + for container_type, container_id, path in iterate_containers(async_path): + metadata_container = get_metadata_container( + backend=self._backend, + container_id=container_id, + container_type=container_type, + ) + if metadata_container: + if is_container_synced_and_remove_junk(path): + synced_containers.append(metadata_container) + else: + + unsynced_containers.append(metadata_container) + else: + not_found.append(path) + + synced_containers = [obj for obj in synced_containers if obj] + unsynced_containers = [obj for obj in unsynced_containers if obj] + + return synced_containers, unsynced_containers, not_found + + def resolve_async_path(self, container: ApiExperiment) -> Path: + return self._base_path / ASYNC_DIRECTORY / container.type.create_dir_name(container.id) + + def resolve_offline_container_dir(self, offline_id: UniqueId): + return self._base_path / OFFLINE_DIRECTORY / offline_id + + def iterate_sync_containers(self): + return iterate_containers(self._base_path / SYNC_DIRECTORY) diff --git a/src/neptune/new/cli/path_option.py b/src/neptune/new/cli/path_option.py new file mode 100644 index 000000000..4b8a8cbdb --- /dev/null +++ b/src/neptune/new/cli/path_option.py @@ -0,0 +1,58 @@ +# +# Copyright (c) 2022, Neptune Labs Sp. z o.o. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +__all__ = ["path_option"] + +from pathlib import Path + +import click + +from neptune.common.exceptions import NeptuneException # noqa: F401 +from neptune.new.constants import NEPTUNE_DATA_DIRECTORY +from neptune.new.exceptions import ( # noqa: F401 + CannotSynchronizeOfflineRunsWithoutProject, + NeptuneConnectionLostException, + ProjectNotFound, + RunNotFound, +) +from neptune.new.internal.backends.api_model import ( # noqa: F401 + ApiExperiment, + Project, +) +from neptune.new.internal.backends.neptune_backend import NeptuneBackend # noqa: F401 +from neptune.new.internal.disk_queue import DiskQueue # noqa: F401 +from neptune.new.internal.operation import Operation # noqa: F401 + + +def get_neptune_path(ctx, param, path: str) -> Path: + # check if path exists and contains a '.neptune' folder + path = Path(path) + if (path / NEPTUNE_DATA_DIRECTORY).is_dir(): + return path / NEPTUNE_DATA_DIRECTORY + elif path.name == NEPTUNE_DATA_DIRECTORY and path.is_dir(): + return path + else: + raise click.BadParameter("Path {} does not contain a '{}' folder.".format(path, NEPTUNE_DATA_DIRECTORY)) + + +path_option = click.option( + "--path", + type=click.Path(exists=True, file_okay=False, resolve_path=True), + default=Path.cwd(), + callback=get_neptune_path, + metavar="", + help="path to a directory containing a '.neptune' folder with stored objects", +) diff --git a/src/neptune/new/cli/status.py b/src/neptune/new/cli/status.py index abed60456..770c29ad1 100644 --- a/src/neptune/new/cli/status.py +++ b/src/neptune/new/cli/status.py @@ -20,23 +20,17 @@ import textwrap from pathlib import Path from typing import ( - List, + Optional, Sequence, - Tuple, ) from neptune.new.cli.abstract_backend_runner import AbstractBackendRunner +from neptune.new.cli.container_manager import ContainersManager from neptune.new.cli.utils import ( - get_metadata_container, get_offline_dirs, get_qualified_name, - is_container_synced, - iterate_containers, -) -from neptune.new.constants import ( - ASYNC_DIRECTORY, - OFFLINE_NAME_PREFIX, ) +from neptune.new.constants import OFFLINE_NAME_PREFIX from neptune.new.envs import PROJECT_ENV_NAME from neptune.new.internal.backends.api_model import ApiExperiment from neptune.new.internal.utils.logger import logger @@ -53,60 +47,21 @@ class StatusRunner(AbstractBackendRunner): - def partition_containers( - self, - base_path: Path, - ) -> Tuple[List[ApiExperiment], List[ApiExperiment], int]: - synced_containers = [] - unsynced_containers = [] - async_path = base_path / ASYNC_DIRECTORY - for container_type, container_id, path in iterate_containers(async_path): - metadata_container = get_metadata_container( - backend=self._backend, - container_id=container_id, - container_type=container_type, - ) - - if is_container_synced(path): - synced_containers.append(metadata_container) - else: - unsynced_containers.append(metadata_container) - - not_found = len([obj for obj in synced_containers + unsynced_containers if not obj]) - synced_containers = [obj for obj in synced_containers if obj] - unsynced_containers = [obj for obj in unsynced_containers if obj] - - return synced_containers, unsynced_containers, not_found - @staticmethod def list_containers( base_path: Path, - synced_containers: Sequence[ApiExperiment], + synced_containers: Optional[Sequence[ApiExperiment]], unsynced_containers: Sequence[ApiExperiment], offline_dirs: Sequence[str], ) -> None: - def trashed(cont: ApiExperiment): - return " (Trashed)" if cont.trashed else "" if not synced_containers and not unsynced_containers and not offline_dirs: logger.info("There are no Neptune objects in %s", base_path) sys.exit(1) - if unsynced_containers: - logger.info("Unsynchronized objects:") - for container in unsynced_containers: - logger.info("- %s%s", get_qualified_name(container), trashed(container)) - - if synced_containers: - logger.info("Synchronized objects:") - for container in synced_containers: - logger.info("- %s%s", get_qualified_name(container), trashed(container)) + StatusRunner.log_unsync_objects(unsynced_containers) - if offline_dirs: - logger.info("Unsynchronized offline objects:") - for container_id in offline_dirs: - logger.info("- %s", f"{OFFLINE_NAME_PREFIX}{container_id}") - logger.info("\n%s", textwrap.fill(offline_run_explainer, width=90)) + StatusRunner.log_offline_objects(offline_dirs) if not unsynced_containers: logger.info("\nThere are no unsynchronized objects in %s", base_path) @@ -116,12 +71,35 @@ def trashed(cont: ApiExperiment): logger.info("\nPlease run with the `neptune sync --help` to see example commands.") + @staticmethod + def trashed(cont: ApiExperiment): + return " (Trashed)" if cont.trashed else "" + + @staticmethod + def log_offline_objects(offline_dirs, info=True): + if offline_dirs: + logger.info("Unsynchronized offline objects:") + for container_id in offline_dirs: + logger.info("- %s", f"{OFFLINE_NAME_PREFIX}{container_id}") + if info: + logger.info("\n%s", textwrap.fill(offline_run_explainer, width=90)) + + @staticmethod + def log_unsync_objects(unsynced_containers): + if unsynced_containers: + logger.info("Unsynchronized objects:") + for container in unsynced_containers: + logger.info("- %s%s", get_qualified_name(container), StatusRunner.trashed(container)) + def synchronization_status(self, base_path: Path) -> None: - synced_containers, unsynced_containers, not_found = self.partition_containers(base_path) - if not_found > 0: + container_manager = ContainersManager(self._backend, base_path) + synced_containers, unsynced_containers, not_found = container_manager.partition_containers_and_clean_junk( + base_path + ) + if len(not_found) > 0: logger.warning( - "WARNING: %s objects was skipped because they do not exist anymore.", - not_found, + "\nWARNING: %s objects was skipped because they do not exist anymore.", + len(not_found), ) offline_dirs = get_offline_dirs(base_path) self.list_containers(base_path, synced_containers, unsynced_containers, offline_dirs) diff --git a/src/neptune/new/cli/sync.py b/src/neptune/new/cli/sync.py index aa37a7a43..769516798 100644 --- a/src/neptune/new/cli/sync.py +++ b/src/neptune/new/cli/sync.py @@ -34,7 +34,7 @@ get_offline_dirs, get_project, get_qualified_name, - is_container_synced, + is_container_synced_and_remove_junk, iterate_containers, split_dir_name, ) @@ -121,7 +121,7 @@ def sync_execution( def sync_all_registered_containers(self, base_path: Path) -> None: async_path = base_path / ASYNC_DIRECTORY for container_type, unique_id, path in iterate_containers(async_path): - if not is_container_synced(path): + if not is_container_synced_and_remove_junk(path): container = get_metadata_container( backend=self._backend, container_id=unique_id, diff --git a/src/neptune/new/cli/utils.py b/src/neptune/new/cli/utils.py index 4bcfefafe..8b5a564c8 100644 --- a/src/neptune/new/cli/utils.py +++ b/src/neptune/new/cli/utils.py @@ -18,7 +18,7 @@ "get_metadata_container", "get_project", "get_qualified_name", - "is_container_synced", + "is_container_synced_and_remove_junk", "get_offline_dirs", "iterate_containers", "split_dir_name", @@ -107,18 +107,16 @@ def get_qualified_name(experiment: ApiExperiment) -> QualifiedName: return QualifiedName(f"{experiment.workspace}/{experiment.project_name}/{experiment.sys_id}") -def is_container_synced(experiment_path: Path) -> bool: - return all(_is_execution_synced(execution_path) for execution_path in experiment_path.iterdir()) +def is_container_synced_and_remove_junk(experiment_path: Path) -> bool: + return all(_is_execution_synced_and_remove_junk(execution_path) for execution_path in experiment_path.iterdir()) -def _is_execution_synced(execution_path: Path) -> bool: - disk_queue = DiskQueue( - execution_path, - lambda x: x.to_dict(), - Operation.from_dict, - threading.RLock(), - ) - return disk_queue.is_empty() +def _is_execution_synced_and_remove_junk(execution_path: Path) -> bool: + """ + The DiskQueue.close() method remove junk metadata from disk when queue is empty. + """ + with DiskQueue(execution_path, lambda x: x.to_dict(), Operation.from_dict, threading.RLock()) as disk_queue: + return disk_queue.is_empty() def split_dir_name(dir_name: str) -> Tuple[ContainerType, UniqueId]: diff --git a/src/neptune/new/exceptions.py b/src/neptune/new/exceptions.py index 7806920af..d4f3b2983 100644 --- a/src/neptune/new/exceptions.py +++ b/src/neptune/new/exceptions.py @@ -313,7 +313,10 @@ def __init__(self, container_id: str, container_type: ContainerType): self.container_id = container_id self.container_type = container_type super().__init__( - "{} with ID {} not found. It may have been deleted.".format(container_type.value.capitalize(), container_id) + "{} with ID {} not found. It may have been deleted. " + "You can use the 'neptune clear' command to delete junk objects from local storage.".format( + container_type.value.capitalize(), container_id + ) ) diff --git a/src/neptune/new/internal/disk_queue.py b/src/neptune/new/internal/disk_queue.py index 143d9889d..6217c6047 100644 --- a/src/neptune/new/internal/disk_queue.py +++ b/src/neptune/new/internal/disk_queue.py @@ -148,6 +148,9 @@ def flush(self): self._last_put_file.flush() def close(self): + """ + Close and remove underlying files if queue is empty + """ self._reader.close() self._writer.close() self._last_ack_file.close() @@ -225,3 +228,9 @@ def _serialize(self, obj: T, version: int) -> dict: def _deserialize(self, data: dict) -> Tuple[T, int]: return self._from_dict(data["obj"]), data["version"] + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.close() diff --git a/src/neptune/new/sync/__init__.py b/src/neptune/new/sync/__init__.py index 589e9750a..d8352e064 100644 --- a/src/neptune/new/sync/__init__.py +++ b/src/neptune/new/sync/__init__.py @@ -26,7 +26,7 @@ "get_metadata_container", "get_project", "get_qualified_name", - "is_container_synced", + "is_container_synced_and_remove_junk", "get_offline_dirs", "iterate_containers", "split_dir_name", @@ -57,7 +57,7 @@ get_offline_dirs, get_project, get_qualified_name, - is_container_synced, + is_container_synced_and_remove_junk, iterate_containers, split_dir_name, ) diff --git a/src/neptune/new/sync/utils.py b/src/neptune/new/sync/utils.py index 8805822b6..978f27ab0 100644 --- a/src/neptune/new/sync/utils.py +++ b/src/neptune/new/sync/utils.py @@ -18,7 +18,7 @@ "get_metadata_container", "get_project", "get_qualified_name", - "is_container_synced", + "is_container_synced_and_remove_junk", "get_offline_dirs", "iterate_containers", "split_dir_name", @@ -31,7 +31,7 @@ get_offline_dirs, get_project, get_qualified_name, - is_container_synced, + is_container_synced_and_remove_junk, iterate_containers, split_dir_name, ) diff --git a/tests/neptune/new/cli/test_clear.py b/tests/neptune/new/cli/test_clear.py new file mode 100644 index 000000000..885ab5d9c --- /dev/null +++ b/tests/neptune/new/cli/test_clear.py @@ -0,0 +1,111 @@ +# +# Copyright (c) 2022, Neptune Labs Sp. z o.o. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import os +from unittest.mock import MagicMock + +import pytest + +from neptune.new.cli.clear import ClearRunner +from neptune.new.cli.utils import get_qualified_name +from neptune.new.constants import ( + ASYNC_DIRECTORY, + OFFLINE_DIRECTORY, +) +from neptune.new.internal.container_type import ContainerType +from neptune.new.internal.operation import Operation +from tests.neptune.new.cli.utils import ( + generate_get_metadata_container, + prepare_metadata_container, +) + + +@pytest.fixture(name="backend") +def backend_fixture(): + return MagicMock() + + +@pytest.fixture(name="clear_runner") +def status_runner_fixture(backend): + return ClearRunner(backend=backend) + + +@pytest.mark.parametrize("container_type", list(ContainerType)) +def test_clean_containers(tmp_path, mocker, capsys, backend, clear_runner, container_type): + # given + unsynced_container = prepare_metadata_container(container_type=container_type, path=tmp_path, last_ack_version=1) + synced_container = prepare_metadata_container(container_type=container_type, path=tmp_path, last_ack_version=3) + offline_containers = prepare_metadata_container(container_type=container_type, path=tmp_path, last_ack_version=None) + get_container_impl = generate_get_metadata_container(registered_containers=(unsynced_container, synced_container)) + + # and + mocker.patch.object(backend, "get_metadata_container", get_container_impl) + mocker.patch.object(Operation, "from_dict") + + assert os.path.exists(tmp_path / ASYNC_DIRECTORY / container_type.create_dir_name(unsynced_container.id)) + assert os.path.exists(tmp_path / ASYNC_DIRECTORY / container_type.create_dir_name(synced_container.id)) + assert os.path.exists(tmp_path / OFFLINE_DIRECTORY / container_type.create_dir_name(offline_containers.id)) + + # when + clear_runner.clear(tmp_path, force=True) + + # then + assert not os.path.exists(tmp_path / ASYNC_DIRECTORY / container_type.create_dir_name(unsynced_container.id)) + assert not os.path.exists(tmp_path / ASYNC_DIRECTORY / container_type.create_dir_name(synced_container.id)) + assert not os.path.exists(tmp_path / OFFLINE_DIRECTORY / container_type.create_dir_name(offline_containers.id)) + + # and + captured = capsys.readouterr() + assert captured.out.splitlines() == [ + "", + "Unsynchronized objects:", + f"- {get_qualified_name(unsynced_container)}", + "", + "Unsynchronized offline objects:", + f"- offline/{container_type.create_dir_name(offline_containers.id)}", + f"Deleted: {tmp_path / OFFLINE_DIRECTORY / container_type.create_dir_name(offline_containers.id)}", + f"Deleted: {tmp_path / ASYNC_DIRECTORY / container_type.create_dir_name(unsynced_container.id)}", + ] + + +@pytest.mark.parametrize("container_type", list(ContainerType)) +def test_clean_deleted_containers(tmp_path, mocker, capsys, backend, clear_runner, container_type): + # given + unsynced_container = prepare_metadata_container(container_type=container_type, path=tmp_path, last_ack_version=1) + synced_container = prepare_metadata_container(container_type=container_type, path=tmp_path, last_ack_version=3) + empty_get_container_impl = generate_get_metadata_container(registered_containers=[]) + + # and + mocker.patch.object(backend, "get_metadata_container", empty_get_container_impl) + mocker.patch.object(Operation, "from_dict") + + assert os.path.exists(tmp_path / ASYNC_DIRECTORY / container_type.create_dir_name(synced_container.id)) + assert os.path.exists(tmp_path / ASYNC_DIRECTORY / container_type.create_dir_name(unsynced_container.id)) + + # when + clear_runner.clear(tmp_path, force=True) + + # then + assert not os.path.exists(tmp_path / ASYNC_DIRECTORY / container_type.create_dir_name(synced_container.id)) + assert not os.path.exists(tmp_path / ASYNC_DIRECTORY / container_type.create_dir_name(unsynced_container.id)) + + # and + captured = capsys.readouterr() + assert set(captured.out.splitlines()) == { + f"Can't fetch ContainerType.{container_type.name} {synced_container.id}. Skipping.", + f"Can't fetch ContainerType.{container_type.name} {unsynced_container.id}. Skipping.", + f"Deleted: {tmp_path / ASYNC_DIRECTORY / container_type.create_dir_name(synced_container.id)}", + f"Deleted: {tmp_path / ASYNC_DIRECTORY / container_type.create_dir_name(unsynced_container.id)}", + } diff --git a/tests/neptune/new/cli/test_status.py b/tests/neptune/new/cli/test_status.py index eddb806bc..4593b6042 100644 --- a/tests/neptune/new/cli/test_status.py +++ b/tests/neptune/new/cli/test_status.py @@ -56,8 +56,6 @@ def test_list_containers(tmp_path, mocker, capsys, backend, status_runner, conta assert captured.out.splitlines() == [ "Unsynchronized objects:", f"- {get_qualified_name(unsynced_container)}", - "Synchronized objects:", - f"- {get_qualified_name(synced_container)}", "", "Please run with the `neptune sync --help` to see example commands.", ] @@ -105,8 +103,6 @@ def test_list_trashed_containers(tmp_path, mocker, capsys, backend, status_runne assert captured.out.splitlines() == [ "Unsynchronized objects:", f"- {get_qualified_name(unsynced_container)} (Trashed)", - "Synchronized objects:", - f"- {get_qualified_name(synced_container)} (Trashed)", "", "Please run with the `neptune sync --help` to see example commands.", ]