PR changes

src/neptune/new/cli/__init__.py

@@ -14,36 +14,9 @@
 # limitations under the License.
 #
 
-__all__ = [
-    "AbstractBackendRunner",
-    "sync",
-    "status",
-    "StatusRunner",
-    "SyncRunner",
-    "get_metadata_container",
-    "get_project",
-    "get_qualified_name",
-    "is_container_synced",
-    "get_offline_dirs",
-    "iterate_containers",
-    "create_dir_name",
-    "split_dir_name",
-]
+__all__ = ["sync", "status"]
 
-from .abstract_backend_runner import AbstractBackendRunner
-from .commands import (
+from neptune.new.cli.commands import (
     status,
     sync,
 )
-from .status import StatusRunner
-from .sync import SyncRunner
-from .utils import (
-    create_dir_name,
-    get_metadata_container,
-    get_offline_dirs,
-    get_project,
-    get_qualified_name,
-    is_container_synced,
-    iterate_containers,
-    split_dir_name,
-)

src/neptune/new/cli/sync.py

@@ -30,7 +30,6 @@
 
 from neptune.new.cli.abstract_backend_runner import AbstractBackendRunner
 from neptune.new.cli.utils import (
-    create_dir_name,
     get_metadata_container,
     get_offline_dirs,
     get_project,
@@ -140,7 +139,7 @@ def sync_selected_registered_containers(
                 container_id=name,
             )
             if run:
-                run_path = base_path / ASYNC_DIRECTORY / f"{create_dir_name(run.type, run.id)}"
+                run_path = base_path / ASYNC_DIRECTORY / f"{run.type.create_dir_name(run.id)}"
                 run_path_deprecated = base_path / ASYNC_DIRECTORY / f"{run.id}"
                 if run_path.exists():
                     self.sync_run(run_path=run_path, run=run)
@@ -169,7 +168,7 @@ def _move_offline_run(
         server_id: UniqueId,
         server_type: ContainerType,
     ) -> None:
-        online_dir = create_dir_name(container_type=server_type, container_id=server_id)
+        online_dir = server_type.create_dir_name(container_id=server_id)
         # create async directory for run
         (base_path / ASYNC_DIRECTORY / online_dir).mkdir(parents=True)
         # mv offline directory inside async one

src/neptune/new/cli/utils.py

@@ -21,7 +21,6 @@
     "is_container_synced",
     "get_offline_dirs",
     "iterate_containers",
-    "create_dir_name",
     "split_dir_name",
 ]
 
@@ -122,10 +121,6 @@ def _is_execution_synced(execution_path: Path) -> bool:
     return disk_queue.is_empty()
 
 
-def create_dir_name(container_type: ContainerType, container_id: UniqueId) -> str:
-    return f"{container_type.value}__{container_id}"
-
-
 def split_dir_name(dir_name: str) -> Tuple[ContainerType, UniqueId]:
     parts = dir_name.split("__")
     if len(parts) == 2:

src/neptune/new/internal/container_type.py

@@ -16,6 +16,8 @@
 
 import enum
 
+from neptune.new.internal.id_formats import UniqueId
+
 
 class ContainerType(str, enum.Enum):
     RUN = "run"
@@ -35,3 +37,6 @@ def from_api(api_type: str) -> "ContainerType":
             return ContainerType.MODEL_VERSION
         else:
             return ContainerType(api_type)
+
+    def create_dir_name(self, container_id: UniqueId) -> str:
+        return f"{self.value}__{container_id}"

src/neptune/new/internal/operation_processors/async_operation_processor.py

@@ -29,7 +29,6 @@
     Optional,
 )
 
-from neptune.new.cli.utils import create_dir_name
 from neptune.new.constants import (
     ASYNC_DIRECTORY,
     NEPTUNE_DATA_DIRECTORY,
@@ -91,7 +90,7 @@ def __init__(
     @staticmethod
     def _init_data_path(container_id: UniqueId, container_type: ContainerType):
         now = datetime.now()
-        container_dir = f"{NEPTUNE_DATA_DIRECTORY}/{ASYNC_DIRECTORY}/{create_dir_name(container_type, container_id)}"
+        container_dir = f"{NEPTUNE_DATA_DIRECTORY}/{ASYNC_DIRECTORY}/{container_type.create_dir_name(container_id)}"
         data_path = f"{container_dir}/exec-{now.timestamp()}-{now.strftime('%Y-%m-%d_%H.%M.%S.%f')}"
         data_path = data_path.replace(" ", "_").replace(":", ".")
         return data_path

src/neptune/new/internal/operation_processors/offline_operation_processor.py

@@ -18,7 +18,6 @@
 import threading
 from typing import Optional
 
-from neptune.new.cli.utils import create_dir_name
 from neptune.new.constants import (
     NEPTUNE_DATA_DIRECTORY,
     OFFLINE_DIRECTORY,
@@ -44,7 +43,7 @@ def __init__(self, container_id: UniqueId, container_type: ContainerType, lock:
 
     @staticmethod
     def _init_data_path(container_id: UniqueId, container_type: ContainerType):
-        return f"{NEPTUNE_DATA_DIRECTORY}/{OFFLINE_DIRECTORY}/{create_dir_name(container_type, container_id)}"
+        return f"{NEPTUNE_DATA_DIRECTORY}/{OFFLINE_DIRECTORY}/{container_type.create_dir_name(container_id)}"
 
     def enqueue_operation(self, op: Operation, wait: bool) -> None:
         self._queue.put(op)

src/neptune/new/internal/operation_processors/operation_storage.py

@@ -20,7 +20,6 @@
 import os
 from pathlib import Path
 
-from neptune.new.cli.utils import create_dir_name
 from neptune.new.constants import NEPTUNE_DATA_DIRECTORY
 from neptune.new.internal.container_type import ContainerType
 from neptune.new.internal.id_formats import UniqueId
@@ -44,4 +43,4 @@ def upload_path(self) -> Path:
 
     @staticmethod
     def _get_container_dir(type_dir: str, container_id: UniqueId, container_type: ContainerType):
-        return f"{NEPTUNE_DATA_DIRECTORY}/{type_dir}/{create_dir_name(container_type, container_id)}"
+        return f"{NEPTUNE_DATA_DIRECTORY}/{type_dir}/{container_type.create_dir_name(container_id)}"

src/neptune/new/internal/operation_processors/sync_operation_processor.py

@@ -18,7 +18,6 @@
 from datetime import datetime
 from typing import Optional
 
-from neptune.new.cli.utils import create_dir_name
 from neptune.new.constants import (
     NEPTUNE_DATA_DIRECTORY,
     SYNC_DIRECTORY,
@@ -41,7 +40,7 @@ def __init__(self, container_id: UniqueId, container_type: ContainerType, backen
     @staticmethod
     def _init_data_path(container_id: UniqueId, container_type: ContainerType):
         now = datetime.now()
-        container_dir = f"{NEPTUNE_DATA_DIRECTORY}/{SYNC_DIRECTORY}/{create_dir_name(container_type, container_id)}"
+        container_dir = f"{NEPTUNE_DATA_DIRECTORY}/{SYNC_DIRECTORY}/{container_type.create_dir_name(container_id)}"
         data_path = f"{container_dir}/exec-{now.timestamp()}-{now.strftime('%Y-%m-%d_%H.%M.%S.%f')}"
         return data_path
 

src/neptune/new/sync/__init__.py

@@ -29,10 +29,10 @@
     "is_container_synced",
     "get_offline_dirs",
     "iterate_containers",
-    "create_dir_name",
     "split_dir_name",
 ]
 
+from neptune.common.deprecation import warn_once
 from neptune.new.cli.commands import (
     ApiExperiment,
     CannotSynchronizeOfflineRunsWithoutProject,
@@ -49,12 +49,10 @@
     status,
     sync,
 )
-
-from .abstract_backend_runner import AbstractBackendRunner
-from .status import StatusRunner
-from .sync import SyncRunner
-from .utils import (
-    create_dir_name,
+from neptune.new.sync.abstract_backend_runner import AbstractBackendRunner
+from neptune.new.sync.status import StatusRunner
+from neptune.new.sync.sync import SyncRunner
+from neptune.new.sync.utils import (
     get_metadata_container,
     get_offline_dirs,
     get_project,
@@ -63,3 +61,10 @@
     iterate_containers,
     split_dir_name,
 )
+
+warn_once(
+    message="You're using a legacy neptune.new.sync package."
+    " It will be removed since `neptune-client==1.0.0`."
+    " Please use neptune.new.cli",
+    stack_level=2,
+)

src/neptune/new/sync/utils.py

@@ -21,14 +21,12 @@
     "is_container_synced",
     "get_offline_dirs",
     "iterate_containers",
-    "create_dir_name",
     "split_dir_name",
 ]
 
 # backwards compatibility
 # flake8: noqa
 from neptune.new.cli.utils import (
-    create_dir_name,
     get_metadata_container,
     get_offline_dirs,
     get_project,

tests/neptune/new/cli/test_status.py

@@ -17,7 +17,7 @@
 
 import pytest
 
-from neptune.new.cli import StatusRunner
+from neptune.new.cli.status import StatusRunner
 from neptune.new.cli.utils import get_qualified_name
 from neptune.new.internal.container_type import ContainerType
 from neptune.new.internal.operation import Operation

tests/neptune/new/cli/test_sync.py

@@ -18,7 +18,7 @@
 
 import pytest
 
-from neptune.new.cli import SyncRunner
+from neptune.new.cli.sync import SyncRunner
 from neptune.new.cli.utils import get_qualified_name
 from neptune.new.internal.container_type import ContainerType
 from neptune.new.internal.operation import Operation