Skip to content

Commit

Permalink
Changed source files and entry point path resolving behaviour (#1007)
Browse files Browse the repository at this point in the history
  • Loading branch information
Raalsky committed Sep 29, 2022
1 parent 6e56209 commit 3ea7219
Show file tree
Hide file tree
Showing 5 changed files with 193 additions and 26 deletions.
3 changes: 3 additions & 0 deletions CHANGELOG.md
Expand Up @@ -3,6 +3,9 @@
### Fixes
- Update jsonschema requirement with explicit `format` specifier ([#1010](https://github.com/neptune-ai/neptune-client/pull/1010))

### Changes
- More consistent and strict way of git repository, source files and entrypoint detection ([#1007](https://github.com/neptune-ai/neptune-client/pull/1007))

## neptune-client 0.16.9

### Fixes
Expand Down
17 changes: 3 additions & 14 deletions neptune/new/internal/utils/git.py
Expand Up @@ -19,6 +19,7 @@
from typing import Optional

from neptune.new.types.atoms import GitRef
from neptune.vendor.lib_programname import get_path_executed_script

_logger = logging.getLogger(__name__)

Expand Down Expand Up @@ -75,17 +76,5 @@ def get_git_repo_path(initial_path: str) -> Optional[str]:


def discover_git_repo_location() -> Optional[str]:
git_path = None

# pylint:disable=bad-option-value,import-outside-toplevel
import __main__

if hasattr(__main__, "__file__"):
git_path = get_git_repo_path(
initial_path=os.path.dirname(os.path.abspath(__main__.__file__))
)

if not git_path:
git_path = get_git_repo_path(initial_path=os.getcwd())

return git_path
potential_initial_path = os.path.dirname(os.path.abspath(get_path_executed_script()))
return get_git_repo_path(initial_path=potential_initial_path)
16 changes: 9 additions & 7 deletions neptune/new/internal/utils/source_code.py
Expand Up @@ -15,13 +15,13 @@
#
import logging
import os
import sys
from typing import TYPE_CHECKING, List, Optional

from neptune.internal.storage.storage_utils import normalize_file_name
from neptune.new.attributes import constants as attr_consts
from neptune.new.internal.utils import get_absolute_paths, get_common_root
from neptune.utils import is_ipython
from neptune.vendor.lib_programname import empty_path, get_path_executed_script

if TYPE_CHECKING:
from neptune.new import Run
Expand All @@ -31,20 +31,22 @@


def upload_source_code(source_files: Optional[List[str]], run: "Run") -> None:
if not is_ipython() and os.path.isfile(sys.argv[0]):
entry_filepath = get_path_executed_script()

if not is_ipython() and entry_filepath != empty_path and os.path.isfile(entry_filepath):
if source_files is None:
entrypoint = os.path.basename(sys.argv[0])
source_files = sys.argv[0]
entrypoint = os.path.basename(entry_filepath)
source_files = str(entry_filepath)
elif not source_files:
entrypoint = os.path.basename(sys.argv[0])
entrypoint = os.path.basename(entry_filepath)
else:
common_root = get_common_root(get_absolute_paths(source_files))
if common_root is not None:
entrypoint = normalize_file_name(
os.path.relpath(os.path.abspath(sys.argv[0]), common_root)
os.path.relpath(os.path.abspath(entry_filepath), common_root)
)
else:
entrypoint = normalize_file_name(os.path.abspath(sys.argv[0]))
entrypoint = normalize_file_name(os.path.abspath(entry_filepath))
run[attr_consts.SOURCE_CODE_ENTRYPOINT_ATTRIBUTE_PATH] = entrypoint

if source_files is not None:
Expand Down
173 changes: 173 additions & 0 deletions neptune/vendor/lib_programname.py
@@ -0,0 +1,173 @@
# MIT License
#
# Copyright (c) 1990-2022 Robert Nowotny
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

import pathlib
import sys

import __main__ # noqa

__all__ = ["empty_path", "get_path_executed_script"]

empty_path = pathlib.Path()


def get_path_executed_script() -> pathlib.Path:
"""
getting the full path of the program from which a Python module is running
>>> ### TEST get it via __main__.__file__
>>> # Setup
>>> # force __main__.__file__ valid
>>> save_main_file = str(__main__.__file__)
>>> __main__.__file__ = __file__
>>> # Test via __main__.__file__
>>> assert get_path_executed_script() == pathlib.Path(__file__).resolve()
>>> ### TEST get it via sys.argv
>>> # Setup
>>> # force __main__.__file__ invalid
>>> __main__.__file__ = str((pathlib.Path(__file__).parent / 'invalid_file.py')) # .resolve() seems not to work on a non existing file in python 3.5
>>> # force sys.argv valid
>>> save_sys_argv = list(sys.argv)
>>> valid_path = str((pathlib.Path(__file__).resolve()))
>>> sys.argv = [valid_path]
>>> # Test via sys.argv
>>> assert get_path_executed_script() == pathlib.Path(__file__).resolve()
>>> ### TEST get it via stack
>>> # Setup
>>> # force sys.argv invalid
>>> invalid_path = str((pathlib.Path(__file__).parent / 'invalid_file.py')) # .resolve() seems not to work on a non existing file in python 3.5
>>> sys.argv = [invalid_path]
>>> assert get_path_executed_script()
>>> # teardown
>>> __main__.__file__ = save_main_file
>>> sys.argv = list(save_sys_argv)
"""

# try to get it from __main__.__file__ - does not work under pytest, doctest
path_candidate = get_fullpath_from_main_file()
if path_candidate != empty_path:
return path_candidate

# try to get it from sys_argv - does not work when loaded from uwsgi, works in eclipse and pydev
path_candidate = get_fullpath_from_sys_argv()
if path_candidate != empty_path:
return path_candidate

return empty_path


def get_fullpath_from_main_file() -> pathlib.Path:
"""try to get it from __main__.__file__ - does not work under pytest, doctest
>>> # test no attrib __main__.__file__
>>> save_main_file = str(__main__.__file__)
>>> delattr(__main__, '__file__')
>>> assert get_fullpath_from_main_file() == empty_path
>>> setattr(__main__, '__file__', save_main_file)
"""
if not hasattr(sys.modules["__main__"], "__file__"):
return empty_path

arg_string = str(sys.modules["__main__"].__file__)
valid_executable_path = get_valid_executable_path_or_empty_path(arg_string)
return valid_executable_path


def get_fullpath_from_sys_argv() -> pathlib.Path:
"""try to get it from sys_argv - does not work when loaded from uwsgi, works in eclipse and pydev
>>> # force test invalid sys.path
>>> save_sys_argv = list(sys.argv)
>>> invalid_path = str((pathlib.Path(__file__).parent / 'invalid_file.py')) # .resolve() seems not to work on a non existing file in python 3.5
>>> sys.argv = [invalid_path]
>>> assert get_fullpath_from_sys_argv() == pathlib.Path()
>>> sys.argv = list(save_sys_argv)
>>> # force test valid sys.path
>>> save_sys_path = list(sys.argv)
>>> valid_path = str((pathlib.Path(__file__).resolve()))
>>> sys.argv = [valid_path]
>>> assert get_fullpath_from_sys_argv() == pathlib.Path(valid_path)
>>> sys.argv = list(save_sys_argv)
"""

for arg_string in sys.argv:
valid_executable_path = get_valid_executable_path_or_empty_path(arg_string)
if valid_executable_path != empty_path:
return valid_executable_path
return empty_path


def get_valid_executable_path_or_empty_path(arg_string: str) -> pathlib.Path:
arg_string = remove_doctest_and_docrunner_parameters(arg_string)
arg_string = add_python_extension_if_not_there(arg_string)
path = pathlib.Path(arg_string)
if path.is_file():
path = path.resolve() # .resolve does not work on a non existing file in python 3.5
return path
else:
return empty_path


def remove_doctest_and_docrunner_parameters(arg_string: str) -> str:
"""
>>> # Setup
>>> arg_string_with_parameter = __file__ + '::::::some docrunner parameter'
>>> arg_string_without_parameter = __file__
>>> # Test with and without docrunner parameters
>>> assert remove_doctest_and_docrunner_parameters(arg_string_with_parameter) == __file__
>>> assert remove_doctest_and_docrunner_parameters(arg_string_without_parameter) == __file__
"""
path = arg_string.split("::", 1)[0]
return path


def add_python_extension_if_not_there(arg_string: str) -> str:
"""
>>> # Setup
>>> arg_string_with_py = __file__
>>> arg_string_without_py = __file__.rsplit('.py',1)[0]
>>> # Test with and without .py suffix
>>> assert add_python_extension_if_not_there(arg_string_with_py) == __file__
>>> assert add_python_extension_if_not_there(arg_string_without_py) == __file__
"""

if not arg_string.endswith(".py"):
arg_string = arg_string + ".py"
return arg_string
10 changes: 5 additions & 5 deletions tests/neptune/new/client/test_run.py
Expand Up @@ -92,15 +92,15 @@ def test_resume(self):
self.assertEqual(exp._id, AN_API_RUN.id)
self.assertIsInstance(exp.get_structure()["test"], String)

@patch("neptune.new.internal.utils.source_code.sys.argv", ["main.py"])
@patch("neptune.new.internal.utils.source_code.get_path_executed_script", lambda: "main.py")
@patch("neptune.new.internal.init.run.os.path.isfile", new=lambda file: "." in file)
@patch(
"neptune.new.internal.utils.glob",
new=lambda path, recursive=False: [path.replace("*", "file.txt")],
)
@patch(
"neptune.new.internal.utils.os.path.abspath",
new=lambda path: os.path.normpath("/home/user/main_dir/" + path),
new=lambda path: os.path.normpath(os.path.join("/home/user/main_dir", path)),
)
@patch("neptune.new.internal.utils.os.getcwd", new=lambda: "/home/user/main_dir")
@unittest.skipIf(IS_WINDOWS, "Linux/Mac test")
Expand All @@ -120,7 +120,7 @@ def test_entrypoint(self):
exp = init_run(mode="debug", source_files=["../other_dir/*"])
self.assertEqual(exp["source_code/entrypoint"].fetch(), "../main_dir/main.py")

@patch("neptune.new.internal.utils.source_code.sys.argv", ["main.py"])
@patch("neptune.vendor.lib_programname.sys.argv", ["main.py"])
@patch("neptune.new.internal.utils.source_code.is_ipython", new=lambda: True)
def test_entrypoint_in_interactive_python(self):
exp = init_run(mode="debug")
Expand All @@ -139,7 +139,7 @@ def test_entrypoint_in_interactive_python(self):
with self.assertRaises(MissingFieldException):
exp["source_code/entrypoint"].fetch()

@patch("neptune.new.internal.utils.source_code.sys.argv", ["main.py"])
@patch("neptune.new.internal.utils.source_code.get_path_executed_script", lambda: "main.py")
@patch("neptune.new.internal.utils.source_code.get_common_root", new=lambda _: None)
@patch("neptune.new.internal.init.run.os.path.isfile", new=lambda file: "." in file)
@patch(
Expand All @@ -148,7 +148,7 @@ def test_entrypoint_in_interactive_python(self):
)
@patch(
"neptune.new.internal.utils.os.path.abspath",
new=lambda path: os.path.normpath("/home/user/main_dir/" + path),
new=lambda path: os.path.normpath(os.path.join("/home/user/main_dir", path)),
)
def test_entrypoint_without_common_root(self):
exp = init_run(mode="debug", source_files=["../*"])
Expand Down

0 comments on commit 3ea7219

Please sign in to comment.