Changed source files and entry point path resolving behaviour

CHANGELOG.md

@@ -3,6 +3,9 @@
 ### Fixes
 - Update jsonschema requirement with explicit `format` specifier ([#1010](https://github.com/neptune-ai/neptune-client/pull/1010))
 
+### Changes
+- More consistent and strict way of git repository, source files and entrypoint detection ([#1007](https://github.com/neptune-ai/neptune-client/pull/1007))
+
 ## neptune-client 0.16.9
 
 ### Fixes

neptune/new/internal/utils/git.py

@@ -19,6 +19,7 @@
 from typing import Optional
 
 from neptune.new.types.atoms import GitRef
+from neptune.vendor.lib_programname import get_path_executed_script
 
 _logger = logging.getLogger(__name__)
 
@@ -75,17 +76,5 @@ def get_git_repo_path(initial_path: str) -> Optional[str]:
 
 
 def discover_git_repo_location() -> Optional[str]:
-    git_path = None
-
-    # pylint:disable=bad-option-value,import-outside-toplevel
-    import __main__
-
-    if hasattr(__main__, "__file__"):
-        git_path = get_git_repo_path(
-            initial_path=os.path.dirname(os.path.abspath(__main__.__file__))
-        )
-
-    if not git_path:
-        git_path = get_git_repo_path(initial_path=os.getcwd())
-
-    return git_path
+    potential_initial_path = os.path.dirname(os.path.abspath(get_path_executed_script()))
+    return get_git_repo_path(initial_path=potential_initial_path)

neptune/new/internal/utils/source_code.py

@@ -15,13 +15,13 @@
 #
 import logging
 import os
-import sys
 from typing import TYPE_CHECKING, List, Optional
 
 from neptune.internal.storage.storage_utils import normalize_file_name
 from neptune.new.attributes import constants as attr_consts
 from neptune.new.internal.utils import get_absolute_paths, get_common_root
 from neptune.utils import is_ipython
+from neptune.vendor.lib_programname import empty_path, get_path_executed_script
 
 if TYPE_CHECKING:
     from neptune.new import Run
@@ -31,20 +31,22 @@
 
 
 def upload_source_code(source_files: Optional[List[str]], run: "Run") -> None:
-    if not is_ipython() and os.path.isfile(sys.argv[0]):
+    entry_filepath = get_path_executed_script()
+
+    if not is_ipython() and entry_filepath != empty_path and os.path.isfile(entry_filepath):
         if source_files is None:
-            entrypoint = os.path.basename(sys.argv[0])
-            source_files = sys.argv[0]
+            entrypoint = os.path.basename(entry_filepath)
+            source_files = str(entry_filepath)
         elif not source_files:
-            entrypoint = os.path.basename(sys.argv[0])
+            entrypoint = os.path.basename(entry_filepath)
         else:
             common_root = get_common_root(get_absolute_paths(source_files))
             if common_root is not None:
                 entrypoint = normalize_file_name(
-                    os.path.relpath(os.path.abspath(sys.argv[0]), common_root)
+                    os.path.relpath(os.path.abspath(entry_filepath), common_root)
                 )
             else:
-                entrypoint = normalize_file_name(os.path.abspath(sys.argv[0]))
+                entrypoint = normalize_file_name(os.path.abspath(entry_filepath))
         run[attr_consts.SOURCE_CODE_ENTRYPOINT_ATTRIBUTE_PATH] = entrypoint
 
     if source_files is not None:

neptune/vendor/lib_programname.py

@@ -0,0 +1,173 @@
+# MIT License
+#
+# Copyright (c) 1990-2022 Robert Nowotny
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+import pathlib
+import sys
+
+import __main__  # noqa
+
+__all__ = ["empty_path", "get_path_executed_script"]
+
+empty_path = pathlib.Path()
+
+
+def get_path_executed_script() -> pathlib.Path:
+    """
+    getting the full path of the program from which a Python module is running
+
+    >>> ### TEST get it via __main__.__file__
+    >>> # Setup
+    >>> # force __main__.__file__ valid
+    >>> save_main_file = str(__main__.__file__)
+    >>> __main__.__file__ = __file__
+
+    >>> # Test via __main__.__file__
+    >>> assert get_path_executed_script() == pathlib.Path(__file__).resolve()
+
+
+    >>> ### TEST get it via sys.argv
+    >>> # Setup
+    >>> # force __main__.__file__ invalid
+    >>> __main__.__file__ = str((pathlib.Path(__file__).parent / 'invalid_file.py'))  # .resolve() seems not to work on a non existing file in python 3.5
+
+    >>> # force sys.argv valid
+    >>> save_sys_argv = list(sys.argv)
+    >>> valid_path = str((pathlib.Path(__file__).resolve()))
+    >>> sys.argv = [valid_path]
+
+    >>> # Test via sys.argv
+    >>> assert get_path_executed_script() == pathlib.Path(__file__).resolve()
+
+
+    >>> ### TEST get it via stack
+    >>> # Setup
+    >>> # force sys.argv invalid
+    >>> invalid_path = str((pathlib.Path(__file__).parent / 'invalid_file.py'))  # .resolve() seems not to work on a non existing file in python 3.5
+    >>> sys.argv = [invalid_path]
+
+
+    >>> assert get_path_executed_script()
+
+    >>> # teardown
+    >>> __main__.__file__ = save_main_file
+    >>> sys.argv = list(save_sys_argv)
+
+    """
+
+    # try to get it from __main__.__file__ - does not work under pytest, doctest
+    path_candidate = get_fullpath_from_main_file()
+    if path_candidate != empty_path:
+        return path_candidate
+
+    # try to get it from sys_argv - does not work when loaded from uwsgi, works in eclipse and pydev
+    path_candidate = get_fullpath_from_sys_argv()
+    if path_candidate != empty_path:
+        return path_candidate
+
+    return empty_path
+
+
+def get_fullpath_from_main_file() -> pathlib.Path:
+    """try to get it from __main__.__file__ - does not work under pytest, doctest
+
+    >>> # test no attrib __main__.__file__
+    >>> save_main_file = str(__main__.__file__)
+    >>> delattr(__main__, '__file__')
+    >>> assert get_fullpath_from_main_file() == empty_path
+    >>> setattr(__main__, '__file__', save_main_file)
+
+    """
+    if not hasattr(sys.modules["__main__"], "__file__"):
+        return empty_path
+
+    arg_string = str(sys.modules["__main__"].__file__)
+    valid_executable_path = get_valid_executable_path_or_empty_path(arg_string)
+    return valid_executable_path
+
+
+def get_fullpath_from_sys_argv() -> pathlib.Path:
+    """try to get it from sys_argv - does not work when loaded from uwsgi, works in eclipse and pydev
+
+    >>> # force test invalid sys.path
+    >>> save_sys_argv = list(sys.argv)
+    >>> invalid_path = str((pathlib.Path(__file__).parent / 'invalid_file.py'))  # .resolve() seems not to work on a non existing file in python 3.5
+    >>> sys.argv = [invalid_path]
+    >>> assert get_fullpath_from_sys_argv() == pathlib.Path()
+    >>> sys.argv = list(save_sys_argv)
+
+    >>> # force test valid sys.path
+    >>> save_sys_path = list(sys.argv)
+    >>> valid_path = str((pathlib.Path(__file__).resolve()))
+    >>> sys.argv = [valid_path]
+    >>> assert get_fullpath_from_sys_argv() == pathlib.Path(valid_path)
+    >>> sys.argv = list(save_sys_argv)
+
+
+    """
+
+    for arg_string in sys.argv:
+        valid_executable_path = get_valid_executable_path_or_empty_path(arg_string)
+        if valid_executable_path != empty_path:
+            return valid_executable_path
+    return empty_path
+
+
+def get_valid_executable_path_or_empty_path(arg_string: str) -> pathlib.Path:
+    arg_string = remove_doctest_and_docrunner_parameters(arg_string)
+    arg_string = add_python_extension_if_not_there(arg_string)
+    path = pathlib.Path(arg_string)
+    if path.is_file():
+        path = path.resolve()  # .resolve does not work on a non existing file in python 3.5
+        return path
+    else:
+        return empty_path
+
+
+def remove_doctest_and_docrunner_parameters(arg_string: str) -> str:
+    """
+    >>> # Setup
+    >>> arg_string_with_parameter = __file__ + '::::::some docrunner parameter'
+    >>> arg_string_without_parameter = __file__
+
+    >>> # Test with and without docrunner parameters
+    >>> assert remove_doctest_and_docrunner_parameters(arg_string_with_parameter) == __file__
+    >>> assert remove_doctest_and_docrunner_parameters(arg_string_without_parameter) == __file__
+    """
+    path = arg_string.split("::", 1)[0]
+    return path
+
+
+def add_python_extension_if_not_there(arg_string: str) -> str:
+    """
+    >>> # Setup
+    >>> arg_string_with_py = __file__
+    >>> arg_string_without_py = __file__.rsplit('.py',1)[0]
+
+    >>> # Test with and without .py suffix
+    >>> assert add_python_extension_if_not_there(arg_string_with_py) == __file__
+    >>> assert add_python_extension_if_not_there(arg_string_without_py) == __file__
+
+    """
+
+    if not arg_string.endswith(".py"):
+        arg_string = arg_string + ".py"
+    return arg_string

tests/neptune/new/client/test_run.py

@@ -92,15 +92,15 @@ def test_resume(self):
             self.assertEqual(exp._id, AN_API_RUN.id)
             self.assertIsInstance(exp.get_structure()["test"], String)
 
-    @patch("neptune.new.internal.utils.source_code.sys.argv", ["main.py"])
+    @patch("neptune.new.internal.utils.source_code.get_path_executed_script", lambda: "main.py")
     @patch("neptune.new.internal.init.run.os.path.isfile", new=lambda file: "." in file)
     @patch(
         "neptune.new.internal.utils.glob",
         new=lambda path, recursive=False: [path.replace("*", "file.txt")],
     )
     @patch(
         "neptune.new.internal.utils.os.path.abspath",
-        new=lambda path: os.path.normpath("/home/user/main_dir/" + path),
+        new=lambda path: os.path.normpath(os.path.join("/home/user/main_dir", path)),
     )
     @patch("neptune.new.internal.utils.os.getcwd", new=lambda: "/home/user/main_dir")
     @unittest.skipIf(IS_WINDOWS, "Linux/Mac test")
@@ -120,7 +120,7 @@ def test_entrypoint(self):
         exp = init_run(mode="debug", source_files=["../other_dir/*"])
         self.assertEqual(exp["source_code/entrypoint"].fetch(), "../main_dir/main.py")
 
-    @patch("neptune.new.internal.utils.source_code.sys.argv", ["main.py"])
+    @patch("neptune.vendor.lib_programname.sys.argv", ["main.py"])
     @patch("neptune.new.internal.utils.source_code.is_ipython", new=lambda: True)
     def test_entrypoint_in_interactive_python(self):
         exp = init_run(mode="debug")
@@ -139,7 +139,7 @@ def test_entrypoint_in_interactive_python(self):
         with self.assertRaises(MissingFieldException):
             exp["source_code/entrypoint"].fetch()
 
-    @patch("neptune.new.internal.utils.source_code.sys.argv", ["main.py"])
+    @patch("neptune.new.internal.utils.source_code.get_path_executed_script", lambda: "main.py")
     @patch("neptune.new.internal.utils.source_code.get_common_root", new=lambda _: None)
     @patch("neptune.new.internal.init.run.os.path.isfile", new=lambda file: "." in file)
     @patch(
@@ -148,7 +148,7 @@ def test_entrypoint_in_interactive_python(self):
     )
     @patch(
         "neptune.new.internal.utils.os.path.abspath",
-        new=lambda path: os.path.normpath("/home/user/main_dir/" + path),
+        new=lambda path: os.path.normpath(os.path.join("/home/user/main_dir", path)),
     )
     def test_entrypoint_without_common_root(self):
         exp = init_run(mode="debug", source_files=["../*"])