[Projects] Auto save in new_project() (mlrun#2031)

mlrun/projects/project.py

@@ -96,6 +96,7 @@ def new_project(
     secrets: dict = None,
     description: str = None,
     subpath: str = None,
+    skip_save: bool = False,
 ) -> "MlrunProject":
     """Create a new MLRun project, optionally load it from a yaml/zip/git template
 
@@ -131,6 +132,7 @@ def new_project(
     :param secrets:      key:secret dict or SecretsStore used to download sources
     :param description:  text describing the project
     :param subpath:      project subpath (relative to the context dir)
+    :param skip_save:    do not save the created project in the DB
 
     :returns: project object
     """
@@ -166,6 +168,8 @@ def new_project(
         project.spec.description = description
     mlrun.mlconf.default_project = project.metadata.name
     pipeline_context.set(project)
+    if not skip_save and mlrun.mlconf.dbpath:
+        project.save()
     return project
 
 
@@ -1982,6 +1986,7 @@ def clear_context(self):
     def save(self, filepath=None):
         self.export(filepath)
         self.save_to_db()
+        return self
 
     def save_to_db(self):
         db = mlrun.db.get_run_db(secrets=self._secrets)

tests/artifacts/test_artifacts.py

@@ -120,8 +120,8 @@ def test_generate_target_path():
 
 
 def test_export_import():
-    project = mlrun.new_project("log-mod")
-    target_project = mlrun.new_project("log-mod2")
+    project = mlrun.new_project("log-mod", skip_save=True)
+    target_project = mlrun.new_project("log-mod2", skip_save=True)
     model = project.log_model(
         "mymod",
         body=b"123",
@@ -149,7 +149,7 @@ def test_export_import():
 
 
 def test_inline_body():
-    project = mlrun.new_project("inline")
+    project = mlrun.new_project("inline", skip_save=True)
 
     # log an artifact and save the content/body in the object (inline)
     artifact = project.log_artifact(

tests/artifacts/test_model.py

@@ -40,7 +40,7 @@ def test_model_update():
     model = ModelArtifact("my-model", model_file="a.pkl")
     target_path = results_dir + "model/"
 
-    project = mlrun.new_project("test-proj")
+    project = mlrun.new_project("test-proj", skip_save=True)
     artifact = project.log_artifact(model, upload=False, artifact_path=target_path)
 
     artifact_uri = f"store://artifacts/{artifact.project}/{artifact.db_key}"

tests/projects/base_pipeline.py

@@ -20,7 +20,7 @@ def _create_project(
         project_name,
     ):
         self.project = mlrun.new_project(
-            project_name, f"{self.project_dir}/{project_name}"
+            project_name, f"{self.project_dir}/{project_name}", skip_save=True
         )
         self.project.set_artifact("data", target_path=self.data_url)
         self.project.spec.params = {"label_column": "label"}

tests/projects/test_local_pipeline.py

@@ -20,7 +20,7 @@ def _set_functions(self):
         )
 
     def test_set_artifact(self):
-        self.project = mlrun.new_project("test-sa")
+        self.project = mlrun.new_project("test-sa", skip_save=True)
         self.project.set_artifact(
             "data1", mlrun.artifacts.Artifact(target_path=self.data_url)
         )
@@ -38,7 +38,9 @@ def test_set_artifact(self):
 
     def test_import_artifacts(self):
         results_path = str(pathlib.Path(tests.conftest.results) / "project")
-        project = mlrun.new_project("test-sa2", context=str(self.assets_path))
+        project = mlrun.new_project(
+            "test-sa2", context=str(self.assets_path), skip_save=True
+        )
         project.spec.artifact_path = results_path
         # use inline body (in the yaml)
         project.set_artifact("y", "artifact.yaml")

tests/projects/test_project.py

@@ -14,7 +14,7 @@
 
 def test_sync_functions():
     project_name = "project-name"
-    project = mlrun.new_project(project_name)
+    project = mlrun.new_project(project_name, skip_save=True)
     project.set_function("hub://describe", "describe")
     project_function_object = project.spec._function_objects
     project_file_path = pathlib.Path(tests.conftest.results) / "project.yaml"
@@ -122,7 +122,7 @@ def test_export_project_dir_doesnt_exist():
         / "another-new-dir"
         / "project.yaml"
     )
-    project = mlrun.projects.project.new_project(project_name)
+    project = mlrun.projects.project.new_project(project_name, skip_save=True)
     project.export(filepath=project_file_path)
 
 
@@ -131,18 +131,18 @@ def test_new_project_context_doesnt_exist():
     project_dir_path = (
         pathlib.Path(tests.conftest.results) / "new-dir" / "another-new-dir"
     )
-    mlrun.projects.project.new_project(project_name, project_dir_path)
+    mlrun.projects.project.new_project(project_name, project_dir_path, skip_save=True)
 
 
 def test_create_project_with_invalid_name():
     invalid_name = "project_name"
     with pytest.raises(mlrun.errors.MLRunInvalidArgumentError):
-        mlrun.projects.project.new_project(invalid_name, init_git=False)
+        mlrun.projects.project.new_project(invalid_name, init_git=False, skip_save=True)
 
 
 def test_get_set_params():
     project_name = "project-name"
-    project = mlrun.new_project(project_name)
+    project = mlrun.new_project(project_name, skip_save=True)
     param_key = "param-key"
     param_value = "param-value"
     project.params[param_key] = param_value
@@ -157,7 +157,7 @@ def test_user_project():
     usernames = ["valid-username", "require_Normalization"]
     for username in usernames:
         os.environ["V3IO_USERNAME"] = username
-        project = mlrun.new_project(project_name, user_project=True)
+        project = mlrun.new_project(project_name, user_project=True, skip_save=True)
         assert (
             project.metadata.name
             == f"{project_name}-{inflection.dasherize(username.lower())}"
@@ -226,7 +226,7 @@ def test_function_run_cli():
     # run function stored in the project spec
     project_dir_path = pathlib.Path(tests.conftest.results) / "project-run-func"
     function_path = pathlib.Path(__file__).parent / "assets" / "handler.py"
-    project = mlrun.new_project("run-cli", str(project_dir_path))
+    project = mlrun.new_project("run-cli", str(project_dir_path), skip_save=True)
     project.set_function(
         str(function_path),
         "my-func",
@@ -242,7 +242,7 @@ def test_function_run_cli():
 
 
 def test_get_artifact_uri():
-    project = mlrun.new_project("arti")
+    project = mlrun.new_project("arti", skip_save=True)
     uri = project.get_artifact_uri("x")
     assert uri == "store://artifacts/arti/x"
     uri = project.get_artifact_uri("y", category="model", tag="prod")
@@ -251,7 +251,9 @@ def test_get_artifact_uri():
 
 def test_export_to_zip():
     project_dir_path = pathlib.Path(tests.conftest.results) / "zip-project"
-    project = mlrun.new_project("tozip", context=str(project_dir_path / "code"))
+    project = mlrun.new_project(
+        "tozip", context=str(project_dir_path / "code"), skip_save=True
+    )
     project.set_function("hub://describe", "desc")
     with (project_dir_path / "code" / "f.py").open("w") as f:
         f.write("print(1)\n")

tests/projects/test_project_create.py

@@ -20,14 +20,20 @@ def setup_method(self, method):
 
     def test_yaml_template(self):
         project = mlrun.new_project(
-            "newproj", "./", from_template=str(self.assets_path / "project.yaml")
+            "newproj",
+            "./",
+            from_template=str(self.assets_path / "project.yaml"),
+            skip_save=True,
         )
         assert project.spec.description == "test", "failed to load yaml template"
 
     def test_zip_template(self):
         shutil.rmtree(project_dir, ignore_errors=True)
         project = mlrun.new_project(
-            "newproj2", project_dir, from_template=str(self.assets_path / "project.zip")
+            "newproj2",
+            project_dir,
+            from_template=str(self.assets_path / "project.zip"),
+            skip_save=True,
         )
         assert project.spec.description == "test", "failed to load yaml template"
 
@@ -41,6 +47,7 @@ def test_git_template(self):
             "newproj3",
             project_dir,
             from_template="git://github.com/mlrun/project-demo.git",
+            skip_save=True,
         )
         assert project.spec.description == "test", "failed to load yaml template"
 

tests/rundb/test_httpdb.py

@@ -187,7 +187,7 @@ def test_log(create_server):
     server: Server = create_server()
     db = server.conn
     prj, uid, body = "p19", "3920", b"log data"
-    proj_obj = mlrun.new_project(prj)
+    proj_obj = mlrun.new_project(prj, skip_save=True)
     db.create_project(proj_obj)
 
     db.store_run({"metadata": {"name": "run-name"}, "asd": "asd"}, uid, prj)
@@ -201,7 +201,7 @@ def test_run(create_server):
     server: Server = create_server()
     db = server.conn
     prj, uid = "p18", "3i920"
-    proj_obj = mlrun.new_project(prj)
+    proj_obj = mlrun.new_project(prj, skip_save=True)
     db.create_project(proj_obj)
 
     run_as_dict = RunObject().to_dict()
@@ -235,7 +235,7 @@ def test_runs(create_server):
     server: Server = create_server()
     db = server.conn
     prj = "p180"
-    proj_obj = mlrun.new_project(prj)
+    proj_obj = mlrun.new_project(prj, skip_save=True)
     db.create_project(proj_obj)
 
     runs = db.list_runs()
@@ -298,7 +298,7 @@ def test_set_get_function(create_server):
 
     func, name, proj = {"x": 1, "y": 2}, "f1", "p2"
     tag = uuid4().hex
-    proj_obj = mlrun.new_project(proj)
+    proj_obj = mlrun.new_project(proj, skip_save=True)
     db.create_project(proj_obj)
 
     db.store_function(func, name, proj, tag=tag)
@@ -315,7 +315,7 @@ def test_list_functions(create_server):
     db: HTTPRunDB = server.conn
 
     proj = "p4"
-    proj_obj = mlrun.new_project(proj)
+    proj_obj = mlrun.new_project(proj, skip_save=True)
     db.create_project(proj_obj)
 
     count = 5
@@ -325,7 +325,7 @@ def test_list_functions(create_server):
         tag = uuid4().hex
         db.store_function(func, name, proj, tag=tag)
     proj_p7 = "p7"
-    proj_p7_obj = mlrun.new_project(proj_p7)
+    proj_p7_obj = mlrun.new_project(proj_p7, skip_save=True)
     db.create_project(proj_p7_obj)
 
     db.store_function({}, "f2", proj_p7, tag=uuid4().hex)
@@ -410,7 +410,7 @@ def test_feature_sets(create_server):
     db: HTTPRunDB = server.conn
 
     project = "newproj"
-    proj_obj = mlrun.new_project(project)
+    proj_obj = mlrun.new_project(project, skip_save=True)
     db.create_project(proj_obj)
 
     count = 5
@@ -499,7 +499,7 @@ def test_feature_vectors(create_server):
     db: HTTPRunDB = server.conn
 
     project = "newproj"
-    proj_obj = mlrun.new_project(project)
+    proj_obj = mlrun.new_project(project, skip_save=True)
     db.create_project(proj_obj)
 
     count = 5

tests/serving/test_serving.py

@@ -295,7 +295,7 @@ def test_v2_explain():
 
 
 def test_v2_get_modelmeta():
-    project = mlrun.new_project("tstsrv")
+    project = mlrun.new_project("tstsrv", skip_save=True)
     fn = mlrun.new_function("tst", kind="serving")
     model_uri = _log_model(project)
     print(model_uri)