From 69a26b896882367a55efe4e9c64f58e508f3156c Mon Sep 17 00:00:00 2001 From: Ben Wilson Date: Wed, 10 Nov 2021 14:19:38 -0500 Subject: [PATCH 01/18] Add --serve-artifacts-opt and --artifacts-only options to mlflow server Signed-off-by: Ben Wilson --- mlflow/cli.py | 23 ++++++++++ mlflow/server/__init__.py | 8 ++++ mlflow/server/handlers.py | 92 +++++++++++++++++++++++++++++++++++++++ tests/test_cli.py | 60 ++++++++++++++++++++++++- 4 files changed, 182 insertions(+), 1 deletion(-) diff --git a/mlflow/cli.py b/mlflow/cli.py index d91793629a0af..63d87e65e6af3 100644 --- a/mlflow/cli.py +++ b/mlflow/cli.py @@ -320,6 +320,25 @@ def _validate_static_prefix(ctx, param, value): # pylint: disable=unused-argume "Default: Within file store, if a file:/ URI is provided. If a sql backend is" " used, then this option is required.", ) +@click.option( + "--serve-artifacts-opt", + is_flag=True, + help="If specified, enables serving of artifact uploads, downloads, and list requests " + "by routing these requests to the storage location that is specified by " + "'--artifact-destination' directly through a proxy. The default location that " + "these requests are served from is a local './mlartifacts' directory which can be " + "overridden via '--artifact-destination' arguments. " + "Default: False", +) +@click.option( + "--artifacts-only", + is_flag=True, + help="If specified, configures the mlflow server to be used only for proxied artifact serving. " + "With this mode enabled, functionality of the mlflow tracking service (e.g. run creation, " + "metric logging, and parameter logging are disabled. The server will only expose " + "endpoints for uploading, downloading, and listing artifacts. " + "Default: False", +) @cli_args.ARTIFACTS_DESTINATION @cli_args.HOST @cli_args.PORT @@ -348,6 +367,8 @@ def _validate_static_prefix(ctx, param, value): # pylint: disable=unused-argume def server( backend_store_uri, default_artifact_root, + serve_artifacts_opt, + artifacts_only, artifacts_destination, host, port, @@ -395,6 +416,8 @@ def server( _run_server( backend_store_uri, default_artifact_root, + serve_artifacts_opt, + artifacts_only, artifacts_destination, host, port, diff --git a/mlflow/server/__init__.py b/mlflow/server/__init__.py index deecd40f12e46..c82311eb034eb 100644 --- a/mlflow/server/__init__.py +++ b/mlflow/server/__init__.py @@ -20,6 +20,8 @@ ARTIFACT_ROOT_ENV_VAR = "_MLFLOW_SERVER_ARTIFACT_ROOT" ARTIFACTS_DESTINATION_ENV_VAR = "_MLFLOW_SERVER_ARTIFACT_DESTINATION" PROMETHEUS_EXPORTER_ENV_VAR = "prometheus_multiproc_dir" +SERVE_ARTIFACTS_ENV_VAR = "_MLFLOW_SERVER_SERVE_ARTIFACTS" +ARTIFACTS_ONLY_ENV_VAR = "_MLFLOW_SERVER_ARTIFACTS_ONLY" REL_STATIC_DIR = "js/build" @@ -106,6 +108,8 @@ def _build_gunicorn_command(gunicorn_opts, host, port, workers): def _run_server( file_store_path, default_artifact_root, + serve_artifacts_opt, + artifacts_only, artifacts_destination, host, port, @@ -126,6 +130,10 @@ def _run_server( env_map[BACKEND_STORE_URI_ENV_VAR] = file_store_path if default_artifact_root: env_map[ARTIFACT_ROOT_ENV_VAR] = default_artifact_root + if serve_artifacts_opt: + env_map[SERVE_ARTIFACTS_ENV_VAR] = "true" + if artifacts_only: + env_map[ARTIFACTS_ONLY_ENV_VAR] = "true" if artifacts_destination: env_map[ARTIFACTS_DESTINATION_ENV_VAR] = artifacts_destination if static_prefix: diff --git a/mlflow/server/handlers.py b/mlflow/server/handlers.py index e3a3c2fbd4add..92363eb1bce8f 100644 --- a/mlflow/server/handlers.py +++ b/mlflow/server/handlers.py @@ -262,6 +262,43 @@ def wrapper(*args, **kwargs): ] +def _disable_mlflow_artifacts_endpoint(func): + @wraps(func) + def wrapper(*args, **kwargs): + from mlflow.server import SERVE_ARTIFACTS_ENV_VAR + + if not os.environ.get(SERVE_ARTIFACTS_ENV_VAR): + return Response( + ( + "Endpoints for mlflow artifacts service are disabled. To enable them, " + "run `mlflow server` with `--serve-artfiacts-opt`" + ), + 503, + ) + return func(*args, **kwargs) + + return wrapper + + +def _disable_mlflow_artifacts_only(func): + @wraps(func) + def wrapper(*args, **kwargs): + from mlflow.server import ARTIFACTS_ONLY_ENV_VAR + + if os.environ.get(ARTIFACTS_ONLY_ENV_VAR): + return Response( + ( + "Endpoints disabled due to the mlflow server running in `--artifacts-only` " + "mode. To enable tracking server functionality, run `mlflow server` without " + "`--artifacts-only`" + ), + 503, + ) + return func(*args, **kwargs) + + return wrapper + + @catch_mlflow_exception def get_artifact_handler(): from querystring_parser import parser @@ -279,7 +316,11 @@ def _not_implemented(): return response +# Tracking Server APIs + + @catch_mlflow_exception +@_disable_mlflow_artifacts_only def _create_experiment(): request_message = _get_request_message(CreateExperiment()) tags = [ExperimentTag(tag.key, tag.value) for tag in request_message.tags] @@ -294,6 +335,7 @@ def _create_experiment(): @catch_mlflow_exception +@_disable_mlflow_artifacts_only def _get_experiment(): request_message = _get_request_message(GetExperiment()) response_message = GetExperiment.Response() @@ -305,6 +347,7 @@ def _get_experiment(): @catch_mlflow_exception +@_disable_mlflow_artifacts_only def _get_experiment_by_name(): request_message = _get_request_message(GetExperimentByName()) response_message = GetExperimentByName.Response() @@ -322,6 +365,7 @@ def _get_experiment_by_name(): @catch_mlflow_exception +@_disable_mlflow_artifacts_only def _delete_experiment(): request_message = _get_request_message(DeleteExperiment()) _get_tracking_store().delete_experiment(request_message.experiment_id) @@ -332,6 +376,7 @@ def _delete_experiment(): @catch_mlflow_exception +@_disable_mlflow_artifacts_only def _restore_experiment(): request_message = _get_request_message(RestoreExperiment()) _get_tracking_store().restore_experiment(request_message.experiment_id) @@ -342,6 +387,7 @@ def _restore_experiment(): @catch_mlflow_exception +@_disable_mlflow_artifacts_only def _update_experiment(): request_message = _get_request_message(UpdateExperiment()) if request_message.new_name: @@ -355,6 +401,7 @@ def _update_experiment(): @catch_mlflow_exception +@_disable_mlflow_artifacts_only def _create_run(): request_message = _get_request_message(CreateRun()) @@ -374,6 +421,7 @@ def _create_run(): @catch_mlflow_exception +@_disable_mlflow_artifacts_only def _update_run(): request_message = _get_request_message(UpdateRun()) run_id = request_message.run_id or request_message.run_uuid @@ -387,6 +435,7 @@ def _update_run(): @catch_mlflow_exception +@_disable_mlflow_artifacts_only def _delete_run(): request_message = _get_request_message(DeleteRun()) _get_tracking_store().delete_run(request_message.run_id) @@ -397,6 +446,7 @@ def _delete_run(): @catch_mlflow_exception +@_disable_mlflow_artifacts_only def _restore_run(): request_message = _get_request_message(RestoreRun()) _get_tracking_store().restore_run(request_message.run_id) @@ -407,6 +457,7 @@ def _restore_run(): @catch_mlflow_exception +@_disable_mlflow_artifacts_only def _log_metric(): request_message = _get_request_message(LogMetric()) metric = Metric( @@ -421,6 +472,7 @@ def _log_metric(): @catch_mlflow_exception +@_disable_mlflow_artifacts_only def _log_param(): request_message = _get_request_message(LogParam()) param = Param(request_message.key, request_message.value) @@ -433,6 +485,7 @@ def _log_param(): @catch_mlflow_exception +@_disable_mlflow_artifacts_only def _set_experiment_tag(): request_message = _get_request_message(SetExperimentTag()) tag = ExperimentTag(request_message.key, request_message.value) @@ -444,6 +497,7 @@ def _set_experiment_tag(): @catch_mlflow_exception +@_disable_mlflow_artifacts_only def _set_tag(): request_message = _get_request_message(SetTag()) tag = RunTag(request_message.key, request_message.value) @@ -456,6 +510,7 @@ def _set_tag(): @catch_mlflow_exception +@_disable_mlflow_artifacts_only def _delete_tag(): request_message = _get_request_message(DeleteTag()) _get_tracking_store().delete_tag(request_message.run_id, request_message.key) @@ -466,6 +521,7 @@ def _delete_tag(): @catch_mlflow_exception +@_disable_mlflow_artifacts_only def _get_run(): request_message = _get_request_message(GetRun()) response_message = GetRun.Response() @@ -477,6 +533,7 @@ def _get_run(): @catch_mlflow_exception +@_disable_mlflow_artifacts_only def _search_runs(): request_message = _get_request_message(SearchRuns()) response_message = SearchRuns.Response() @@ -500,6 +557,7 @@ def _search_runs(): @catch_mlflow_exception +@_disable_mlflow_artifacts_only def _list_artifacts(): request_message = _get_request_message(ListArtifacts()) response_message = ListArtifacts.Response() @@ -518,6 +576,7 @@ def _list_artifacts(): @catch_mlflow_exception +@_disable_mlflow_artifacts_only def _get_metric_history(): request_message = _get_request_message(GetMetricHistory()) response_message = GetMetricHistory.Response() @@ -530,6 +589,7 @@ def _get_metric_history(): @catch_mlflow_exception +@_disable_mlflow_artifacts_only def _list_experiments(): request_message = _get_request_message(ListExperiments()) # `ListFields` returns a list of (FieldDescriptor, value) tuples for *present* fields: @@ -547,11 +607,13 @@ def _list_experiments(): @catch_mlflow_exception +@_disable_mlflow_artifacts_only def _get_artifact_repo(run): return get_artifact_repository(run.info.artifact_uri) @catch_mlflow_exception +@_disable_mlflow_artifacts_only def _log_batch(): _validate_batch_log_api_req(_get_request_json()) request_message = _get_request_message(LogBatch()) @@ -568,6 +630,7 @@ def _log_batch(): @catch_mlflow_exception +@_disable_mlflow_artifacts_only def _log_model(): request_message = _get_request_message(LogModel()) try: @@ -603,7 +666,11 @@ def _wrap_response(response_message): return response +# Model Registry APIs + + @catch_mlflow_exception +@_disable_mlflow_artifacts_only def _create_registered_model(): request_message = _get_request_message(CreateRegisteredModel()) registered_model = _get_model_registry_store().create_registered_model( @@ -616,6 +683,7 @@ def _create_registered_model(): @catch_mlflow_exception +@_disable_mlflow_artifacts_only def _get_registered_model(): request_message = _get_request_message(GetRegisteredModel()) registered_model = _get_model_registry_store().get_registered_model(name=request_message.name) @@ -624,6 +692,7 @@ def _get_registered_model(): @catch_mlflow_exception +@_disable_mlflow_artifacts_only def _update_registered_model(): request_message = _get_request_message(UpdateRegisteredModel()) name = request_message.name @@ -636,6 +705,7 @@ def _update_registered_model(): @catch_mlflow_exception +@_disable_mlflow_artifacts_only def _rename_registered_model(): request_message = _get_request_message(RenameRegisteredModel()) name = request_message.name @@ -648,6 +718,7 @@ def _rename_registered_model(): @catch_mlflow_exception +@_disable_mlflow_artifacts_only def _delete_registered_model(): request_message = _get_request_message(DeleteRegisteredModel()) _get_model_registry_store().delete_registered_model(name=request_message.name) @@ -655,6 +726,7 @@ def _delete_registered_model(): @catch_mlflow_exception +@_disable_mlflow_artifacts_only def _list_registered_models(): request_message = _get_request_message(ListRegisteredModels()) registered_models = _get_model_registry_store().list_registered_models( @@ -668,6 +740,7 @@ def _list_registered_models(): @catch_mlflow_exception +@_disable_mlflow_artifacts_only def _search_registered_models(): request_message = _get_request_message(SearchRegisteredModels()) store = _get_model_registry_store() @@ -685,6 +758,7 @@ def _search_registered_models(): @catch_mlflow_exception +@_disable_mlflow_artifacts_only def _get_latest_versions(): request_message = _get_request_message(GetLatestVersions()) latest_versions = _get_model_registry_store().get_latest_versions( @@ -696,6 +770,7 @@ def _get_latest_versions(): @catch_mlflow_exception +@_disable_mlflow_artifacts_only def _set_registered_model_tag(): request_message = _get_request_message(SetRegisteredModelTag()) tag = RegisteredModelTag(key=request_message.key, value=request_message.value) @@ -704,6 +779,7 @@ def _set_registered_model_tag(): @catch_mlflow_exception +@_disable_mlflow_artifacts_only def _delete_registered_model_tag(): request_message = _get_request_message(DeleteRegisteredModelTag()) _get_model_registry_store().delete_registered_model_tag( @@ -713,6 +789,7 @@ def _delete_registered_model_tag(): @catch_mlflow_exception +@_disable_mlflow_artifacts_only def _create_model_version(): request_message = _get_request_message(CreateModelVersion()) model_version = _get_model_registry_store().create_model_version( @@ -728,6 +805,7 @@ def _create_model_version(): @catch_mlflow_exception +@_disable_mlflow_artifacts_only def get_model_version_artifact_handler(): from querystring_parser import parser @@ -740,6 +818,7 @@ def get_model_version_artifact_handler(): @catch_mlflow_exception +@_disable_mlflow_artifacts_only def _get_model_version(): request_message = _get_request_message(GetModelVersion()) model_version = _get_model_registry_store().get_model_version( @@ -751,6 +830,7 @@ def _get_model_version(): @catch_mlflow_exception +@_disable_mlflow_artifacts_only def _update_model_version(): request_message = _get_request_message(UpdateModelVersion()) new_description = None @@ -763,6 +843,7 @@ def _update_model_version(): @catch_mlflow_exception +@_disable_mlflow_artifacts_only def _transition_stage(): request_message = _get_request_message(TransitionModelVersionStage()) model_version = _get_model_registry_store().transition_model_version_stage( @@ -777,6 +858,7 @@ def _transition_stage(): @catch_mlflow_exception +@_disable_mlflow_artifacts_only def _delete_model_version(): request_message = _get_request_message(DeleteModelVersion()) _get_model_registry_store().delete_model_version( @@ -786,6 +868,7 @@ def _delete_model_version(): @catch_mlflow_exception +@_disable_mlflow_artifacts_only def _get_model_version_download_uri(): request_message = _get_request_message(GetModelVersionDownloadUri()) download_uri = _get_model_registry_store().get_model_version_download_uri( @@ -796,6 +879,7 @@ def _get_model_version_download_uri(): @catch_mlflow_exception +@_disable_mlflow_artifacts_only def _search_model_versions(): request_message = _get_request_message(SearchModelVersions()) model_versions = _get_model_registry_store().search_model_versions(request_message.filter) @@ -805,6 +889,7 @@ def _search_model_versions(): @catch_mlflow_exception +@_disable_mlflow_artifacts_only def _set_model_version_tag(): request_message = _get_request_message(SetModelVersionTag()) tag = ModelVersionTag(key=request_message.key, value=request_message.value) @@ -815,6 +900,7 @@ def _set_model_version_tag(): @catch_mlflow_exception +@_disable_mlflow_artifacts_only def _delete_model_version_tag(): request_message = _get_request_message(DeleteModelVersionTag()) _get_model_registry_store().delete_model_version_tag( @@ -823,7 +909,11 @@ def _delete_model_version_tag(): return _wrap_response(DeleteModelVersionTag.Response()) +# MLflow Artifacts APIs + + @catch_mlflow_exception +@_disable_mlflow_artifacts_endpoint def _download_artifact(artifact_path): """ A request handler for `GET /mlflow-artifacts/artifacts/` to download an artifact @@ -850,6 +940,7 @@ def stream_and_remove_file(): @catch_mlflow_exception +@_disable_mlflow_artifacts_endpoint def _upload_artifact(artifact_path): """ A request handler for `PUT /mlflow-artifacts/artifacts/` to upload an artifact @@ -873,6 +964,7 @@ def _upload_artifact(artifact_path): @catch_mlflow_exception +@_disable_mlflow_artifacts_endpoint def _list_artifacts_mlflow_artifacts(): """ A request handler for `GET /mlflow-artifacts/artifacts?path=` to list artifacts in `path` diff --git a/tests/test_cli.py b/tests/test_cli.py index e5c610280f68b..c542602ec1bb2 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,3 +1,4 @@ +import requests from click.testing import CliRunner from unittest import mock import json @@ -22,7 +23,8 @@ from mlflow.exceptions import MlflowException from mlflow.entities import ViewType -from tests.helper_functions import pyfunc_serve_and_score_model +from tests.helper_functions import pyfunc_serve_and_score_model, get_safe_port +from tests.tracking.integration_test_utils import _await_server_up_or_die def test_server_static_prefix_validation(): @@ -42,6 +44,18 @@ def test_server_static_prefix_validation(): run_server_mock.assert_not_called() +def test_server_mlflow_artifacts_options(): + with mock.patch("mlflow.server._run_server") as run_server_mock: + CliRunner().invoke(server, ["--artifacts-only"]) + run_server_mock.assert_called_once() + with mock.patch("mlflow.server._run_server") as run_server_mock: + CliRunner().invoke(server, ["--serve-artifacts-opt"]) + run_server_mock.assert_called_once() + with mock.patch("mlflow.server._run_server") as run_server_mock: + CliRunner().invoke(server, ["--artifacts-only", "--serve-artifacts-opt"]) + run_server_mock.assert_called_once() + + def test_server_default_artifact_root_validation(): with mock.patch("mlflow.server._run_server") as run_server_mock: result = CliRunner().invoke(server, ["--backend-store-uri", "sqlite:///my.db"]) @@ -219,3 +233,47 @@ def predict(self, context, model_input): # pylint: disable=unused-variable assert scoring_response.status_code == 200 served_model_preds = np.array(json.loads(scoring_response.content)) np.testing.assert_array_equal(served_model_preds, model.predict(data, None)) + + +def test_mlflow_tracking_disabled_in_artifacts_only_mode(): + + port = get_safe_port() + cmd = ["mlflow", "server", "--port", str(port), "--artifacts-only"] + process = subprocess.Popen(cmd) + try: + _await_server_up_or_die(port, timeout=10) + resp = requests.get(f"http://localhost:{port}/api/2.0/mlflow/experiments/list") + assert resp.text.startswith( + "Endpoints disabled due to the mlflow server running " "in `--artifacts-only` mode." + ) + assert resp.status_code == 503 + finally: + process.kill() + + +def test_mlflow_artifact_list_in_artifacts_only_mode(): + + port = get_safe_port() + cmd = ["mlflow", "server", "--port", str(port), "--artifacts-only", "--serve-artifacts-opt"] + process = subprocess.Popen(cmd) + try: + _await_server_up_or_die(port, timeout=10) + resp = requests.get(f"http://localhost:{port}/api/2.0/mlflow-artifacts/artifacts") + resp.raise_for_status() + assert resp.status_code == 200 + assert resp.text == "{}" + finally: + process.kill() + + +def test_mlflow_artifact_service_unavailable_without_config(): + + port = get_safe_port() + cmd = ["mlflow", "server", "--port", str(port)] + process = subprocess.Popen(cmd) + try: + _await_server_up_or_die(port, timeout=10) + resp = requests.get(f"http://localhost:{port}/api/2.0/mlflow-artifacts/artifacts") + assert resp.status_code == 503 + finally: + process.kill() From c44e62a642b8cd935fa86bb8ccfa451f097cee46 Mon Sep 17 00:00:00 2001 From: Ben Wilson Date: Wed, 10 Nov 2021 15:44:23 -0500 Subject: [PATCH 02/18] Update examples to show cli arguments in MLflow Artifacts Signed-off-by: Ben Wilson --- examples/mlflow_artifacts/README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/examples/mlflow_artifacts/README.md b/examples/mlflow_artifacts/README.md index 596446249b2b7..36ce1a9b9fb98 100644 --- a/examples/mlflow_artifacts/README.md +++ b/examples/mlflow_artifacts/README.md @@ -16,6 +16,7 @@ First, launch the tracking server with the artifacts service via `mlflow server` ```sh # Launch a tracking server with the artifacts service $ mlflow server \ + --serve-artifacts-opt --artifacts-destination ./mlartifacts \ --default-artifact-root http://localhost:5000/api/2.0/mlflow-artifacts/artifacts/experiments \ --gunicorn-opts "--log-level debug" @@ -23,9 +24,11 @@ $ mlflow server \ Notes: +- `--serve-artifacts-opt` enables the MLflow Artifacts service endpoints to enable proxied serving of artifacts through the REST API - `--artifacts-destination` specifies the base artifact location from which to resolve artifact upload/download/list requests. In this examples, we're using a local directory `./mlartifacts`, but it can be changed to a s3 bucket or - `--default-artifact-root` points to the `experiments` directory of the artifacts service. Therefore, the default artifact location of a newly-created experiment is set to `./mlartifacts/experiments/`. - `--gunicorn-opts "--log-level debug"` is specified to print out request logs but can be omitted if unnecessary. +- `--artifacts-only` disables all other endpoints for the tracking server apart from those involved in listing, uploading, and downloading artifacts. This makes the MLflow server a single-purpose proxy for artifact handling only. Then, run `example.py` that performs upload, download, and list operations for artifacts: From 74fa204668c235b4d1dd7b46c102b60743e9d9c4 Mon Sep 17 00:00:00 2001 From: Ben Wilson Date: Wed, 10 Nov 2021 15:49:36 -0500 Subject: [PATCH 03/18] Update examples to turn on the REST API endpoints Signed-off-by: Ben Wilson --- examples/mlflow_artifacts/docker-compose.yml | 1 + examples/mlflow_artifacts/example.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/examples/mlflow_artifacts/docker-compose.yml b/examples/mlflow_artifacts/docker-compose.yml index db951d3e1503e..bb016f9658f45 100644 --- a/examples/mlflow_artifacts/docker-compose.yml +++ b/examples/mlflow_artifacts/docker-compose.yml @@ -54,6 +54,7 @@ services: --port 5500 --artifacts-destination s3://bucket --gunicorn-opts "--log-level debug" + --serve-artifacts-opt postgres: image: postgres diff --git a/examples/mlflow_artifacts/example.py b/examples/mlflow_artifacts/example.py index 16dc8f31e830a..0244802936968 100644 --- a/examples/mlflow_artifacts/example.py +++ b/examples/mlflow_artifacts/example.py @@ -9,6 +9,8 @@ def save_text(path, text): with open(path, "w") as f: f.write(text) +# NOTE: ensure the tracking server has been started with --serve-artifacts-opt to enable +# MLflow artifact serving functionality. def main(): assert "MLFLOW_TRACKING_URI" in os.environ From d1b986d1b3db3ed460c5e616c294bff76cd4f588 Mon Sep 17 00:00:00 2001 From: Ben Wilson Date: Wed, 10 Nov 2021 16:11:59 -0500 Subject: [PATCH 04/18] linting Signed-off-by: Ben Wilson --- examples/mlflow_artifacts/example.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/examples/mlflow_artifacts/example.py b/examples/mlflow_artifacts/example.py index 0244802936968..5dabfb1539a26 100644 --- a/examples/mlflow_artifacts/example.py +++ b/examples/mlflow_artifacts/example.py @@ -9,9 +9,11 @@ def save_text(path, text): with open(path, "w") as f: f.write(text) + # NOTE: ensure the tracking server has been started with --serve-artifacts-opt to enable # MLflow artifact serving functionality. + def main(): assert "MLFLOW_TRACKING_URI" in os.environ From 7dbd46921bd5de416d20adfbf0bf6611b180e852 Mon Sep 17 00:00:00 2001 From: Ben Wilson Date: Wed, 10 Nov 2021 22:27:59 -0500 Subject: [PATCH 05/18] PR changes and adjust test for serve-artifact option flag Signed-off-by: Ben Wilson --- examples/mlflow_artifacts/README.md | 4 ++-- examples/mlflow_artifacts/docker-compose.yml | 2 +- examples/mlflow_artifacts/example.py | 2 +- mlflow/cli.py | 6 +++--- mlflow/server/__init__.py | 4 ++-- mlflow/server/handlers.py | 8 ++++---- tests/test_cli.py | 6 +++--- tests/tracking/test_mlflow_artifacts.py | 6 ++++-- 8 files changed, 20 insertions(+), 18 deletions(-) diff --git a/examples/mlflow_artifacts/README.md b/examples/mlflow_artifacts/README.md index 36ce1a9b9fb98..3fa5270031cce 100644 --- a/examples/mlflow_artifacts/README.md +++ b/examples/mlflow_artifacts/README.md @@ -16,7 +16,7 @@ First, launch the tracking server with the artifacts service via `mlflow server` ```sh # Launch a tracking server with the artifacts service $ mlflow server \ - --serve-artifacts-opt + --serve-artifacts --artifacts-destination ./mlartifacts \ --default-artifact-root http://localhost:5000/api/2.0/mlflow-artifacts/artifacts/experiments \ --gunicorn-opts "--log-level debug" @@ -24,7 +24,7 @@ $ mlflow server \ Notes: -- `--serve-artifacts-opt` enables the MLflow Artifacts service endpoints to enable proxied serving of artifacts through the REST API +- `--serve-artifacts` enables the MLflow Artifacts service endpoints to enable proxied serving of artifacts through the REST API - `--artifacts-destination` specifies the base artifact location from which to resolve artifact upload/download/list requests. In this examples, we're using a local directory `./mlartifacts`, but it can be changed to a s3 bucket or - `--default-artifact-root` points to the `experiments` directory of the artifacts service. Therefore, the default artifact location of a newly-created experiment is set to `./mlartifacts/experiments/`. - `--gunicorn-opts "--log-level debug"` is specified to print out request logs but can be omitted if unnecessary. diff --git a/examples/mlflow_artifacts/docker-compose.yml b/examples/mlflow_artifacts/docker-compose.yml index bb016f9658f45..865d681158f95 100644 --- a/examples/mlflow_artifacts/docker-compose.yml +++ b/examples/mlflow_artifacts/docker-compose.yml @@ -54,7 +54,7 @@ services: --port 5500 --artifacts-destination s3://bucket --gunicorn-opts "--log-level debug" - --serve-artifacts-opt + --serve-artifacts postgres: image: postgres diff --git a/examples/mlflow_artifacts/example.py b/examples/mlflow_artifacts/example.py index 5dabfb1539a26..8e9032875793a 100644 --- a/examples/mlflow_artifacts/example.py +++ b/examples/mlflow_artifacts/example.py @@ -10,7 +10,7 @@ def save_text(path, text): f.write(text) -# NOTE: ensure the tracking server has been started with --serve-artifacts-opt to enable +# NOTE: ensure the tracking server has been started with --serve-artifacts to enable # MLflow artifact serving functionality. diff --git a/mlflow/cli.py b/mlflow/cli.py index 63d87e65e6af3..9d2cc9099a906 100644 --- a/mlflow/cli.py +++ b/mlflow/cli.py @@ -321,7 +321,7 @@ def _validate_static_prefix(ctx, param, value): # pylint: disable=unused-argume " used, then this option is required.", ) @click.option( - "--serve-artifacts-opt", + "--serve-artifacts", is_flag=True, help="If specified, enables serving of artifact uploads, downloads, and list requests " "by routing these requests to the storage location that is specified by " @@ -367,7 +367,7 @@ def _validate_static_prefix(ctx, param, value): # pylint: disable=unused-argume def server( backend_store_uri, default_artifact_root, - serve_artifacts_opt, + serve_artifacts, artifacts_only, artifacts_destination, host, @@ -416,7 +416,7 @@ def server( _run_server( backend_store_uri, default_artifact_root, - serve_artifacts_opt, + serve_artifacts, artifacts_only, artifacts_destination, host, diff --git a/mlflow/server/__init__.py b/mlflow/server/__init__.py index c82311eb034eb..819ed18e5313f 100644 --- a/mlflow/server/__init__.py +++ b/mlflow/server/__init__.py @@ -108,7 +108,7 @@ def _build_gunicorn_command(gunicorn_opts, host, port, workers): def _run_server( file_store_path, default_artifact_root, - serve_artifacts_opt, + serve_artifacts, artifacts_only, artifacts_destination, host, @@ -130,7 +130,7 @@ def _run_server( env_map[BACKEND_STORE_URI_ENV_VAR] = file_store_path if default_artifact_root: env_map[ARTIFACT_ROOT_ENV_VAR] = default_artifact_root - if serve_artifacts_opt: + if serve_artifacts: env_map[SERVE_ARTIFACTS_ENV_VAR] = "true" if artifacts_only: env_map[ARTIFACTS_ONLY_ENV_VAR] = "true" diff --git a/mlflow/server/handlers.py b/mlflow/server/handlers.py index 92363eb1bce8f..d5bfdcd3f50da 100644 --- a/mlflow/server/handlers.py +++ b/mlflow/server/handlers.py @@ -262,7 +262,7 @@ def wrapper(*args, **kwargs): ] -def _disable_mlflow_artifacts_endpoint(func): +def _disable_unless_serving_artifacts(func): @wraps(func) def wrapper(*args, **kwargs): from mlflow.server import SERVE_ARTIFACTS_ENV_VAR @@ -913,7 +913,7 @@ def _delete_model_version_tag(): @catch_mlflow_exception -@_disable_mlflow_artifacts_endpoint +@_disable_unless_serving_artifacts def _download_artifact(artifact_path): """ A request handler for `GET /mlflow-artifacts/artifacts/` to download an artifact @@ -940,7 +940,7 @@ def stream_and_remove_file(): @catch_mlflow_exception -@_disable_mlflow_artifacts_endpoint +@_disable_unless_serving_artifacts def _upload_artifact(artifact_path): """ A request handler for `PUT /mlflow-artifacts/artifacts/` to upload an artifact @@ -964,7 +964,7 @@ def _upload_artifact(artifact_path): @catch_mlflow_exception -@_disable_mlflow_artifacts_endpoint +@_disable_unless_serving_artifacts def _list_artifacts_mlflow_artifacts(): """ A request handler for `GET /mlflow-artifacts/artifacts?path=` to list artifacts in `path` diff --git a/tests/test_cli.py b/tests/test_cli.py index c542602ec1bb2..2267c5735e42f 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -49,10 +49,10 @@ def test_server_mlflow_artifacts_options(): CliRunner().invoke(server, ["--artifacts-only"]) run_server_mock.assert_called_once() with mock.patch("mlflow.server._run_server") as run_server_mock: - CliRunner().invoke(server, ["--serve-artifacts-opt"]) + CliRunner().invoke(server, ["--serve-artifacts"]) run_server_mock.assert_called_once() with mock.patch("mlflow.server._run_server") as run_server_mock: - CliRunner().invoke(server, ["--artifacts-only", "--serve-artifacts-opt"]) + CliRunner().invoke(server, ["--artifacts-only", "--serve-artifacts"]) run_server_mock.assert_called_once() @@ -254,7 +254,7 @@ def test_mlflow_tracking_disabled_in_artifacts_only_mode(): def test_mlflow_artifact_list_in_artifacts_only_mode(): port = get_safe_port() - cmd = ["mlflow", "server", "--port", str(port), "--artifacts-only", "--serve-artifacts-opt"] + cmd = ["mlflow", "server", "--port", str(port), "--artifacts-only", "--serve-artifacts"] process = subprocess.Popen(cmd) try: _await_server_up_or_die(port, timeout=10) diff --git a/tests/tracking/test_mlflow_artifacts.py b/tests/tracking/test_mlflow_artifacts.py index d6f0a4361ae64..a75072ea4bc8d 100644 --- a/tests/tracking/test_mlflow_artifacts.py +++ b/tests/tracking/test_mlflow_artifacts.py @@ -3,7 +3,7 @@ import subprocess import tempfile import requests - +import pathlib import pytest import mlflow @@ -26,6 +26,7 @@ def _launch_server(host, port, backend_store_uri, default_artifact_root, artifac str(port), "--backend-store-uri", backend_store_uri, + "--serve-artifacts", "--default-artifact-root", default_artifact_root, "--artifacts-destination", @@ -227,6 +228,7 @@ def is_github_actions(): @pytest.mark.skipif(is_windows(), reason="This example doesn't work on Windows") def test_mlflow_artifacts_example(tmpdir): + root = pathlib.Path(mlflow.__file__).parents[1] # On GitHub Actions, remove generated images to save disk space rmi_option = "--rmi all" if is_github_actions() else "" cmd = f""" @@ -241,5 +243,5 @@ def test_mlflow_artifacts_example(tmpdir): subprocess.run( ["bash", script_path.strpath], check=True, - cwd=os.path.join(os.getcwd(), "examples", "mlflow_artifacts"), + cwd=os.path.join(root, "examples", "mlflow_artifacts"), ) From d6b2f32c90009fa075ab781d0a1e0641245059ab Mon Sep 17 00:00:00 2001 From: Ben Wilson Date: Thu, 11 Nov 2021 11:45:49 -0500 Subject: [PATCH 06/18] PR changes Signed-off-by: Ben Wilson --- examples/mlflow_artifacts/README.md | 2 +- examples/mlflow_artifacts/docker-compose.yml | 1 + mlflow/server/handlers.py | 7 ++++--- tests/test_cli.py | 2 +- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/examples/mlflow_artifacts/README.md b/examples/mlflow_artifacts/README.md index 3fa5270031cce..39105c7f1edf3 100644 --- a/examples/mlflow_artifacts/README.md +++ b/examples/mlflow_artifacts/README.md @@ -16,7 +16,7 @@ First, launch the tracking server with the artifacts service via `mlflow server` ```sh # Launch a tracking server with the artifacts service $ mlflow server \ - --serve-artifacts + --serve-artifacts \ --artifacts-destination ./mlartifacts \ --default-artifact-root http://localhost:5000/api/2.0/mlflow-artifacts/artifacts/experiments \ --gunicorn-opts "--log-level debug" diff --git a/examples/mlflow_artifacts/docker-compose.yml b/examples/mlflow_artifacts/docker-compose.yml index 865d681158f95..2979e04cba22f 100644 --- a/examples/mlflow_artifacts/docker-compose.yml +++ b/examples/mlflow_artifacts/docker-compose.yml @@ -83,6 +83,7 @@ services: --backend-store-uri postgresql://user:password@postgres:5432/db --default-artifact-root http://artifacts-server:5500/api/2.0/mlflow-artifacts/artifacts/experiments --gunicorn-opts "--log-level debug" + --serve-artifacts client: build: diff --git a/mlflow/server/handlers.py b/mlflow/server/handlers.py index d5bfdcd3f50da..b39dc8443c9c3 100644 --- a/mlflow/server/handlers.py +++ b/mlflow/server/handlers.py @@ -270,8 +270,9 @@ def wrapper(*args, **kwargs): if not os.environ.get(SERVE_ARTIFACTS_ENV_VAR): return Response( ( - "Endpoints for mlflow artifacts service are disabled. To enable them, " - "run `mlflow server` with `--serve-artfiacts-opt`" + "Endpoint disabled due to the mlflow server running without " + "`--serve-artifacts`. To enable artifacts server functionaltiy, " + "run `mlflow server` with `--serve-artfiacts`" ), 503, ) @@ -288,7 +289,7 @@ def wrapper(*args, **kwargs): if os.environ.get(ARTIFACTS_ONLY_ENV_VAR): return Response( ( - "Endpoints disabled due to the mlflow server running in `--artifacts-only` " + "Endpoint disabled due to the mlflow server running in `--artifacts-only` " "mode. To enable tracking server functionality, run `mlflow server` without " "`--artifacts-only`" ), diff --git a/tests/test_cli.py b/tests/test_cli.py index ca8aa64366ae2..d021264ed7748 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,4 +1,3 @@ -import requests from click.testing import CliRunner from unittest import mock import json @@ -8,6 +7,7 @@ import tempfile import time import subprocess +import requests from urllib.request import url2pathname from urllib.parse import urlparse, unquote From b0987a5960d981242f7d34a1008745d2d5916084 Mon Sep 17 00:00:00 2001 From: Ben Wilson Date: Thu, 11 Nov 2021 13:44:02 -0500 Subject: [PATCH 07/18] PR updates Signed-off-by: Ben Wilson --- dev/small-requirements.txt | 1 - examples/mlflow_artifacts/docker-compose.yml | 1 + mlflow/azure/client.py | 4 ++-- mlflow/cli.py | 10 +++++++++- mlflow/projects/utils.py | 4 ++-- .../artifact/databricks_artifact_repo.py | 5 +++-- mlflow/store/artifact/http_artifact_repo.py | 7 ++++--- mlflow/utils/file_utils.py | 4 ++-- mlflow/utils/rest_utils.py | 18 +++++++++++++++--- tests/test_cli.py | 19 ++++++++++--------- 10 files changed, 48 insertions(+), 25 deletions(-) diff --git a/dev/small-requirements.txt b/dev/small-requirements.txt index 7f26370baed64..718252f9cc0c2 100644 --- a/dev/small-requirements.txt +++ b/dev/small-requirements.txt @@ -12,4 +12,3 @@ pytest-localserver==0.5.0 moto!=2.0.7 azure-storage-blob>=12.0.0 azure-identity>=1.6.1 -requests \ No newline at end of file diff --git a/examples/mlflow_artifacts/docker-compose.yml b/examples/mlflow_artifacts/docker-compose.yml index 2979e04cba22f..ea2cd504f4c94 100644 --- a/examples/mlflow_artifacts/docker-compose.yml +++ b/examples/mlflow_artifacts/docker-compose.yml @@ -55,6 +55,7 @@ services: --artifacts-destination s3://bucket --gunicorn-opts "--log-level debug" --serve-artifacts + --artifacts-only postgres: image: postgres diff --git a/mlflow/azure/client.py b/mlflow/azure/client.py index 81f5ffe49f3ba..e4c94077affed 100644 --- a/mlflow/azure/client.py +++ b/mlflow/azure/client.py @@ -38,7 +38,7 @@ def put_block(sas_url, block_id, data, headers): with rest_utils.cloud_storage_http_request( "put", request_url, data=data, headers=request_headers ) as response: - response.raise_for_status() + rest_utils.augmented_raise_for_status(response) def put_block_list(sas_url, block_list, headers): @@ -66,7 +66,7 @@ def put_block_list(sas_url, block_list, headers): with rest_utils.cloud_storage_http_request( "put", request_url, data=data, headers=request_headers ) as response: - response.raise_for_status() + rest_utils.augmented_raise_for_status(response) def _append_query_parameters(url, parameters): diff --git a/mlflow/cli.py b/mlflow/cli.py index 9d2cc9099a906..d2d024fd63ae8 100644 --- a/mlflow/cli.py +++ b/mlflow/cli.py @@ -279,7 +279,15 @@ def ui(backend_store_uri, default_artifact_root, artifacts_destination, port, ho # TODO: We eventually want to disable the write path in this version of the server. try: _run_server( - backend_store_uri, default_artifact_root, artifacts_destination, host, port, None, 1 + backend_store_uri, + default_artifact_root, + "false", + "false", + artifacts_destination, + host, + port, + None, + 1, ) except ShellCommandException: eprint("Running the mlflow server failed. Please see the logs above for details.") diff --git a/mlflow/projects/utils.py b/mlflow/projects/utils.py index bfd9f3b2af94f..6d630019cea8f 100644 --- a/mlflow/projects/utils.py +++ b/mlflow/projects/utils.py @@ -28,7 +28,7 @@ MLFLOW_PROJECT_ENTRY_POINT, MLFLOW_PARENT_RUN_ID, ) - +from mlflow.utils.rest_utils import augmented_raise_for_status # TODO: this should be restricted to just Git repos and not S3 and stuff like that _GIT_URI_REGEX = re.compile(r"^[^/]*:") @@ -209,7 +209,7 @@ def _fetch_zip_repo(uri): # https://github.com/mlflow/mlflow/issues/763. response = requests.get(uri) try: - response.raise_for_status() + augmented_raise_for_status(response) except requests.HTTPError as error: raise ExecutionException("Unable to retrieve ZIP file. Reason: %s" % str(error)) return BytesIO(response.content) diff --git a/mlflow/store/artifact/databricks_artifact_repo.py b/mlflow/store/artifact/databricks_artifact_repo.py index 84713b708c4a9..a519b6574ddc1 100644 --- a/mlflow/store/artifact/databricks_artifact_repo.py +++ b/mlflow/store/artifact/databricks_artifact_repo.py @@ -39,6 +39,7 @@ call_endpoint, extract_api_info_for_service, _REST_API_PATH_PREFIX, + augmented_raise_for_status, ) from mlflow.utils.uri import ( extract_and_normalize_path, @@ -247,13 +248,13 @@ def _signed_url_upload_file(self, credentials, local_file): with rest_utils.cloud_storage_http_request( "put", signed_write_uri, data="", headers=headers ) as response: - response.raise_for_status() + augmented_raise_for_status(response) else: with open(local_file, "rb") as file: with rest_utils.cloud_storage_http_request( "put", signed_write_uri, data=file, headers=headers ) as response: - response.raise_for_status() + augmented_raise_for_status(response) except Exception as err: raise MlflowException(err) diff --git a/mlflow/store/artifact/http_artifact_repo.py b/mlflow/store/artifact/http_artifact_repo.py index ccdb9b03ffb11..04c3d27abdfd3 100644 --- a/mlflow/store/artifact/http_artifact_repo.py +++ b/mlflow/store/artifact/http_artifact_repo.py @@ -5,6 +5,7 @@ from mlflow.entities import FileInfo from mlflow.store.artifact.artifact_repo import ArtifactRepository, verify_artifact_path from mlflow.utils.file_utils import relative_path_to_artifact_path +from mlflow.utils.rest_utils import augmented_raise_for_status class HttpArtifactRepository(ArtifactRepository): @@ -25,7 +26,7 @@ def log_artifact(self, local_file, artifact_path=None): url = posixpath.join(self.artifact_uri, *paths) with open(local_file, "rb") as f: resp = self._session.put(url, data=f, timeout=600) - resp.raise_for_status() + augmented_raise_for_status(resp) def log_artifacts(self, local_dir, artifact_path=None): local_dir = os.path.abspath(local_dir) @@ -48,7 +49,7 @@ def list_artifacts(self, path=None): root = tail.lstrip("/") params = {"path": posixpath.join(root, path) if path else root} resp = self._session.get(url, params=params, timeout=10) - resp.raise_for_status() + augmented_raise_for_status(resp) file_infos = [] for f in resp.json().get("files", []): file_info = FileInfo( @@ -63,7 +64,7 @@ def list_artifacts(self, path=None): def _download_file(self, remote_file_path, local_path): url = posixpath.join(self.artifact_uri, remote_file_path) with self._session.get(url, stream=True, timeout=10) as resp: - resp.raise_for_status() + augmented_raise_for_status(resp) with open(local_path, "wb") as f: chunk_size = 1024 * 1024 # 1 MB for chunk in resp.iter_content(chunk_size=chunk_size): diff --git a/mlflow/utils/file_utils.py b/mlflow/utils/file_utils.py index a7e8e54410e5f..2d2bd066b0a6d 100644 --- a/mlflow/utils/file_utils.py +++ b/mlflow/utils/file_utils.py @@ -22,7 +22,7 @@ from mlflow.entities import FileInfo from mlflow.exceptions import MissingConfigException -from mlflow.utils.rest_utils import cloud_storage_http_request +from mlflow.utils.rest_utils import cloud_storage_http_request, augmented_raise_for_status ENCODING = "utf-8" @@ -453,7 +453,7 @@ def download_file_using_http_uri(http_uri, download_path, chunk_size=100000000): providers. """ with cloud_storage_http_request("get", http_uri, stream=True) as response: - response.raise_for_status() + augmented_raise_for_status(response) with open(download_path, "wb") as output_file: for chunk in response.iter_content(chunk_size=chunk_size): if not chunk: diff --git a/mlflow/utils/rest_utils.py b/mlflow/utils/rest_utils.py index 5dbb0c40858c7..d8486c38f27d3 100644 --- a/mlflow/utils/rest_utils.py +++ b/mlflow/utils/rest_utils.py @@ -6,6 +6,7 @@ from packaging.version import Version from requests.adapters import HTTPAdapter from urllib3.util import Retry +from requests.exceptions import HTTPError from mlflow import __version__ from mlflow.protos import databricks_pb2 @@ -83,7 +84,7 @@ def http_request( backoff_factor=2, retry_codes=_TRANSIENT_FAILURE_RESPONSE_CODES, timeout=120, - **kwargs + **kwargs, ): """ Makes an HTTP request with the specified method to the specified hostname/endpoint. Transient @@ -140,7 +141,7 @@ def http_request( headers=headers, verify=verify, timeout=timeout, - **kwargs + **kwargs, ) except Exception as e: raise MlflowException("API request to %s failed with exception %s" % (url, e)) @@ -186,6 +187,17 @@ def verify_rest_response(response, endpoint): return response +def augmented_raise_for_status(response): + """Wrap the standard `requests.response.raise_for_status()` method and return reason""" + try: + response.raise_for_status() + except HTTPError as e: + if response.text: + raise HTTPError(f"{response.status_code} Error: {response.text}") + else: + raise e + + def _get_path(path_prefix, endpoint_path): return "{}{}".format(path_prefix, endpoint_path) @@ -251,7 +263,7 @@ def cloud_storage_http_request( backoff_factor=2, retry_codes=_TRANSIENT_FAILURE_RESPONSE_CODES, timeout=None, - **kwargs + **kwargs, ): """ Performs an HTTP PUT/GET request using Python's `requests` module with automatic retry. diff --git a/tests/test_cli.py b/tests/test_cli.py index d021264ed7748..14b1d03504725 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -8,6 +8,7 @@ import time import subprocess import requests +from requests.exceptions import HTTPError from urllib.request import url2pathname from urllib.parse import urlparse, unquote @@ -22,6 +23,7 @@ from mlflow.store.tracking.file_store import FileStore from mlflow.exceptions import MlflowException from mlflow.entities import ViewType +from mlflow.utils.rest_utils import augmented_raise_for_status from tests.helper_functions import pyfunc_serve_and_score_model, get_safe_port from tests.tracking.integration_test_utils import _await_server_up_or_die @@ -35,7 +37,7 @@ def test_mlflow_server_command(command): try: _await_server_up_or_die(port, timeout=10) resp = requests.get(f"http://localhost:{port}/health") - resp.raise_for_status() + augmented_raise_for_status(resp) assert resp.text == "OK" finally: process.kill() @@ -254,15 +256,14 @@ def test_mlflow_tracking_disabled_in_artifacts_only_mode(): port = get_safe_port() cmd = ["mlflow", "server", "--port", str(port), "--artifacts-only"] process = subprocess.Popen(cmd) - try: + with pytest.raises( + HTTPError, + match="Endpoint disabled due to the mlflow server running in `--artifacts-only` mode.", + ): _await_server_up_or_die(port, timeout=10) resp = requests.get(f"http://localhost:{port}/api/2.0/mlflow/experiments/list") - assert resp.text.startswith( - "Endpoints disabled due to the mlflow server running " "in `--artifacts-only` mode." - ) - assert resp.status_code == 503 - finally: - process.kill() + augmented_raise_for_status(resp) + process.kill() def test_mlflow_artifact_list_in_artifacts_only_mode(): @@ -273,7 +274,7 @@ def test_mlflow_artifact_list_in_artifacts_only_mode(): try: _await_server_up_or_die(port, timeout=10) resp = requests.get(f"http://localhost:{port}/api/2.0/mlflow-artifacts/artifacts") - resp.raise_for_status() + augmented_raise_for_status(resp) assert resp.status_code == 200 assert resp.text == "{}" finally: From 45b949871eb263c825d9d29710862dd260112d9f Mon Sep 17 00:00:00 2001 From: Ben Wilson Date: Fri, 12 Nov 2021 12:09:22 -0500 Subject: [PATCH 08/18] PR feedback changes Signed-off-by: Ben Wilson --- examples/mlflow_artifacts/docker-compose.yml | 1 - mlflow/cli.py | 6 +- mlflow/server/handlers.py | 100 +++++++++---------- mlflow/utils/rest_utils.py | 2 +- tests/test_cli.py | 21 ++-- 5 files changed, 66 insertions(+), 64 deletions(-) diff --git a/examples/mlflow_artifacts/docker-compose.yml b/examples/mlflow_artifacts/docker-compose.yml index ea2cd504f4c94..a700be6107410 100644 --- a/examples/mlflow_artifacts/docker-compose.yml +++ b/examples/mlflow_artifacts/docker-compose.yml @@ -84,7 +84,6 @@ services: --backend-store-uri postgresql://user:password@postgres:5432/db --default-artifact-root http://artifacts-server:5500/api/2.0/mlflow-artifacts/artifacts/experiments --gunicorn-opts "--log-level debug" - --serve-artifacts client: build: diff --git a/mlflow/cli.py b/mlflow/cli.py index d2d024fd63ae8..aa34135a4724d 100644 --- a/mlflow/cli.py +++ b/mlflow/cli.py @@ -281,8 +281,8 @@ def ui(backend_store_uri, default_artifact_root, artifacts_destination, port, ho _run_server( backend_store_uri, default_artifact_root, - "false", - "false", + False, + False, artifacts_destination, host, port, @@ -335,7 +335,7 @@ def _validate_static_prefix(ctx, param, value): # pylint: disable=unused-argume "by routing these requests to the storage location that is specified by " "'--artifact-destination' directly through a proxy. The default location that " "these requests are served from is a local './mlartifacts' directory which can be " - "overridden via '--artifact-destination' arguments. " + "overridden via the '--artifacts-destination' argument. " "Default: False", ) @click.option( diff --git a/mlflow/server/handlers.py b/mlflow/server/handlers.py index b39dc8443c9c3..46f224ed6cce7 100644 --- a/mlflow/server/handlers.py +++ b/mlflow/server/handlers.py @@ -262,7 +262,7 @@ def wrapper(*args, **kwargs): ] -def _disable_unless_serving_artifacts(func): +def _disable_unless_serve_artifacts(func): @wraps(func) def wrapper(*args, **kwargs): from mlflow.server import SERVE_ARTIFACTS_ENV_VAR @@ -270,8 +270,8 @@ def wrapper(*args, **kwargs): if not os.environ.get(SERVE_ARTIFACTS_ENV_VAR): return Response( ( - "Endpoint disabled due to the mlflow server running without " - "`--serve-artifacts`. To enable artifacts server functionaltiy, " + f"Endpoint: {request.url_rule} disabled due to the mlflow server running " + "without `--serve-artifacts`. To enable artifacts server functionaltiy, " "run `mlflow server` with `--serve-artfiacts`" ), 503, @@ -281,7 +281,7 @@ def wrapper(*args, **kwargs): return wrapper -def _disable_mlflow_artifacts_only(func): +def _disable_if_artifacts_only(func): @wraps(func) def wrapper(*args, **kwargs): from mlflow.server import ARTIFACTS_ONLY_ENV_VAR @@ -321,7 +321,7 @@ def _not_implemented(): @catch_mlflow_exception -@_disable_mlflow_artifacts_only +@_disable_if_artifacts_only def _create_experiment(): request_message = _get_request_message(CreateExperiment()) tags = [ExperimentTag(tag.key, tag.value) for tag in request_message.tags] @@ -336,7 +336,7 @@ def _create_experiment(): @catch_mlflow_exception -@_disable_mlflow_artifacts_only +@_disable_if_artifacts_only def _get_experiment(): request_message = _get_request_message(GetExperiment()) response_message = GetExperiment.Response() @@ -348,7 +348,7 @@ def _get_experiment(): @catch_mlflow_exception -@_disable_mlflow_artifacts_only +@_disable_if_artifacts_only def _get_experiment_by_name(): request_message = _get_request_message(GetExperimentByName()) response_message = GetExperimentByName.Response() @@ -366,7 +366,7 @@ def _get_experiment_by_name(): @catch_mlflow_exception -@_disable_mlflow_artifacts_only +@_disable_if_artifacts_only def _delete_experiment(): request_message = _get_request_message(DeleteExperiment()) _get_tracking_store().delete_experiment(request_message.experiment_id) @@ -377,7 +377,7 @@ def _delete_experiment(): @catch_mlflow_exception -@_disable_mlflow_artifacts_only +@_disable_if_artifacts_only def _restore_experiment(): request_message = _get_request_message(RestoreExperiment()) _get_tracking_store().restore_experiment(request_message.experiment_id) @@ -388,7 +388,7 @@ def _restore_experiment(): @catch_mlflow_exception -@_disable_mlflow_artifacts_only +@_disable_if_artifacts_only def _update_experiment(): request_message = _get_request_message(UpdateExperiment()) if request_message.new_name: @@ -402,7 +402,7 @@ def _update_experiment(): @catch_mlflow_exception -@_disable_mlflow_artifacts_only +@_disable_if_artifacts_only def _create_run(): request_message = _get_request_message(CreateRun()) @@ -422,7 +422,7 @@ def _create_run(): @catch_mlflow_exception -@_disable_mlflow_artifacts_only +@_disable_if_artifacts_only def _update_run(): request_message = _get_request_message(UpdateRun()) run_id = request_message.run_id or request_message.run_uuid @@ -436,7 +436,7 @@ def _update_run(): @catch_mlflow_exception -@_disable_mlflow_artifacts_only +@_disable_if_artifacts_only def _delete_run(): request_message = _get_request_message(DeleteRun()) _get_tracking_store().delete_run(request_message.run_id) @@ -447,7 +447,7 @@ def _delete_run(): @catch_mlflow_exception -@_disable_mlflow_artifacts_only +@_disable_if_artifacts_only def _restore_run(): request_message = _get_request_message(RestoreRun()) _get_tracking_store().restore_run(request_message.run_id) @@ -458,7 +458,7 @@ def _restore_run(): @catch_mlflow_exception -@_disable_mlflow_artifacts_only +@_disable_if_artifacts_only def _log_metric(): request_message = _get_request_message(LogMetric()) metric = Metric( @@ -473,7 +473,7 @@ def _log_metric(): @catch_mlflow_exception -@_disable_mlflow_artifacts_only +@_disable_if_artifacts_only def _log_param(): request_message = _get_request_message(LogParam()) param = Param(request_message.key, request_message.value) @@ -486,7 +486,7 @@ def _log_param(): @catch_mlflow_exception -@_disable_mlflow_artifacts_only +@_disable_if_artifacts_only def _set_experiment_tag(): request_message = _get_request_message(SetExperimentTag()) tag = ExperimentTag(request_message.key, request_message.value) @@ -498,7 +498,7 @@ def _set_experiment_tag(): @catch_mlflow_exception -@_disable_mlflow_artifacts_only +@_disable_if_artifacts_only def _set_tag(): request_message = _get_request_message(SetTag()) tag = RunTag(request_message.key, request_message.value) @@ -511,7 +511,7 @@ def _set_tag(): @catch_mlflow_exception -@_disable_mlflow_artifacts_only +@_disable_if_artifacts_only def _delete_tag(): request_message = _get_request_message(DeleteTag()) _get_tracking_store().delete_tag(request_message.run_id, request_message.key) @@ -522,7 +522,7 @@ def _delete_tag(): @catch_mlflow_exception -@_disable_mlflow_artifacts_only +@_disable_if_artifacts_only def _get_run(): request_message = _get_request_message(GetRun()) response_message = GetRun.Response() @@ -534,7 +534,7 @@ def _get_run(): @catch_mlflow_exception -@_disable_mlflow_artifacts_only +@_disable_if_artifacts_only def _search_runs(): request_message = _get_request_message(SearchRuns()) response_message = SearchRuns.Response() @@ -558,7 +558,7 @@ def _search_runs(): @catch_mlflow_exception -@_disable_mlflow_artifacts_only +@_disable_if_artifacts_only def _list_artifacts(): request_message = _get_request_message(ListArtifacts()) response_message = ListArtifacts.Response() @@ -577,7 +577,7 @@ def _list_artifacts(): @catch_mlflow_exception -@_disable_mlflow_artifacts_only +@_disable_if_artifacts_only def _get_metric_history(): request_message = _get_request_message(GetMetricHistory()) response_message = GetMetricHistory.Response() @@ -590,7 +590,7 @@ def _get_metric_history(): @catch_mlflow_exception -@_disable_mlflow_artifacts_only +@_disable_if_artifacts_only def _list_experiments(): request_message = _get_request_message(ListExperiments()) # `ListFields` returns a list of (FieldDescriptor, value) tuples for *present* fields: @@ -608,13 +608,13 @@ def _list_experiments(): @catch_mlflow_exception -@_disable_mlflow_artifacts_only +@_disable_if_artifacts_only def _get_artifact_repo(run): return get_artifact_repository(run.info.artifact_uri) @catch_mlflow_exception -@_disable_mlflow_artifacts_only +@_disable_if_artifacts_only def _log_batch(): _validate_batch_log_api_req(_get_request_json()) request_message = _get_request_message(LogBatch()) @@ -631,7 +631,7 @@ def _log_batch(): @catch_mlflow_exception -@_disable_mlflow_artifacts_only +@_disable_if_artifacts_only def _log_model(): request_message = _get_request_message(LogModel()) try: @@ -671,7 +671,7 @@ def _wrap_response(response_message): @catch_mlflow_exception -@_disable_mlflow_artifacts_only +@_disable_if_artifacts_only def _create_registered_model(): request_message = _get_request_message(CreateRegisteredModel()) registered_model = _get_model_registry_store().create_registered_model( @@ -684,7 +684,7 @@ def _create_registered_model(): @catch_mlflow_exception -@_disable_mlflow_artifacts_only +@_disable_if_artifacts_only def _get_registered_model(): request_message = _get_request_message(GetRegisteredModel()) registered_model = _get_model_registry_store().get_registered_model(name=request_message.name) @@ -693,7 +693,7 @@ def _get_registered_model(): @catch_mlflow_exception -@_disable_mlflow_artifacts_only +@_disable_if_artifacts_only def _update_registered_model(): request_message = _get_request_message(UpdateRegisteredModel()) name = request_message.name @@ -706,7 +706,7 @@ def _update_registered_model(): @catch_mlflow_exception -@_disable_mlflow_artifacts_only +@_disable_if_artifacts_only def _rename_registered_model(): request_message = _get_request_message(RenameRegisteredModel()) name = request_message.name @@ -719,7 +719,7 @@ def _rename_registered_model(): @catch_mlflow_exception -@_disable_mlflow_artifacts_only +@_disable_if_artifacts_only def _delete_registered_model(): request_message = _get_request_message(DeleteRegisteredModel()) _get_model_registry_store().delete_registered_model(name=request_message.name) @@ -727,7 +727,7 @@ def _delete_registered_model(): @catch_mlflow_exception -@_disable_mlflow_artifacts_only +@_disable_if_artifacts_only def _list_registered_models(): request_message = _get_request_message(ListRegisteredModels()) registered_models = _get_model_registry_store().list_registered_models( @@ -741,7 +741,7 @@ def _list_registered_models(): @catch_mlflow_exception -@_disable_mlflow_artifacts_only +@_disable_if_artifacts_only def _search_registered_models(): request_message = _get_request_message(SearchRegisteredModels()) store = _get_model_registry_store() @@ -759,7 +759,7 @@ def _search_registered_models(): @catch_mlflow_exception -@_disable_mlflow_artifacts_only +@_disable_if_artifacts_only def _get_latest_versions(): request_message = _get_request_message(GetLatestVersions()) latest_versions = _get_model_registry_store().get_latest_versions( @@ -771,7 +771,7 @@ def _get_latest_versions(): @catch_mlflow_exception -@_disable_mlflow_artifacts_only +@_disable_if_artifacts_only def _set_registered_model_tag(): request_message = _get_request_message(SetRegisteredModelTag()) tag = RegisteredModelTag(key=request_message.key, value=request_message.value) @@ -780,7 +780,7 @@ def _set_registered_model_tag(): @catch_mlflow_exception -@_disable_mlflow_artifacts_only +@_disable_if_artifacts_only def _delete_registered_model_tag(): request_message = _get_request_message(DeleteRegisteredModelTag()) _get_model_registry_store().delete_registered_model_tag( @@ -790,7 +790,7 @@ def _delete_registered_model_tag(): @catch_mlflow_exception -@_disable_mlflow_artifacts_only +@_disable_if_artifacts_only def _create_model_version(): request_message = _get_request_message(CreateModelVersion()) model_version = _get_model_registry_store().create_model_version( @@ -806,7 +806,7 @@ def _create_model_version(): @catch_mlflow_exception -@_disable_mlflow_artifacts_only +@_disable_if_artifacts_only def get_model_version_artifact_handler(): from querystring_parser import parser @@ -819,7 +819,7 @@ def get_model_version_artifact_handler(): @catch_mlflow_exception -@_disable_mlflow_artifacts_only +@_disable_if_artifacts_only def _get_model_version(): request_message = _get_request_message(GetModelVersion()) model_version = _get_model_registry_store().get_model_version( @@ -831,7 +831,7 @@ def _get_model_version(): @catch_mlflow_exception -@_disable_mlflow_artifacts_only +@_disable_if_artifacts_only def _update_model_version(): request_message = _get_request_message(UpdateModelVersion()) new_description = None @@ -844,7 +844,7 @@ def _update_model_version(): @catch_mlflow_exception -@_disable_mlflow_artifacts_only +@_disable_if_artifacts_only def _transition_stage(): request_message = _get_request_message(TransitionModelVersionStage()) model_version = _get_model_registry_store().transition_model_version_stage( @@ -859,7 +859,7 @@ def _transition_stage(): @catch_mlflow_exception -@_disable_mlflow_artifacts_only +@_disable_if_artifacts_only def _delete_model_version(): request_message = _get_request_message(DeleteModelVersion()) _get_model_registry_store().delete_model_version( @@ -869,7 +869,7 @@ def _delete_model_version(): @catch_mlflow_exception -@_disable_mlflow_artifacts_only +@_disable_if_artifacts_only def _get_model_version_download_uri(): request_message = _get_request_message(GetModelVersionDownloadUri()) download_uri = _get_model_registry_store().get_model_version_download_uri( @@ -880,7 +880,7 @@ def _get_model_version_download_uri(): @catch_mlflow_exception -@_disable_mlflow_artifacts_only +@_disable_if_artifacts_only def _search_model_versions(): request_message = _get_request_message(SearchModelVersions()) model_versions = _get_model_registry_store().search_model_versions(request_message.filter) @@ -890,7 +890,7 @@ def _search_model_versions(): @catch_mlflow_exception -@_disable_mlflow_artifacts_only +@_disable_if_artifacts_only def _set_model_version_tag(): request_message = _get_request_message(SetModelVersionTag()) tag = ModelVersionTag(key=request_message.key, value=request_message.value) @@ -901,7 +901,7 @@ def _set_model_version_tag(): @catch_mlflow_exception -@_disable_mlflow_artifacts_only +@_disable_if_artifacts_only def _delete_model_version_tag(): request_message = _get_request_message(DeleteModelVersionTag()) _get_model_registry_store().delete_model_version_tag( @@ -914,7 +914,7 @@ def _delete_model_version_tag(): @catch_mlflow_exception -@_disable_unless_serving_artifacts +@_disable_unless_serve_artifacts def _download_artifact(artifact_path): """ A request handler for `GET /mlflow-artifacts/artifacts/` to download an artifact @@ -941,7 +941,7 @@ def stream_and_remove_file(): @catch_mlflow_exception -@_disable_unless_serving_artifacts +@_disable_unless_serve_artifacts def _upload_artifact(artifact_path): """ A request handler for `PUT /mlflow-artifacts/artifacts/` to upload an artifact @@ -965,7 +965,7 @@ def _upload_artifact(artifact_path): @catch_mlflow_exception -@_disable_unless_serving_artifacts +@_disable_unless_serve_artifacts def _list_artifacts_mlflow_artifacts(): """ A request handler for `GET /mlflow-artifacts/artifacts?path=` to list artifacts in `path` diff --git a/mlflow/utils/rest_utils.py b/mlflow/utils/rest_utils.py index d8486c38f27d3..2afff0e02fc89 100644 --- a/mlflow/utils/rest_utils.py +++ b/mlflow/utils/rest_utils.py @@ -193,7 +193,7 @@ def augmented_raise_for_status(response): response.raise_for_status() except HTTPError as e: if response.text: - raise HTTPError(f"{response.status_code} Error: {response.text}") + raise HTTPError(f"{e}. Response text: {response.text}") else: raise e diff --git a/tests/test_cli.py b/tests/test_cli.py index 14b1d03504725..2e6c3be6a0693 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -256,13 +256,12 @@ def test_mlflow_tracking_disabled_in_artifacts_only_mode(): port = get_safe_port() cmd = ["mlflow", "server", "--port", str(port), "--artifacts-only"] process = subprocess.Popen(cmd) - with pytest.raises( - HTTPError, - match="Endpoint disabled due to the mlflow server running in `--artifacts-only` mode.", - ): - _await_server_up_or_die(port, timeout=10) - resp = requests.get(f"http://localhost:{port}/api/2.0/mlflow/experiments/list") - augmented_raise_for_status(resp) + _await_server_up_or_die(port, timeout=10) + resp = requests.get(f"http://localhost:{port}/api/2.0/mlflow/experiments/list") + assert ( + "Endpoint disabled due to the mlflow server running in `--artifacts-only` mode." + in resp.text + ) process.kill() @@ -288,7 +287,11 @@ def test_mlflow_artifact_service_unavailable_without_config(): process = subprocess.Popen(cmd) try: _await_server_up_or_die(port, timeout=10) - resp = requests.get(f"http://localhost:{port}/api/2.0/mlflow-artifacts/artifacts") - assert resp.status_code == 503 + endpoint = "/api/2.0/mlflow-artifacts/artifacts" + resp = requests.get(f"http://localhost:{port}{endpoint}") + assert ( + f"Endpoint: {endpoint} disabled due to the mlflow server running without " + "`--serve-artifacts`" in resp.text + ) finally: process.kill() From 3f0cbb061b79cc787bad73a5de1a758945c1b080 Mon Sep 17 00:00:00 2001 From: Ben Wilson Date: Fri, 12 Nov 2021 12:32:13 -0500 Subject: [PATCH 09/18] lint Signed-off-by: Ben Wilson --- tests/test_cli.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index 2e6c3be6a0693..0bfcd149a55bc 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -8,7 +8,6 @@ import time import subprocess import requests -from requests.exceptions import HTTPError from urllib.request import url2pathname from urllib.parse import urlparse, unquote From 05899d924b93889dfd2eea8a554a153119365f15 Mon Sep 17 00:00:00 2001 From: Ben Wilson Date: Mon, 15 Nov 2021 12:00:37 -0500 Subject: [PATCH 10/18] Add ui server support for proxied artifacts and update exception messages Signed-off-by: Ben Wilson --- mlflow/cli.py | 18 ++++++------------ mlflow/server/handlers.py | 6 +++--- mlflow/utils/cli_args.py | 11 +++++++++++ tests/test_cli.py | 4 ++-- 4 files changed, 22 insertions(+), 17 deletions(-) diff --git a/mlflow/cli.py b/mlflow/cli.py index aa34135a4724d..f2ebf8ad6c9c9 100644 --- a/mlflow/cli.py +++ b/mlflow/cli.py @@ -243,10 +243,13 @@ def _validate_server_args(gunicorn_opts=None, workers=None, waitress_opts=None): "Note that this flag does not impact already-created experiments. " "Default: " + DEFAULT_LOCAL_FILE_AND_ARTIFACT_PATH, ) +@cli_args.SERVE_ARTIFACTS @cli_args.ARTIFACTS_DESTINATION @cli_args.PORT @cli_args.HOST -def ui(backend_store_uri, default_artifact_root, artifacts_destination, port, host): +def ui( + backend_store_uri, default_artifact_root, serve_artifacts, artifacts_destination, port, host +): """ Launch the MLflow tracking UI for local viewing of run results. To launch a production server, use the "mlflow server" command instead. @@ -281,7 +284,7 @@ def ui(backend_store_uri, default_artifact_root, artifacts_destination, port, ho _run_server( backend_store_uri, default_artifact_root, - False, + serve_artifacts, False, artifacts_destination, host, @@ -328,16 +331,7 @@ def _validate_static_prefix(ctx, param, value): # pylint: disable=unused-argume "Default: Within file store, if a file:/ URI is provided. If a sql backend is" " used, then this option is required.", ) -@click.option( - "--serve-artifacts", - is_flag=True, - help="If specified, enables serving of artifact uploads, downloads, and list requests " - "by routing these requests to the storage location that is specified by " - "'--artifact-destination' directly through a proxy. The default location that " - "these requests are served from is a local './mlartifacts' directory which can be " - "overridden via the '--artifacts-destination' argument. " - "Default: False", -) +@cli_args.SERVE_ARTIFACTS @click.option( "--artifacts-only", is_flag=True, diff --git a/mlflow/server/handlers.py b/mlflow/server/handlers.py index 46f224ed6cce7..ec8f216ec01e7 100644 --- a/mlflow/server/handlers.py +++ b/mlflow/server/handlers.py @@ -289,9 +289,9 @@ def wrapper(*args, **kwargs): if os.environ.get(ARTIFACTS_ONLY_ENV_VAR): return Response( ( - "Endpoint disabled due to the mlflow server running in `--artifacts-only` " - "mode. To enable tracking server functionality, run `mlflow server` without " - "`--artifacts-only`" + f"Endpoint: {request.url_rule} disabled due to the mlflow server running " + "in `--artifacts-only` mode. To enable tracking server functionality, run " + "`mlflow server` without `--artifacts-only`" ), 503, ) diff --git a/mlflow/utils/cli_args.py b/mlflow/utils/cli_args.py index 71ef7771e5530..a24c962968435 100644 --- a/mlflow/utils/cli_args.py +++ b/mlflow/utils/cli_args.py @@ -99,3 +99,14 @@ "artifact root location is http or mlflow-artifacts URI." ), ) + +SERVE_ARTIFACTS = click.option( + "--serve-artifacts", + is_flag=True, + help="If specified, enables serving of artifact uploads, downloads, and list requests " + "by routing these requests to the storage location that is specified by " + "'--artifact-destination' directly through a proxy. The default location that " + "these requests are served from is a local './mlartifacts' directory which can be " + "overridden via the '--artifacts-destination' argument. " + "Default: False", +) diff --git a/tests/test_cli.py b/tests/test_cli.py index 0bfcd149a55bc..e5275d9ec8e5a 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -258,8 +258,8 @@ def test_mlflow_tracking_disabled_in_artifacts_only_mode(): _await_server_up_or_die(port, timeout=10) resp = requests.get(f"http://localhost:{port}/api/2.0/mlflow/experiments/list") assert ( - "Endpoint disabled due to the mlflow server running in `--artifacts-only` mode." - in resp.text + "Endpoint: /api/2.0/mlflow/experiments/list disabled due to the mlflow server running " + "in `--artifacts-only` mode." in resp.text ) process.kill() From d83c7a32b1c928845b2e2dfb69e39c64cb5a9122 Mon Sep 17 00:00:00 2001 From: Ben Wilson Date: Wed, 24 Nov 2021 19:35:56 -0500 Subject: [PATCH 11/18] Rebase and discard formatting commits Signed-off-by: Ben Wilson --- mlflow/cli.py | 50 +++++++++++++++---------------- mlflow/store/tracking/__init__.py | 1 + mlflow/utils/cli_args.py | 1 + mlflow/utils/uri.py | 22 ++++++++++++++ 4 files changed, 49 insertions(+), 25 deletions(-) diff --git a/mlflow/cli.py b/mlflow/cli.py index f2ebf8ad6c9c9..30dfef1066cc3 100644 --- a/mlflow/cli.py +++ b/mlflow/cli.py @@ -13,14 +13,14 @@ import mlflow.runs import mlflow.store.artifact.cli from mlflow import tracking -from mlflow.store.tracking import DEFAULT_LOCAL_FILE_AND_ARTIFACT_PATH +from mlflow.store.tracking import DEFAULT_LOCAL_FILE_AND_ARTIFACT_PATH, DEFAULT_ARTIFACTS_URI from mlflow.store.artifact.artifact_repository_registry import get_artifact_repository from mlflow.tracking import _get_store from mlflow.utils import cli_args from mlflow.utils.annotations import experimental from mlflow.utils.logging_utils import eprint from mlflow.utils.process import ShellCommandException -from mlflow.utils.uri import is_local_uri +from mlflow.utils.uri import resolve_default_artifact_root from mlflow.entities.lifecycle_stage import LifecycleStage from mlflow.exceptions import MlflowException @@ -233,15 +233,19 @@ def _validate_server_args(gunicorn_opts=None, workers=None, waitress_opts=None): "SQLAlchemy-compatible database connection strings " "(e.g. 'sqlite:///path/to/file.db') or local filesystem URIs " "(e.g. 'file:///absolute/path/to/directory'). By default, data will be logged " - "to the ./mlruns directory.", + f"to {DEFAULT_LOCAL_FILE_AND_ARTIFACT_PATH}", ) @click.option( "--default-artifact-root", metavar="URI", default=None, - help="Path to local directory to store artifacts, for new experiments. " - "Note that this flag does not impact already-created experiments. " - "Default: " + DEFAULT_LOCAL_FILE_AND_ARTIFACT_PATH, + help="Directory in which to store artifacts for any new experiments created. For tracking " + "server backends that rely on SQL, this option is required in order to store artifacts. " + "Note that this flag does not impact already-created experiments with any previous " + "configuration of an MLflow server instance. " + f"By default, data will be logged to the {DEFAULT_ARTIFACTS_URI} uri proxy if " + "the --serve-artifacts option is enabled. Otherwise, the default location will " + f"be {DEFAULT_LOCAL_FILE_AND_ARTIFACT_PATH}.", ) @cli_args.SERVE_ARTIFACTS @cli_args.ARTIFACTS_DESTINATION @@ -266,11 +270,9 @@ def ui( if not backend_store_uri: backend_store_uri = DEFAULT_LOCAL_FILE_AND_ARTIFACT_PATH - if not default_artifact_root: - if is_local_uri(backend_store_uri): - default_artifact_root = backend_store_uri - else: - default_artifact_root = DEFAULT_LOCAL_FILE_AND_ARTIFACT_PATH + default_artifact_root = resolve_default_artifact_root( + serve_artifacts, default_artifact_root, backend_store_uri, True + ) try: initialize_backend_stores(backend_store_uri, default_artifact_root) @@ -326,18 +328,22 @@ def _validate_static_prefix(ctx, param, value): # pylint: disable=unused-argume "--default-artifact-root", metavar="URI", default=None, - help="Local or S3 URI to store artifacts, for new experiments. " - "Note that this flag does not impact already-created experiments. " - "Default: Within file store, if a file:/ URI is provided. If a sql backend is" - " used, then this option is required.", + help="Directory in which to store artifacts for any new experiments created. For tracking " + "server backends that rely on SQL, this option is required in order to store artifacts. " + "Note that this flag does not impact already-created experiments with any previous " + "configuration of an MLflow server instance. " + f"By default, data will be logged to the {DEFAULT_ARTIFACTS_URI} uri proxy if " + "the --serve-artifacts option is enabled. Otherwise, the default location will " + f"be {DEFAULT_LOCAL_FILE_AND_ARTIFACT_PATH}.", ) @cli_args.SERVE_ARTIFACTS @click.option( "--artifacts-only", is_flag=True, + default=False, help="If specified, configures the mlflow server to be used only for proxied artifact serving. " "With this mode enabled, functionality of the mlflow tracking service (e.g. run creation, " - "metric logging, and parameter logging are disabled. The server will only expose " + "metric logging, and parameter logging) is disabled. The server will only expose " "endpoints for uploading, downloading, and listing artifacts. " "Default: False", ) @@ -397,15 +403,9 @@ def server( if not backend_store_uri: backend_store_uri = DEFAULT_LOCAL_FILE_AND_ARTIFACT_PATH - if not default_artifact_root: - if is_local_uri(backend_store_uri): - default_artifact_root = backend_store_uri - else: - eprint( - "Option 'default-artifact-root' is required, when backend store is not " - "local file based." - ) - sys.exit(1) + default_artifact_root = resolve_default_artifact_root( + serve_artifacts, default_artifact_root, backend_store_uri + ) try: initialize_backend_stores(backend_store_uri, default_artifact_root) diff --git a/mlflow/store/tracking/__init__.py b/mlflow/store/tracking/__init__.py index 889f5d7c43ec1..6127275bb2872 100644 --- a/mlflow/store/tracking/__init__.py +++ b/mlflow/store/tracking/__init__.py @@ -10,5 +10,6 @@ # Also used as default location for artifacts, when not provided, in non local file based backends # (eg MySQL) DEFAULT_LOCAL_FILE_AND_ARTIFACT_PATH = "./mlruns" +DEFAULT_ARTIFACTS_URI = "mlflow-artifacts:/" SEARCH_MAX_RESULTS_DEFAULT = 1000 SEARCH_MAX_RESULTS_THRESHOLD = 50000 diff --git a/mlflow/utils/cli_args.py b/mlflow/utils/cli_args.py index a24c962968435..30dc1fc793f8c 100644 --- a/mlflow/utils/cli_args.py +++ b/mlflow/utils/cli_args.py @@ -103,6 +103,7 @@ SERVE_ARTIFACTS = click.option( "--serve-artifacts", is_flag=True, + default=False, help="If specified, enables serving of artifact uploads, downloads, and list requests " "by routing these requests to the storage location that is specified by " "'--artifact-destination' directly through a proxy. The default location that " diff --git a/mlflow/utils/uri.py b/mlflow/utils/uri.py index 808585b74e7ae..55e346d1144f9 100644 --- a/mlflow/utils/uri.py +++ b/mlflow/utils/uri.py @@ -1,10 +1,13 @@ +import sys import posixpath import urllib.parse from mlflow.exceptions import MlflowException from mlflow.protos.databricks_pb2 import INVALID_PARAMETER_VALUE from mlflow.store.db.db_types import DATABASE_ENGINES +from mlflow.store.tracking import DEFAULT_LOCAL_FILE_AND_ARTIFACT_PATH, DEFAULT_ARTIFACTS_URI from mlflow.utils.validation import _validate_db_type_string +from mlflow.utils.logging_utils import eprint _INVALID_DB_URI_MSG = ( "Please refer to https://mlflow.org/docs/latest/tracking.html#storage for " @@ -294,3 +297,22 @@ def dbfs_hdfs_uri_to_fuse_path(dbfs_uri): ) return _DBFS_FUSE_PREFIX + dbfs_uri[len(_DBFS_HDFS_URI_PREFIX) :] + + +def resolve_default_artifact_root( + serve_artifacts, default_artifact_root, backend_store_uri, resolve_to_local=False +): + if serve_artifacts and not default_artifact_root: + default_artifact_root = DEFAULT_ARTIFACTS_URI + elif not serve_artifacts and not default_artifact_root: + if is_local_uri(backend_store_uri): + default_artifact_root = backend_store_uri + elif resolve_to_local: + default_artifact_root = DEFAULT_LOCAL_FILE_AND_ARTIFACT_PATH + else: + eprint( + "Option 'default-artifact-root' is required, when backend store is not " + "local file based." + ) + sys.exit(1) + return default_artifact_root From 6d5641c58090ca5e387f577be81f38b52e85594e Mon Sep 17 00:00:00 2001 From: Ben Wilson Date: Mon, 29 Nov 2021 09:16:05 -0500 Subject: [PATCH 12/18] typos Signed-off-by: Ben Wilson --- mlflow/cli.py | 2 +- mlflow/server/handlers.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mlflow/cli.py b/mlflow/cli.py index 30dfef1066cc3..e211ade1fcb49 100644 --- a/mlflow/cli.py +++ b/mlflow/cli.py @@ -271,7 +271,7 @@ def ui( backend_store_uri = DEFAULT_LOCAL_FILE_AND_ARTIFACT_PATH default_artifact_root = resolve_default_artifact_root( - serve_artifacts, default_artifact_root, backend_store_uri, True + serve_artifacts, default_artifact_root, backend_store_uri, resolve_to_local=True ) try: diff --git a/mlflow/server/handlers.py b/mlflow/server/handlers.py index ec8f216ec01e7..c8b89870aca45 100644 --- a/mlflow/server/handlers.py +++ b/mlflow/server/handlers.py @@ -271,8 +271,8 @@ def wrapper(*args, **kwargs): return Response( ( f"Endpoint: {request.url_rule} disabled due to the mlflow server running " - "without `--serve-artifacts`. To enable artifacts server functionaltiy, " - "run `mlflow server` with `--serve-artfiacts`" + "without `--serve-artifacts`. To enable artifacts server functionality, " + "run `mlflow server` with `--serve-artifacts`" ), 503, ) From 8fffe43d22e7a640c468153ec5d7f23ff33b4c11 Mon Sep 17 00:00:00 2001 From: Ben Wilson Date: Mon, 29 Nov 2021 13:08:45 -0500 Subject: [PATCH 13/18] fix parsing of uri for trailing slash Signed-off-by: Ben Wilson --- mlflow/store/artifact/mlflow_artifacts_repo.py | 4 +++- tests/store/artifact/test_mlflow_artifact_repo.py | 8 +++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/mlflow/store/artifact/mlflow_artifacts_repo.py b/mlflow/store/artifact/mlflow_artifacts_repo.py index 1d624d5ade894..f459bb9d5a62b 100644 --- a/mlflow/store/artifact/mlflow_artifacts_repo.py +++ b/mlflow/store/artifact/mlflow_artifacts_repo.py @@ -1,5 +1,6 @@ from urllib.parse import urlparse from collections import namedtuple +import re from mlflow.store.artifact.http_artifact_repo import HttpArtifactRepository from mlflow.tracking._tracking_service.utils import get_tracking_uri @@ -72,7 +73,8 @@ def resolve_uri(cls, artifact_uri): elif uri_parse.path == base_url: # for operations like list artifacts resolved = base_url else: - resolved = f"{base_url}{track_parse.path}{uri_parse.path.lstrip('/')}" + resolved = f"{base_url}/{track_parse.path}{uri_parse.path}" + resolved = re.sub("//+", "/", resolved) if uri_parse.host and uri_parse.port: resolved_artifacts_uri = ( diff --git a/tests/store/artifact/test_mlflow_artifact_repo.py b/tests/store/artifact/test_mlflow_artifact_repo.py index 2e57bfe961407..2c3e073867519 100644 --- a/tests/store/artifact/test_mlflow_artifact_repo.py +++ b/tests/store/artifact/test_mlflow_artifact_repo.py @@ -8,11 +8,13 @@ from mlflow.exceptions import MlflowException -@pytest.fixture(scope="module", autouse=True) -def set_tracking_uri(): +@pytest.fixture( + scope="module", autouse=True, params=["http://localhost:5000", "http://localhost:5000/"] +) +def set_tracking_uri(request): with mock.patch( "mlflow.store.artifact.mlflow_artifacts_repo.get_tracking_uri", - return_value="http://localhost:5000/", + return_value=request.param, ): yield From 374e2093ff38ff5c7db3d6e408868fa85a3da7b5 Mon Sep 17 00:00:00 2001 From: Ben Wilson Date: Mon, 29 Nov 2021 20:48:14 -0500 Subject: [PATCH 14/18] Test complexity reduction Signed-off-by: Ben Wilson --- .../artifact/test_mlflow_artifact_repo.py | 25 +++++++++++++++---- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/tests/store/artifact/test_mlflow_artifact_repo.py b/tests/store/artifact/test_mlflow_artifact_repo.py index 2c3e073867519..b2de9617aefe8 100644 --- a/tests/store/artifact/test_mlflow_artifact_repo.py +++ b/tests/store/artifact/test_mlflow_artifact_repo.py @@ -8,17 +8,32 @@ from mlflow.exceptions import MlflowException -@pytest.fixture( - scope="module", autouse=True, params=["http://localhost:5000", "http://localhost:5000/"] -) -def set_tracking_uri(request): +@pytest.fixture(scope="module", autouse=True) +def set_tracking_uri(): with mock.patch( "mlflow.store.artifact.mlflow_artifacts_repo.get_tracking_uri", - return_value=request.param, + return_value="http://localhost:5000/", ): yield +@pytest.fixture(scope="module", autouse=False) +def set_alternate_tracking_uri(): + with mock.patch( + "mlflow.store.artifact.mlflow_artifacts_repo.get_tracking_uri", + return_value="http://localhost:5000", + ): + yield + + +@pytest.mark.usefixtures("set_alternate_tracking_uri") +def test_mlflow_artifact_uri_alternate_host_uri(): + submitted = "mlflow-artifacts://myhostname:5045/my/artifacts" + resolved = "http://myhostname:5045/api/2.0/mlflow-artifacts/artifacts/my/artifacts" + artifact_repo = MlflowArtifactsRepository(submitted) + assert artifact_repo.resolve_uri(submitted) == resolved + + def test_artifact_uri_factory(): repo = get_artifact_repository("mlflow-artifacts://test.com") assert isinstance(repo, MlflowArtifactsRepository) From de2e78c9bf1720a166090f136cbf2bbdf0a20c28 Mon Sep 17 00:00:00 2001 From: Ben Wilson Date: Mon, 29 Nov 2021 21:24:19 -0500 Subject: [PATCH 15/18] tracking uri validation checks and test simplification Signed-off-by: Ben Wilson --- .../store/artifact/mlflow_artifacts_repo.py | 5 ++- .../artifact/test_mlflow_artifact_repo.py | 31 +++++-------------- 2 files changed, 9 insertions(+), 27 deletions(-) diff --git a/mlflow/store/artifact/mlflow_artifacts_repo.py b/mlflow/store/artifact/mlflow_artifacts_repo.py index f459bb9d5a62b..c9fd90b469b4d 100644 --- a/mlflow/store/artifact/mlflow_artifacts_repo.py +++ b/mlflow/store/artifact/mlflow_artifacts_repo.py @@ -50,13 +50,12 @@ class MlflowArtifactsRepository(HttpArtifactRepository): def __init__(self, artifact_uri): - super().__init__(self.resolve_uri(artifact_uri)) + super().__init__(self.resolve_uri(artifact_uri, get_tracking_uri())) @classmethod - def resolve_uri(cls, artifact_uri): + def resolve_uri(cls, artifact_uri, tracking_uri): base_url = "/api/2.0/mlflow-artifacts/artifacts" - tracking_uri = get_tracking_uri() track_parse = _parse_artifact_uri(tracking_uri) diff --git a/tests/store/artifact/test_mlflow_artifact_repo.py b/tests/store/artifact/test_mlflow_artifact_repo.py index b2de9617aefe8..3136611fe9039 100644 --- a/tests/store/artifact/test_mlflow_artifact_repo.py +++ b/tests/store/artifact/test_mlflow_artifact_repo.py @@ -17,23 +17,6 @@ def set_tracking_uri(): yield -@pytest.fixture(scope="module", autouse=False) -def set_alternate_tracking_uri(): - with mock.patch( - "mlflow.store.artifact.mlflow_artifacts_repo.get_tracking_uri", - return_value="http://localhost:5000", - ): - yield - - -@pytest.mark.usefixtures("set_alternate_tracking_uri") -def test_mlflow_artifact_uri_alternate_host_uri(): - submitted = "mlflow-artifacts://myhostname:5045/my/artifacts" - resolved = "http://myhostname:5045/api/2.0/mlflow-artifacts/artifacts/my/artifacts" - artifact_repo = MlflowArtifactsRepository(submitted) - assert artifact_repo.resolve_uri(submitted) == resolved - - def test_artifact_uri_factory(): repo = get_artifact_repository("mlflow-artifacts://test.com") assert isinstance(repo, MlflowArtifactsRepository) @@ -46,29 +29,29 @@ def test_mlflow_artifact_uri_formats_resolved(): ( f"mlflow-artifacts://myhostname:4242{base_path}/hostport", f"http://myhostname:4242{base_url}{base_path}/hostport", + "http://myhostname:4242", ), ( f"mlflow-artifacts://myhostname{base_path}/host", f"http://myhostname{base_url}{base_path}/host", + "http://myhostname", ), ( f"mlflow-artifacts:{base_path}/nohost", f"http://localhost:5000{base_url}{base_path}/nohost", + "http://localhost:5000/", ), ( f"mlflow-artifacts://{base_path}/redundant", f"http://localhost:5000{base_url}{base_path}/redundant", + "http://localhost:5000", ), - ( - "mlflow-artifacts:/", - f"http://localhost:5000{base_url}", - ), + ("mlflow-artifacts:/", f"http://localhost:5000{base_url}", "http://localhost:5000/"), ] failing_conditions = [f"mlflow-artifacts://5000/{base_path}", "mlflow-artifacts://5000/"] - for submit, resolved in conditions: - artifact_repo = MlflowArtifactsRepository(submit) - assert artifact_repo.resolve_uri(submit) == resolved + for submit, resolved, tracking_uri in conditions: + assert MlflowArtifactsRepository.resolve_uri(submit, tracking_uri) == resolved for failing_condition in failing_conditions: with pytest.raises( MlflowException, From d530c50a581f55642b47a17db0f402c0a744d58e Mon Sep 17 00:00:00 2001 From: Ben Wilson Date: Mon, 29 Nov 2021 21:41:11 -0500 Subject: [PATCH 16/18] Parameterize the uri resolution tests to aid in test debugging Signed-off-by: Ben Wilson --- .../artifact/test_mlflow_artifact_repo.py | 61 +++++++++++-------- 1 file changed, 34 insertions(+), 27 deletions(-) diff --git a/tests/store/artifact/test_mlflow_artifact_repo.py b/tests/store/artifact/test_mlflow_artifact_repo.py index 3136611fe9039..14c36feb93f18 100644 --- a/tests/store/artifact/test_mlflow_artifact_repo.py +++ b/tests/store/artifact/test_mlflow_artifact_repo.py @@ -22,36 +22,43 @@ def test_artifact_uri_factory(): assert isinstance(repo, MlflowArtifactsRepository) -def test_mlflow_artifact_uri_formats_resolved(): - base_url = "/api/2.0/mlflow-artifacts/artifacts" - base_path = "/my/artifact/path" - conditions = [ - ( - f"mlflow-artifacts://myhostname:4242{base_path}/hostport", - f"http://myhostname:4242{base_url}{base_path}/hostport", - "http://myhostname:4242", - ), - ( - f"mlflow-artifacts://myhostname{base_path}/host", - f"http://myhostname{base_url}{base_path}/host", - "http://myhostname", - ), - ( - f"mlflow-artifacts:{base_path}/nohost", - f"http://localhost:5000{base_url}{base_path}/nohost", - "http://localhost:5000/", - ), - ( - f"mlflow-artifacts://{base_path}/redundant", - f"http://localhost:5000{base_url}{base_path}/redundant", - "http://localhost:5000", - ), - ("mlflow-artifacts:/", f"http://localhost:5000{base_url}", "http://localhost:5000/"), - ] - failing_conditions = [f"mlflow-artifacts://5000/{base_path}", "mlflow-artifacts://5000/"] +base_url = "/api/2.0/mlflow-artifacts/artifacts" +base_path = "/my/artifact/path" +conditions = [ + ( + f"mlflow-artifacts://myhostname:4242{base_path}/hostport", + f"http://myhostname:4242{base_url}{base_path}/hostport", + "http://myhostname:4242", + ), + ( + f"mlflow-artifacts://myhostname{base_path}/host", + f"http://myhostname{base_url}{base_path}/host", + "http://myhostname", + ), + ( + f"mlflow-artifacts:{base_path}/nohost", + f"http://localhost:5000{base_url}{base_path}/nohost", + "http://localhost:5000/", + ), + ( + f"mlflow-artifacts://{base_path}/redundant", + f"http://localhost:5000{base_url}{base_path}/redundant", + "http://localhost:5000", + ), + ("mlflow-artifacts:/", f"http://localhost:5000{base_url}", "http://localhost:5000/"), +] + + +@pytest.mark.parametrize("submit, resolved, tracking_uri", conditions) +def test_mlflow_artifact_uri_formats_resolved(submit, resolved, tracking_uri): for submit, resolved, tracking_uri in conditions: assert MlflowArtifactsRepository.resolve_uri(submit, tracking_uri) == resolved + + +def test_mlflow_artifact_uri_raises_with_invalid_tracking_uri(): + failing_conditions = [f"mlflow-artifacts://5000/{base_path}", "mlflow-artifacts://5000/"] + for failing_condition in failing_conditions: with pytest.raises( MlflowException, From 5db552b6ab0c9c301fe5f811ee871715cb5fbb35 Mon Sep 17 00:00:00 2001 From: Ben Wilson Date: Mon, 29 Nov 2021 21:54:24 -0500 Subject: [PATCH 17/18] Cleaner test syntax Signed-off-by: Ben Wilson --- tests/store/artifact/test_mlflow_artifact_repo.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/tests/store/artifact/test_mlflow_artifact_repo.py b/tests/store/artifact/test_mlflow_artifact_repo.py index 14c36feb93f18..62a0e14cda479 100644 --- a/tests/store/artifact/test_mlflow_artifact_repo.py +++ b/tests/store/artifact/test_mlflow_artifact_repo.py @@ -28,32 +28,28 @@ def test_artifact_uri_factory(): ( f"mlflow-artifacts://myhostname:4242{base_path}/hostport", f"http://myhostname:4242{base_url}{base_path}/hostport", - "http://myhostname:4242", ), ( f"mlflow-artifacts://myhostname{base_path}/host", f"http://myhostname{base_url}{base_path}/host", - "http://myhostname", ), ( f"mlflow-artifacts:{base_path}/nohost", f"http://localhost:5000{base_url}{base_path}/nohost", - "http://localhost:5000/", ), ( f"mlflow-artifacts://{base_path}/redundant", f"http://localhost:5000{base_url}{base_path}/redundant", - "http://localhost:5000", ), - ("mlflow-artifacts:/", f"http://localhost:5000{base_url}", "http://localhost:5000/"), + ("mlflow-artifacts:/", f"http://localhost:5000{base_url}"), ] -@pytest.mark.parametrize("submit, resolved, tracking_uri", conditions) -def test_mlflow_artifact_uri_formats_resolved(submit, resolved, tracking_uri): +@pytest.mark.parametrize("tracking_uri", ["http://localhost:5000", "http://localhost:5000/"]) +@pytest.mark.parametrize("artifact_uri, resolved_uri", conditions) +def test_mlflow_artifact_uri_formats_resolved(artifact_uri, resolved_uri, tracking_uri): - for submit, resolved, tracking_uri in conditions: - assert MlflowArtifactsRepository.resolve_uri(submit, tracking_uri) == resolved + assert MlflowArtifactsRepository.resolve_uri(artifact_uri, tracking_uri) == resolved_uri def test_mlflow_artifact_uri_raises_with_invalid_tracking_uri(): From 85d191c604ee691a0644364d69ebb0e529023968 Mon Sep 17 00:00:00 2001 From: Ben Wilson Date: Tue, 30 Nov 2021 17:04:49 -0500 Subject: [PATCH 18/18] Add and adjust pydoc strings Signed-off-by: Ben Wilson --- mlflow/cli.py | 6 +++--- mlflow/store/tracking/__init__.py | 5 +++++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/mlflow/cli.py b/mlflow/cli.py index e211ade1fcb49..55154ddda520b 100644 --- a/mlflow/cli.py +++ b/mlflow/cli.py @@ -243,9 +243,9 @@ def _validate_server_args(gunicorn_opts=None, workers=None, waitress_opts=None): "server backends that rely on SQL, this option is required in order to store artifacts. " "Note that this flag does not impact already-created experiments with any previous " "configuration of an MLflow server instance. " - f"By default, data will be logged to the {DEFAULT_ARTIFACTS_URI} uri proxy if " - "the --serve-artifacts option is enabled. Otherwise, the default location will " - f"be {DEFAULT_LOCAL_FILE_AND_ARTIFACT_PATH}.", + "If the --serve-artifacts option is specified, the default artifact root is " + f"{DEFAULT_ARTIFACTS_URI}. Otherwise, the default artifact root is " + f"{DEFAULT_LOCAL_FILE_AND_ARTIFACT_PATH}.", ) @cli_args.SERVE_ARTIFACTS @cli_args.ARTIFACTS_DESTINATION diff --git a/mlflow/store/tracking/__init__.py b/mlflow/store/tracking/__init__.py index 6127275bb2872..c708e39366564 100644 --- a/mlflow/store/tracking/__init__.py +++ b/mlflow/store/tracking/__init__.py @@ -10,6 +10,11 @@ # Also used as default location for artifacts, when not provided, in non local file based backends # (eg MySQL) DEFAULT_LOCAL_FILE_AND_ARTIFACT_PATH = "./mlruns" +# Used for defining the artifacts uri (`--default-artifact-root`) for the tracking server when +# configuring the server to use the option `--serve-artifacts` mode. This default can be +# overridden by specifying an override to `--default-artifact-root` for the MLflow tracking server. +# When the server is not operating in `--serve-artifacts` configuration, the default artifact +# storage location will be `DEFAULT_LOCAL_FILE_AND_ARTIFACT_PATH`. DEFAULT_ARTIFACTS_URI = "mlflow-artifacts:/" SEARCH_MAX_RESULTS_DEFAULT = 1000 SEARCH_MAX_RESULTS_THRESHOLD = 50000