diff --git a/pyproject.toml b/pyproject.toml index 633e78e9df16b..8611ef9323deb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,14 +30,12 @@ files = [ "src/lightning_lite", "src/lightning_app", ] +# This section is for folders with "-" as they are not valid python modules exclude = [ + "src/lightning_app/cli/app-template", "src/lightning_app/cli/component-template", "src/lightning_app/cli/pl-app-template", "src/lightning_app/cli/react-ui-template", - "src/lightning_app/cli/app-template", - "src/lightning_app/components/database", - "src/lightning_app/components/multi_node", - "src/lightning_app/frontend/just_py/just_py", ] install_types = "True" non_interactive = "True" @@ -67,7 +65,9 @@ module = [ "lightning_app.api.request_types", "lightning_app.cli.commands.app_commands", "lightning_app.cli.commands.connection", - "lightning_app.cli.react-ui-template.example_app", + "lightning_app.cli.commands.lightning_cli", + "lightning_app.cli.commands.cmd_install", + "lightning_app.cli.cmd_install", "lightning_app.components.database.client", "lightning_app.components.database.server", "lightning_app.components.database.utilities", diff --git a/src/lightning_app/CHANGELOG.md b/src/lightning_app/CHANGELOG.md index 0f65db620a60b..889bf82b8e42a 100644 --- a/src/lightning_app/CHANGELOG.md +++ b/src/lightning_app/CHANGELOG.md @@ -28,6 +28,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). - Added a `configure_layout` method to the `LightningWork` which can be used to control how the work is handled in the layout of a parent flow ([#15926](https://github.com/Lightning-AI/lightning/pull/15926)) +- Added the ability to run a Lightning App or Component directly from the Gallery using `lightning run app organization/name` ([#15941](https://github.com/Lightning-AI/lightning/pull/15941)) + - Added automatic conversion of list and dict of works and flows to structures ([#15961](https://github.com/Lightning-AI/lightning/pull/15961)) diff --git a/src/lightning_app/cli/cmd_install.py b/src/lightning_app/cli/cmd_install.py index db0467212f147..579a921179b4c 100644 --- a/src/lightning_app/cli/cmd_install.py +++ b/src/lightning_app/cli/cmd_install.py @@ -5,6 +5,7 @@ import sys from typing import Dict, Optional, Tuple +import click import requests from packaging.version import Version @@ -14,7 +15,117 @@ logger = Logger(__name__) -def gallery_component(name: str, yes_arg: bool, version_arg: str, cwd: str = None) -> None: +@click.group(name="install") +def install() -> None: + """Install Lightning AI selfresources.""" + pass + + +@install.command("app") +@click.argument("name", type=str) +@click.option( + "--yes", + "-y", + is_flag=True, + help="disables prompt to ask permission to create env and run install cmds", +) +@click.option( + "--version", + "-v", + type=str, + help="Specify the version to install. By default it uses 'latest'", + default="latest", + show_default=True, +) +@click.option( + "--overwrite", + "-f", + is_flag=True, + default=False, + help="When set, overwrite the app directory without asking if it already exists.", +) +def install_app(name: str, yes: bool, version: str, overwrite: bool = False) -> None: + _install_app_command(name, yes, version, overwrite=overwrite) + + +@install.command("component") +@click.argument("name", type=str) +@click.option( + "--yes", + "-y", + is_flag=True, + help="disables prompt to ask permission to create env and run install cmds", +) +@click.option( + "--version", + "-v", + type=str, + help="Specify the version to install. By default it uses 'latest'", + default="latest", + show_default=True, +) +def install_component(name: str, yes: bool, version: str) -> None: + _install_component_command(name, yes, version) + + +def _install_app_command(name: str, yes: bool, version: str, overwrite: bool = False) -> None: + if "github.com" in name: + if version != "latest": + logger.warn( + "When installing from GitHub, only the 'latest' version is supported. " + f"The provided version ({version}) will be ignored." + ) + return non_gallery_app(name, yes, overwrite=overwrite) + else: + return gallery_app(name, yes, version, overwrite=overwrite) + + +def _install_component_command(name: str, yes: bool, version: str, overwrite: bool = False) -> None: + if "github.com" in name: + if version != "latest": + logger.warn( + "When installing from GitHub, only the 'latest' version is supported. " + f"The provided version ({version}) will be ignored." + ) + return non_gallery_component(name, yes) + else: + return gallery_component(name, yes, version) + + +def gallery_apps_and_components( + name: str, yes_arg: bool, version_arg: str, cwd: str = None, overwrite: bool = False +) -> Optional[str]: + + try: + org, app_or_component = name.split("/") + except Exception: + return None + + entry, kind = _resolve_entry(app_or_component, version_arg) + + if kind == "app": + # give the user the chance to do a manual install + source_url, git_url, folder_name, git_sha = _show_install_app_prompt( + entry, app_or_component, org, yes_arg, resource_type="app" + ) + # run installation if requested + _install_app_from_source(source_url, git_url, folder_name, cwd=cwd, overwrite=overwrite, git_sha=git_sha) + + return os.path.join(os.getcwd(), folder_name, entry["appEntrypointFile"]) + + elif kind == "component": + # give the user the chance to do a manual install + git_url = _show_install_component_prompt(entry, app_or_component, org, yes_arg) + + # run installation if requested + _install_component_from_source(git_url) + + return os.path.join(os.getcwd(), entry["appEntrypointFile"]) + + return None + + +def gallery_component(name: str, yes_arg: bool, version_arg: str, cwd: str = None) -> str: # make sure org/component-name name is correct org, component = _validate_name(name, resource_type="component", example="lightning/LAI-slack-component") @@ -28,7 +139,9 @@ def gallery_component(name: str, yes_arg: bool, version_arg: str, cwd: str = Non git_url = _show_install_component_prompt(component_entry, component, org, yes_arg) # run installation if requested - _install_component(git_url) + _install_component_from_source(git_url) + + return os.path.join(os.getcwd(), component_entry["entrypointFile"]) def non_gallery_component(gh_url: str, yes_arg: bool, cwd: str = None) -> None: @@ -37,10 +150,10 @@ def non_gallery_component(gh_url: str, yes_arg: bool, cwd: str = None) -> None: git_url = _show_non_gallery_install_component_prompt(gh_url, yes_arg) # run installation if requested - _install_component(git_url) + _install_component_from_source(git_url) -def gallery_app(name: str, yes_arg: bool, version_arg: str, cwd: str = None, overwrite: bool = False) -> None: +def gallery_app(name: str, yes_arg: bool, version_arg: str, cwd: str = None, overwrite: bool = False) -> str: # make sure org/app-name syntax is correct org, app = _validate_name(name, resource_type="app", example="lightning/quick-start") @@ -57,7 +170,9 @@ def gallery_app(name: str, yes_arg: bool, version_arg: str, cwd: str = None, ove ) # run installation if requested - _install_app(source_url, git_url, folder_name, cwd=cwd, overwrite=overwrite, git_sha=git_sha) + _install_app_from_source(source_url, git_url, folder_name, cwd=cwd, overwrite=overwrite, git_sha=git_sha) + + return os.path.join(os.getcwd(), folder_name, app_entry["appEntrypointFile"]) def non_gallery_app(gh_url: str, yes_arg: bool, cwd: str = None, overwrite: bool = False) -> None: @@ -66,7 +181,7 @@ def non_gallery_app(gh_url: str, yes_arg: bool, cwd: str = None, overwrite: bool repo_url, folder_name = _show_non_gallery_install_app_prompt(gh_url, yes_arg) # run installation if requested - _install_app(repo_url, repo_url, folder_name, cwd=cwd, overwrite=overwrite) + _install_app_from_source(repo_url, repo_url, folder_name, cwd=cwd, overwrite=overwrite) def _show_install_component_prompt(entry: Dict[str, str], component: str, org: str, yes_arg: bool) -> str: @@ -299,7 +414,35 @@ def _validate_name(name: str, resource_type: str, example: str) -> Tuple[str, st return org, resource -def _resolve_resource(registry_url: str, name: str, version_arg: str, resource_type: str) -> Dict[str, str]: +def _resolve_entry(name, version_arg) -> Tuple[Optional[Dict], Optional[str]]: + entry = None + kind = None + + # resolve registry (orgs can have a private registry through their environment variables) + registry_url = _resolve_app_registry() + + # load the app resource + entry = _resolve_resource(registry_url, name=name, version_arg=version_arg, resource_type="app", raise_error=False) + + if not entry: + + registry_url = _resolve_component_registry() + + # load the component resource + entry = _resolve_resource( + registry_url, name=name, version_arg=version_arg, resource_type="component", raise_error=False + ) + kind = "component" if entry else None + + else: + kind = "app" + + return entry, kind + + +def _resolve_resource( + registry_url: str, name: str, version_arg: str, resource_type: str, raise_error: bool = True +) -> Dict[str, str]: gallery_entries = [] try: response = requests.get(registry_url) @@ -327,7 +470,10 @@ def _resolve_resource(registry_url: str, name: str, version_arg: str, resource_t all_versions.append(x["version"]) if len(entries) == 0: - raise SystemExit(f"{resource_type}: '{name}' is not available on ⚡ Lightning AI ⚡") + if raise_error: + raise SystemExit(f"{resource_type}: '{name}' is not available on ⚡ Lightning AI ⚡") + else: + return None entry = None if version_arg == "latest": @@ -337,11 +483,14 @@ def _resolve_resource(registry_url: str, name: str, version_arg: str, resource_t if e["version"] == version_arg: entry = e break - if entry is None: - raise Exception( - f"{resource_type}: 'Version {version_arg} for {name}' is not available on ⚡ Lightning AI ⚡. " - f"Here is the list of all availables versions:{os.linesep}{os.linesep.join(all_versions)}" - ) + if entry is None and raise_error: + if raise_error: + raise Exception( + f"{resource_type}: 'Version {version_arg} for {name}' is not available on ⚡ Lightning AI ⚡. " + f"Here is the list of all availables versions:{os.linesep}{os.linesep.join(all_versions)}" + ) + else: + return None return entry @@ -381,7 +530,7 @@ def _install_with_env(repo_url: str, folder_name: str, cwd: str = None) -> None: logger.info(m) -def _install_app( +def _install_app_from_source( source_url: str, git_url: str, folder_name: str, cwd: str = None, overwrite: bool = False, git_sha: str = None ) -> None: """Installing lighting app from the `git_url` @@ -458,7 +607,7 @@ def _install_app( logger.info(m) -def _install_component(git_url: str) -> None: +def _install_component_from_source(git_url: str) -> None: logger.info("⚡ RUN: pip install") out = subprocess.check_output(["pip", "install", git_url]) diff --git a/src/lightning_app/cli/lightning_cli.py b/src/lightning_app/cli/lightning_cli.py index 8fcf7a7f10dc1..11138ea6fbcd4 100644 --- a/src/lightning_app/cli/lightning_cli.py +++ b/src/lightning_app/cli/lightning_cli.py @@ -232,7 +232,14 @@ def _run_app( secret: tuple, run_app_comment_commands: bool, ) -> None: - file = _prepare_file(file) + + if not os.path.exists(file): + original_file = file + file = cmd_install.gallery_apps_and_components(file, True, "latest", overwrite=False) # type: ignore[assignment] # noqa E501 + if file is None: + click.echo(f"The provided entrypoint `{original_file}` doesn't exist.") + sys.exit(1) + run_app_comment_commands = True if not cloud and cluster_id is not None: raise click.ClickException("Using the flag --cluster-id in local execution is not supported.") @@ -288,7 +295,7 @@ def run() -> None: @run.command("app") -@click.argument("file", type=click.Path(exists=True)) +@click.argument("file", type=str) @click.option("--cloud", type=bool, default=False, is_flag=True) @click.option( "--cluster-id", @@ -361,6 +368,7 @@ def run_app( _main.add_command(get_list) _main.add_command(delete) _main.add_command(create) +_main.add_command(cmd_install.install) @_main.command("ssh") @@ -444,74 +452,6 @@ def ssh(app_name: str = None, component_name: str = None) -> None: os.execv(ssh_path, ["-tt", f"{component_id}@{ssh_endpoint}"]) -@_main.group() -def install() -> None: - """Install a Lightning App and/or component.""" - - -@install.command("app") -@click.argument("name", type=str) -@click.option( - "--yes", - "-y", - is_flag=True, - help="disables prompt to ask permission to create env and run install cmds", -) -@click.option( - "--version", - "-v", - type=str, - help="Specify the version to install. By default it uses 'latest'", - default="latest", - show_default=True, -) -@click.option( - "--overwrite", - "-f", - is_flag=True, - default=False, - help="When set, overwrite the app directory without asking if it already exists.", -) -def install_app(name: str, yes: bool, version: str, overwrite: bool = False) -> None: - if "github.com" in name: - if version != "latest": - logger.warn( - f"The provided version {version} isn't the officially supported one. " - f"The provided version will be ignored." - ) - cmd_install.non_gallery_app(name, yes, overwrite=overwrite) - else: - cmd_install.gallery_app(name, yes, version, overwrite=overwrite) - - -@install.command("component") -@click.argument("name", type=str) -@click.option( - "--yes", - "-y", - is_flag=True, - help="disables prompt to ask permission to create env and run install cmds", -) -@click.option( - "--version", - "-v", - type=str, - help="Specify the version to install. By default it uses 'latest'", - default="latest", - show_default=True, -) -def install_component(name: str, yes: bool, version: str) -> None: - if "github.com" in name: - if version != "latest": - logger.warn( - f"The provided version {version} isn't the officially supported one. " - f"The provided version will be ignored." - ) - cmd_install.non_gallery_component(name, yes) - else: - cmd_install.gallery_component(name, yes, version) - - @_main.group() def init() -> None: """Init a Lightning App and/or component.""" diff --git a/tests/tests_app/cli/test_cmd_install.py b/tests/tests_app/cli/test_cmd_install.py index aa0c34ba6ed2d..2e2086348cb58 100644 --- a/tests/tests_app/cli/test_cmd_install.py +++ b/tests/tests_app/cli/test_cmd_install.py @@ -17,19 +17,19 @@ def test_valid_org_app_name(): # assert a bad app name should fail fake_app = "fakeuser/impossible/name" - result = runner.invoke(lightning_cli.install_app, [fake_app]) + result = runner.invoke(lightning_cli.cmd_install.install_app, [fake_app]) assert "app name format must have organization/app-name" in result.output # assert a good name (but unavailable name) should work fake_app = "fakeuser/ALKKLJAUHREKJ21234KLAKJDLF" - result = runner.invoke(lightning_cli.install_app, [fake_app]) + result = runner.invoke(lightning_cli.cmd_install.install_app, [fake_app]) assert f"app: '{fake_app}' is not available on ⚡ Lightning AI ⚡" in result.output assert result.exit_code # assert a good (and availablea name) works # This should be an app that's always in the gallery real_app = "lightning/invideo" - result = runner.invoke(lightning_cli.install_app, [real_app]) + result = runner.invoke(lightning_cli.cmd_install.install_app, [real_app]) assert "Press enter to continue:" in result.output @@ -47,16 +47,16 @@ def test_valid_unpublished_app_name(): assert "WARNING" in str(e.output) # assert aborted install - result = runner.invoke(lightning_cli.install_app, [real_app], input="q") + result = runner.invoke(lightning_cli.cmd_install.install_app, [real_app], input="q") assert "Installation aborted!" in result.output # assert a bad app name should fail fake_app = "https://github.com/Lightning-AI/install-appdd" - result = runner.invoke(lightning_cli.install_app, [fake_app, "--yes"]) + result = runner.invoke(lightning_cli.cmd_install.install_app, [fake_app, "--yes"]) assert "Looks like the github url was not found" in result.output # assert a good (and availablea name) works - result = runner.invoke(lightning_cli.install_app, [real_app]) + result = runner.invoke(lightning_cli.cmd_install.install_app, [real_app]) assert "Press enter to continue:" in result.output @@ -81,17 +81,17 @@ def test_valid_org_component_name(): # assert a bad name should fail fake_component = "fakeuser/impossible/name" - result = runner.invoke(lightning_cli.install_component, [fake_component]) + result = runner.invoke(lightning_cli.cmd_install.install_component, [fake_component]) assert "component name format must have organization/component-name" in result.output # assert a good name (but unavailable name) should work fake_component = "fakeuser/ALKKLJAUHREKJ21234KLAKJDLF" - result = runner.invoke(lightning_cli.install_component, [fake_component]) + result = runner.invoke(lightning_cli.cmd_install.install_component, [fake_component]) assert f"component: '{fake_component}' is not available on ⚡ Lightning AI ⚡" in result.output # assert a good (and availablea name) works fake_component = "lightning/lit-slack-messenger" - result = runner.invoke(lightning_cli.install_component, [fake_component]) + result = runner.invoke(lightning_cli.cmd_install.install_component, [fake_component]) assert "Press enter to continue:" in result.output @@ -100,13 +100,13 @@ def test_unpublished_component_url_parsing(): # assert a bad name should fail (no git@) fake_component = "https://github.com/Lightning-AI/LAI-slack-messenger" - result = runner.invoke(lightning_cli.install_component, [fake_component]) + result = runner.invoke(lightning_cli.cmd_install.install_component, [fake_component]) assert "Error, your github url must be in the following format" in result.output # assert a good (and availablea name) works sha = "14f333456ffb6758bd19458e6fa0bf12cf5575e1" real_component = f"git+https://github.com/Lightning-AI/LAI-slack-messenger.git@{sha}" - result = runner.invoke(lightning_cli.install_component, [real_component]) + result = runner.invoke(lightning_cli.cmd_install.install_component, [real_component]) assert "Press enter to continue:" in result.output @@ -148,26 +148,26 @@ def test_prompt_actions(): runner = CliRunner() # assert that the user can cancel the command with any letter other than y - result = runner.invoke(lightning_cli.install_app, [app_to_use], input="b") + result = runner.invoke(lightning_cli.cmd_install.install_app, [app_to_use], input="b") assert "Installation aborted!" in result.output # assert that the install happens with --yes - # result = runner.invoke(lightning_cli.install_app, [app_to_use, "--yes"]) + # result = runner.invoke(lightning_cli.cmd_install.install_app, [app_to_use, "--yes"]) # assert result.exit_code == 0 # assert that the install happens with y - # result = runner.invoke(lightning_cli.install_app, [app_to_use], input='y') + # result = runner.invoke(lightning_cli.cmd_install.install_app, [app_to_use], input='y') # assert result.exit_code == 0 # # assert that the install happens with yes - # result = runner.invoke(lightning_cli.install_app, [app_to_use], input='yes') + # result = runner.invoke(lightning_cli.cmd_install.install_app, [app_to_use], input='yes') # assert result.exit_code == 0 # assert that the install happens with pressing enter - # result = runner.invoke(lightning_cli.install_app, [app_to_use]) + # result = runner.invoke(lightning_cli.cmd_install.install_app, [app_to_use]) # TODO: how to check the output when the user types ctrl+c? - # result = runner.invoke(lightning_cli.install_app, [app_to_use], input='') + # result = runner.invoke(lightning_cli.cmd_install.install_app, [app_to_use], input='') @mock.patch("lightning_app.cli.cmd_install.subprocess", mock.MagicMock()) @@ -178,7 +178,7 @@ def test_version_arg_component(tmpdir, monkeypatch): # Version does not exist component_name = "lightning/lit-slack-messenger" version_arg = "NOT-EXIST" - result = runner.invoke(lightning_cli.install_component, [component_name, f"--version={version_arg}"]) + result = runner.invoke(lightning_cli.cmd_install.install_component, [component_name, f"--version={version_arg}"]) assert f"component: 'Version {version_arg} for {component_name}' is not" in str(result.exception) assert result.exit_code == 1 @@ -186,7 +186,9 @@ def test_version_arg_component(tmpdir, monkeypatch): # This somwehow fail in test but not when you actually run it version_arg = "0.0.1" runner = CliRunner() - result = runner.invoke(lightning_cli.install_component, [component_name, f"--version={version_arg}", "--yes"]) + result = runner.invoke( + lightning_cli.cmd_install.install_component, [component_name, f"--version={version_arg}", "--yes"] + ) assert result.exit_code == 0 @@ -198,14 +200,14 @@ def test_version_arg_app(tmpdir): app_name = "lightning/invideo" version_arg = "NOT-EXIST" runner = CliRunner() - result = runner.invoke(lightning_cli.install_app, [app_name, f"--version={version_arg}"]) + result = runner.invoke(lightning_cli.cmd_install.install_app, [app_name, f"--version={version_arg}"]) assert f"app: 'Version {version_arg} for {app_name}' is not" in str(result.exception) assert result.exit_code == 1 # Version exists version_arg = "0.0.2" runner = CliRunner() - result = runner.invoke(lightning_cli.install_app, [app_name, f"--version={version_arg}", "--yes"]) + result = runner.invoke(lightning_cli.cmd_install.install_app, [app_name, f"--version={version_arg}", "--yes"]) assert result.exit_code == 0 @@ -236,7 +238,9 @@ def test_install_resolve_latest_version(mock_show_install_app_prompt, tmpdir): }, ] } - runner.invoke(lightning_cli.install_app, [app_name, "--yes"]) # no version specified so latest is installed + runner.invoke( + lightning_cli.cmd_install.install_app, [app_name, "--yes"] + ) # no version specified so latest is installed assert mock_show_install_app_prompt.called assert mock_show_install_app_prompt.call_args[0][0]["version"] == "0.0.4" @@ -274,7 +278,7 @@ def test_install_app_shows_error(tmpdir): app_folder_dir.mkdir() with pytest.raises(SystemExit, match=f"Folder {str(app_folder_dir)} exists, please delete it and try again."): - cmd_install._install_app( + cmd_install._install_app_from_source( source_url=mock.ANY, git_url=mock.ANY, folder_name=str(app_folder_dir), overwrite=False ) @@ -360,7 +364,9 @@ def test_install_app_process(subprocess_mock, source_url, git_url, git_sha, tmpd app_folder_dir = Path(tmpdir / "some_random_directory").absolute() app_folder_dir.mkdir() - cmd_install._install_app(source_url, git_url, folder_name=str(app_folder_dir), overwrite=True, git_sha=git_sha) + cmd_install._install_app_from_source( + source_url, git_url, folder_name=str(app_folder_dir), overwrite=True, git_sha=git_sha + ) assert subprocess_mock.check_output.call_args_list[0].args == (["git", "clone", git_url],) if git_sha: assert subprocess_mock.check_output.call_args_list[1].args == (["git", "checkout", git_sha],)