Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We鈥檒l occasionally send you account related emails.

Already on GitHub? Sign in to your account

Init auto #3690

Draft
wants to merge 5 commits into
base: main
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 2 additions & 0 deletions google/cloud/aiplatform/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,7 @@
aiplatform.init(project='my_project')
"""
init = initializer.global_config.init
init_auto = initializer.global_config.init_auto

get_pipeline_df = metadata.metadata._LegacyExperimentService.get_pipeline_df

Expand Down Expand Up @@ -124,6 +125,7 @@
"explain",
"gapic",
"init",
"init_auto",
"helpers",
"hyperparameter_tuning",
"log",
Expand Down
96 changes: 95 additions & 1 deletion google/cloud/aiplatform/initializer.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,15 +17,18 @@


from concurrent import futures
import uuid
import json
import inspect
import logging
import os
import types
from typing import Iterator, List, Optional, Type, TypeVar, Union

from google.api_core import client_options
from google.api_core import gapic_v1
import google.auth
import requests

from google.auth import credentials as auth_credentials
from google.auth.exceptions import GoogleAuthError

Expand All @@ -43,6 +46,8 @@
encryption_spec_v1 as gca_encryption_spec_v1,
encryption_spec_v1beta1 as gca_encryption_spec_v1beta1,
)
from google.cloud import resourcemanager_v3


_TVertexAiServiceClientWithOverride = TypeVar(
"_TVertexAiServiceClientWithOverride",
Expand Down Expand Up @@ -109,6 +114,74 @@ def __init__(self):
self._api_endpoint = None
self._api_transport = None

def init_auto(self, *args, **kwargs):
"""
Create a brand new project for the user and initialize the SDK with it.

The project will be created under the same parent org or folder as the current environment default project.

Any arguments passed to this function will be passed to the init function,
except for project, location and credentials.
"""
# if the user has already created a project, use that
auto_project = os.getenv("VERTEX_AUTO_PROJECT")
credentials = None
if not auto_project:
# create a new project
# find credentials and project using auth library
# for more robust version, see _Config._set_project_as_env_var_or_google_auth_default
# default_project will be None
# unless the user has set one using gcloud config set project
credentials, default_project = google.auth.default(
scopes=["https://www.googleapis.com/auth/cloud-platform"]
)
if not credentials.token:
# refresh token if needed
request = google.auth.transport.requests.Request()
credentials.refresh(request)
if not credentials.token:
raise RuntimeError("Failed to get credentials")
headers = {
"Authorization": f"Bearer {credentials.token}",
"Content-Type": "application/json",
}
last_ancestor = None
parent = None
# find parent folder or organization if it exists
if default_project:
response = requests.post(
f"https://cloudresourcemanager.googleapis.com/v1/projects/{default_project}:getAncestry",
headers=headers,
)
if response.status_code != 200:
raise RuntimeError(
f"Failed to get project ancestry: {response.content}"
)
last_ancestor = json.loads(response.content)["ancestor"][-1][
"resourceId"
]
# ignores if the ancestor is not a folder or an organization
# projects cannot be parents of projects
if last_ancestor and last_ancestor["type"] != "project":
parent = f"{last_ancestor['type']}s/{last_ancestor['id']}"
auto_project = _create_new_vertex_project(parent)
os.environ["VERTEX_AUTO_PROJECT"] = auto_project

# pick a default location
location = (
kwargs.get("location", None)
or os.getenv("GOOGLE_CLOUD_REGION")
or os.getenv("CLOUD_ML_REGION")
or "us-central1"
)
return self.init(
project=auto_project,
location=location,
credentials=credentials,
*args,
**kwargs,
)

def init(
self,
*,
Expand Down Expand Up @@ -527,6 +600,27 @@ def create_client(
max_workers=min(32, max(4, (os.cpu_count() or 0) * 5))
)

# helper function only for use with the init_auto method
# creates a new project with a UUID and readable display name
# returns the project ID when the operation is complete
# if a parent resource ID exists based on user credentials,
# the project will have the same parent resource ID
# otherwise, there will be no parent
def _create_new_vertex_project(parent=None):
client = resourcemanager_v3.ProjectsClient()
display_name = "Vertex AI Auto Project"
auto_project = f"vertex-auto-{uuid.uuid4().hex[:8]}"
request = resourcemanager_v3.CreateProjectRequest(
project=resourcemanager_v3.Project(
project_id=auto_project, display_name=display_name, parent=parent
)
)
operation = client.create_project(request=request)
print("Waiting for operation to complete...")
response = operation.result()
print(f"Created project {response.project_id}")
return response.project_id


def _get_function_name_from_stack_frame(frame) -> str:
"""Gates fully qualified function or method name.
Expand Down
3 changes: 2 additions & 1 deletion vertexai/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,11 @@

__version__ = aiplatform_version.__version__

from google.cloud.aiplatform import init
from google.cloud.aiplatform import init, init_auto
from vertexai import preview

__all__ = [
"init",
"init_auto",
"preview",
]