Skip to content

Commit

Permalink
Adding test to check fro PF eval dependencies on private API's (#3135)
Browse files Browse the repository at this point in the history
# Description

Please add an informative description that covers that changes made by
the pull request and link all relevant issues.

# All Promptflow Contribution checklist:
- [ ] **The pull request does not introduce [breaking changes].**
- [ ] **CHANGELOG is updated for new features, bug fixes or other
significant changes.**
- [ ] **I have read the [contribution guidelines](../CONTRIBUTING.md).**
- [ ] **Create an issue and link to the pull request to get dedicated
review from promptflow team. Learn more: [suggested
workflow](../CONTRIBUTING.md#suggested-workflow).**

## General Guidelines and Best Practices
- [ ] Title of the pull request is clear and informative.
- [ ] There are a small number of commits, each of which have an
informative message. This means that previously merged commits do not
appear in the history of the PR. For more information on cleaning up the
commits in your PR, [see this
page](https://github.com/Azure/azure-powershell/blob/master/documentation/development-docs/cleaning-up-commits.md).

### Testing Guidelines
- [ ] Pull request includes test coverage for the included changes.

---------

Co-authored-by: Clement Wang <47586720+wangchao1230@users.noreply.github.com>
  • Loading branch information
singankit and wangchao1230 committed May 9, 2024
1 parent 912d05f commit 0690f41
Show file tree
Hide file tree
Showing 4 changed files with 89 additions and 1 deletion.
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
# ---------------------------------------------------------
# Copyright (c) Microsoft Corporation. All rights reserved.
# ---------------------------------------------------------

# This file captures promptflow-evals dependencies on private API of promptflow.
# In case changes are made please reach out to promptflow-evals team to update the dependencies.

# flake8: noqa
from promptflow.azure.operations._async_run_uploader import AsyncRunUploader
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
# ---------------------------------------------------------
# Copyright (c) Microsoft Corporation. All rights reserved.
# ---------------------------------------------------------

import tempfile
from pathlib import Path
from typing import Callable
from unittest.mock import patch
Expand Down Expand Up @@ -260,3 +260,45 @@ def test_upload_flex_flow_run_with_global_azureml(self, pf: PFClient, randstr: C

# check the run is uploaded to cloud.
Local2CloudTestHelper.check_local_to_cloud_run(pf, run)

@pytest.mark.skipif(condition=not pytest.is_live, reason="Bug - 3089145 Replay failed for test 'test_upload_run'")
def test_upload_run_pf_eval_dependencies(
self,
pf: PFClient,
randstr: Callable[[str], str],
):
# This test captures promptflow-evals dependencies on private API of promptflow.
# In case changes are made please reach out to promptflow-evals team to update the dependencies.

name = randstr("batch_run_name_for_upload")
local_pf = Local2CloudTestHelper.get_local_pf(name)
# submit a local batch run.
run = local_pf.run(
flow=f"{FLOWS_DIR}/simple_hello_world",
data=f"{DATAS_DIR}/webClassification3.jsonl",
name=name,
column_mapping={"name": "${data.url}"},
display_name="sdk-cli-test-run-local-to-cloud",
tags={"sdk-cli-test": "true"},
description="test sdk local to cloud",
)
assert run.status == RunStatus.COMPLETED

# check the run is uploaded to cloud
Local2CloudTestHelper.check_local_to_cloud_run(pf, run, check_run_details_in_cloud=True)

from promptflow.azure._dependencies._pf_evals import AsyncRunUploader
from promptflow._sdk._constants import Local2Cloud

async_uploader = AsyncRunUploader._from_run_operations(run, pf.runs)
instance_results = local_pf.runs.get_details(run, all_results=True)

with tempfile.TemporaryDirectory() as temp_dir:
file_name = Local2Cloud.FLOW_INSTANCE_RESULTS_FILE_NAME
local_file = Path(temp_dir) / file_name
instance_results.to_json(local_file, orient="records", lines=True)

# overriding instance_results.jsonl file
remote_file = (f"{Local2Cloud.BLOB_ROOT_PROMPTFLOW}"
f"/{Local2Cloud.BLOB_ARTIFACTS}/{run.name}/{Local2Cloud.FLOW_INSTANCE_RESULTS_FILE_NAME}")
async_run_allowing_running_loop(async_uploader._upload_local_file_to_blob, local_file, remote_file)
10 changes: 10 additions & 0 deletions src/promptflow-devkit/promptflow/_dependencies/_pf_evals.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
# ---------------------------------------------------------
# Copyright (c) Microsoft Corporation. All rights reserved.
# ---------------------------------------------------------

# This file captures promptflow-evals dependencies on private API of promptflow.
# In case changes are made please reach out to promptflow-evals team to update the dependencies.

# flake8: noqa
from promptflow._sdk._constants import LINE_NUMBER, Local2Cloud
from promptflow._sdk._configuration import Configuration
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
from unittest.mock import patch

import pytest
from promptflow._dependencies._pf_evals import LINE_NUMBER, Local2Cloud, Configuration

DUMMY_TRACE_DESTINATION = ("azureml://subscriptions/sub_id/resourceGroups/resource_group_name"
"/providers/Microsoft.MachineLearningServices/workspaces/workspace_name")


@pytest.fixture
def patch_config_validation():
with patch("promptflow._sdk._configuration.Configuration._validate", return_value=None):
yield


@pytest.mark.unittest
class TestPromptflowEvalsDependencies:

def test_pf_eval_constants_dependencies(self):
assert LINE_NUMBER == "line_number"
assert Local2Cloud.FLOW_INSTANCE_RESULTS_FILE_NAME == "instance_results.jsonl"
assert Local2Cloud.BLOB_ROOT_PROMPTFLOW == "promptflow"
assert Local2Cloud.BLOB_ARTIFACTS == "PromptFlowArtifacts"

def test_pf_eval_configuration_dependencies(self, patch_config_validation):
config = Configuration(overrides={"trace.destination": DUMMY_TRACE_DESTINATION})
assert config.get_trace_destination() == DUMMY_TRACE_DESTINATION

0 comments on commit 0690f41

Please sign in to comment.