From c34d70f6646b5496b34f040476363e2780d74677 Mon Sep 17 00:00:00 2001 From: Oracen Date: Tue, 1 Nov 2022 13:53:15 +0000 Subject: [PATCH 1/9] Test date coercion works --- tests/test_converter.py | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/tests/test_converter.py b/tests/test_converter.py index 683cc8ac..1183a98d 100644 --- a/tests/test_converter.py +++ b/tests/test_converter.py @@ -4,7 +4,7 @@ import sys from collections import deque from dataclasses import dataclass -from datetime import datetime +from datetime import datetime, timezone from enum import Enum, IntEnum from typing import ( Any, @@ -164,6 +164,37 @@ def test_encode_search_attribute_values(): temporalio.converter.encode_search_attribute_values(["foo", 123]) +def test_decode_search_attributes(): + """Tests decode from protobuf for python types""" + + def payload(key, dtype, data, encoding=None): + if encoding is None: + encoding = {"encoding": b"json/plain"} + check = temporalio.api.common.v1.Payload( + data=bytes(data, encoding="utf-8"), + metadata={"type": bytes(dtype, encoding="utf-8"), **encoding}, + ) + return temporalio.api.common.v1.SearchAttributes(indexed_fields={key: check}) + + # Check basic keyword parsing works + kw_check = temporalio.converter.decode_search_attributes( + payload("kw", "Keyword", '"test-id"') + ) + assert kw_check["kw"][0] == "test-id" + + # Ensure original DT functionality works + dt_check = temporalio.converter.decode_search_attributes( + payload("dt", "Datetime", '"2020-01-01T00:00:00"') + ) + assert dt_check["dt"][0] == datetime(2020, 1, 1, 0, 0, 0) + + # Check timezone aware works as server is using ISO 8601 + dttz_check = temporalio.converter.decode_search_attributes( + payload("dt", "Datetime", '"2020-01-01T00:00:00Z"') + ) + assert dttz_check["dt"][0] == datetime(2020, 1, 1, 0, 0, 0, tzinfo=timezone.utc) + + NewIntType = NewType("NewIntType", int) MyDataClassAlias = MyDataClass From 58add4edd30654ef4fb79d354c1ccc277d0a1a89 Mon Sep 17 00:00:00 2001 From: Oracen Date: Tue, 1 Nov 2022 14:06:03 +0000 Subject: [PATCH 2/9] Add in dateutil to Python < 3.11 --- poetry.lock | 39 +++++++++++++++++++++++++++------------ pyproject.toml | 4 ++++ tests/test_converter.py | 6 ++++++ 3 files changed, 37 insertions(+), 12 deletions(-) diff --git a/poetry.lock b/poetry.lock index 9bd138d3..6a044bd7 100644 --- a/poetry.lock +++ b/poetry.lock @@ -26,7 +26,7 @@ python-versions = ">=3.5" dev = ["cloudpickle", "coverage[toml] (>=5.0.2)", "furo", "hypothesis", "mypy (>=0.900,!=0.940)", "pre-commit", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "sphinx", "sphinx-notfound-page", "zope.interface"] docs = ["furo", "sphinx", "sphinx-notfound-page", "zope.interface"] tests = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "zope.interface"] -tests_no_zope = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins"] +tests-no-zope = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins"] [[package]] name = "Automat" @@ -143,7 +143,7 @@ optional = false python-versions = ">=3.6.0" [package.extras] -unicode_backport = ["unicodedata2"] +unicode-backport = ["unicodedata2"] [[package]] name = "cibuildwheel" @@ -381,9 +381,9 @@ python-versions = ">=3.6.1,<4.0" [package.extras] colors = ["colorama (>=0.4.3,<0.5.0)"] -pipfile_deprecated_finder = ["pipreqs", "requirementslib"] +pipfile-deprecated-finder = ["pipreqs", "requirementslib"] plugins = ["setuptools"] -requirements_deprecated_finder = ["pip-api", "pipreqs"] +requirements-deprecated-finder = ["pip-api", "pipreqs"] [[package]] name = "jaraco.classes" @@ -760,6 +760,17 @@ python-versions = ">=3.6" [package.dependencies] pytest = ">=5.0.0" +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +category = "main" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" + +[package.dependencies] +six = ">=1.5" + [[package]] name = "pywin32-ctypes" version = "0.2.0" @@ -800,7 +811,7 @@ urllib3 = ">=1.21.1,<1.27" [package.extras] socks = ["PySocks (>=1.5.6,!=1.5.7)"] -use_chardet_on_py3 = ["chardet (>=3.0.2,<6)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] [[package]] name = "requests-toolbelt" @@ -960,20 +971,20 @@ typing-extensions = ">=3.6.5" "zope.interface" = ">=4.4.2" [package.extras] -all_non_platform = ["PyHamcrest (>=1.9.0)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "contextvars (>=2.4,<3)", "cryptography (>=2.6)", "cython-test-exception-raiser (>=1.0.2,<2)", "h2 (>=3.0,<5.0)", "idna (>=2.4)", "priority (>=1.1.0,<2.0)", "pyasn1", "pyopenssl (>=16.0.0)", "pyserial (>=3.0)", "pywin32 (!=226)", "service-identity (>=18.1.0)"] +all-non-platform = ["PyHamcrest (>=1.9.0)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "contextvars (>=2.4,<3)", "cryptography (>=2.6)", "cython-test-exception-raiser (>=1.0.2,<2)", "h2 (>=3.0,<5.0)", "idna (>=2.4)", "priority (>=1.1.0,<2.0)", "pyasn1", "pyopenssl (>=16.0.0)", "pyserial (>=3.0)", "pywin32 (!=226)", "service-identity (>=18.1.0)"] conch = ["appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "cryptography (>=2.6)", "pyasn1"] -conch_nacl = ["PyNaCl", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "cryptography (>=2.6)", "pyasn1"] +conch-nacl = ["PyNaCl", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "cryptography (>=2.6)", "pyasn1"] contextvars = ["contextvars (>=2.4,<3)"] dev = ["coverage (>=6b1,<7)", "pydoctor (>=21.9.0,<21.10.0)", "pyflakes (>=2.2,<3.0)", "python-subunit (>=1.4,<2.0)", "readthedocs-sphinx-ext (>=2.1,<3.0)", "sphinx (>=4.1.2,<6)", "sphinx-rtd-theme (>=0.5,<1.0)", "towncrier (>=19.2,<20.0)", "twistedchecker (>=0.7,<1.0)"] -dev_release = ["pydoctor (>=21.9.0,<21.10.0)", "readthedocs-sphinx-ext (>=2.1,<3.0)", "sphinx (>=4.1.2,<6)", "sphinx-rtd-theme (>=0.5,<1.0)", "towncrier (>=19.2,<20.0)"] +dev-release = ["pydoctor (>=21.9.0,<21.10.0)", "readthedocs-sphinx-ext (>=2.1,<3.0)", "sphinx (>=4.1.2,<6)", "sphinx-rtd-theme (>=0.5,<1.0)", "towncrier (>=19.2,<20.0)"] http2 = ["h2 (>=3.0,<5.0)", "priority (>=1.1.0,<2.0)"] -macos_platform = ["PyHamcrest (>=1.9.0)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "contextvars (>=2.4,<3)", "cryptography (>=2.6)", "cython-test-exception-raiser (>=1.0.2,<2)", "h2 (>=3.0,<5.0)", "idna (>=2.4)", "priority (>=1.1.0,<2.0)", "pyasn1", "pyobjc-core", "pyobjc-framework-CFNetwork", "pyobjc-framework-Cocoa", "pyopenssl (>=16.0.0)", "pyserial (>=3.0)", "pywin32 (!=226)", "service-identity (>=18.1.0)"] +macos-platform = ["PyHamcrest (>=1.9.0)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "contextvars (>=2.4,<3)", "cryptography (>=2.6)", "cython-test-exception-raiser (>=1.0.2,<2)", "h2 (>=3.0,<5.0)", "idna (>=2.4)", "priority (>=1.1.0,<2.0)", "pyasn1", "pyobjc-core", "pyobjc-framework-CFNetwork", "pyobjc-framework-Cocoa", "pyopenssl (>=16.0.0)", "pyserial (>=3.0)", "pywin32 (!=226)", "service-identity (>=18.1.0)"] mypy = ["PyHamcrest (>=1.9.0)", "PyNaCl", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "contextvars (>=2.4,<3)", "coverage (>=6b1,<7)", "cryptography (>=2.6)", "cython-test-exception-raiser (>=1.0.2,<2)", "h2 (>=3.0,<5.0)", "idna (>=2.4)", "mypy (==0.930)", "mypy-zope (==0.3.4)", "priority (>=1.1.0,<2.0)", "pyasn1", "pydoctor (>=21.9.0,<21.10.0)", "pyflakes (>=2.2,<3.0)", "pyopenssl (>=16.0.0)", "pyserial (>=3.0)", "python-subunit (>=1.4,<2.0)", "pywin32 (!=226)", "readthedocs-sphinx-ext (>=2.1,<3.0)", "service-identity (>=18.1.0)", "sphinx (>=4.1.2,<6)", "sphinx-rtd-theme (>=0.5,<1.0)", "towncrier (>=19.2,<20.0)", "twistedchecker (>=0.7,<1.0)", "types-pyOpenSSL", "types-setuptools"] -osx_platform = ["PyHamcrest (>=1.9.0)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "contextvars (>=2.4,<3)", "cryptography (>=2.6)", "cython-test-exception-raiser (>=1.0.2,<2)", "h2 (>=3.0,<5.0)", "idna (>=2.4)", "priority (>=1.1.0,<2.0)", "pyasn1", "pyobjc-core", "pyobjc-framework-CFNetwork", "pyobjc-framework-Cocoa", "pyopenssl (>=16.0.0)", "pyserial (>=3.0)", "pywin32 (!=226)", "service-identity (>=18.1.0)"] +osx-platform = ["PyHamcrest (>=1.9.0)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "contextvars (>=2.4,<3)", "cryptography (>=2.6)", "cython-test-exception-raiser (>=1.0.2,<2)", "h2 (>=3.0,<5.0)", "idna (>=2.4)", "priority (>=1.1.0,<2.0)", "pyasn1", "pyobjc-core", "pyobjc-framework-CFNetwork", "pyobjc-framework-Cocoa", "pyopenssl (>=16.0.0)", "pyserial (>=3.0)", "pywin32 (!=226)", "service-identity (>=18.1.0)"] serial = ["pyserial (>=3.0)", "pywin32 (!=226)"] test = ["PyHamcrest (>=1.9.0)", "cython-test-exception-raiser (>=1.0.2,<2)"] tls = ["idna (>=2.4)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)"] -windows_platform = ["PyHamcrest (>=1.9.0)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "contextvars (>=2.4,<3)", "cryptography (>=2.6)", "cython-test-exception-raiser (>=1.0.2,<2)", "h2 (>=3.0,<5.0)", "idna (>=2.4)", "priority (>=1.1.0,<2.0)", "pyasn1", "pyopenssl (>=16.0.0)", "pyserial (>=3.0)", "pywin32 (!=226)", "pywin32 (!=226)", "service-identity (>=18.1.0)"] +windows-platform = ["PyHamcrest (>=1.9.0)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "contextvars (>=2.4,<3)", "cryptography (>=2.6)", "cython-test-exception-raiser (>=1.0.2,<2)", "h2 (>=3.0,<5.0)", "idna (>=2.4)", "priority (>=1.1.0,<2.0)", "pyasn1", "pyopenssl (>=16.0.0)", "pyserial (>=3.0)", "pywin32 (!=226)", "pywin32 (!=226)", "service-identity (>=18.1.0)"] [[package]] name = "twisted-iocpsupport" @@ -1071,7 +1082,7 @@ opentelemetry = ["opentelemetry-api", "opentelemetry-sdk"] [metadata] lock-version = "1.1" python-versions = "^3.7" -content-hash = "b7760614decf80c88ae7969affe1d734497ff51249f140bc7bdd6641ab0f39ff" +content-hash = "964270048c4fba33698095dca6fdee978f4f53adb6df027eebf78af48bee9650" [metadata.files] appdirs = [ @@ -1630,6 +1641,10 @@ pytest-timeout = [ {file = "pytest-timeout-2.1.0.tar.gz", hash = "sha256:c07ca07404c612f8abbe22294b23c368e2e5104b521c1790195561f37e1ac3d9"}, {file = "pytest_timeout-2.1.0-py3-none-any.whl", hash = "sha256:f6f50101443ce70ad325ceb4473c4255e9d74e3c7cd0ef827309dfa4c0d975c6"}, ] +python-dateutil = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] pywin32-ctypes = [ {file = "pywin32-ctypes-0.2.0.tar.gz", hash = "sha256:24ffc3b341d457d48e8922352130cf2644024a4ff09762a2261fd34c36ee5942"}, {file = "pywin32_ctypes-0.2.0-py2.py3-none-any.whl", hash = "sha256:9dc2d991b3479cc2df15930958b674a48a227d5361d413827a4cfd0b5876fc98"}, diff --git a/pyproject.toml b/pyproject.toml index 607cf544..bc9a3dd0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,6 +30,7 @@ opentelemetry-api = { version = "^1.11.1", optional = true } opentelemetry-sdk = { version = "^1.11.1", optional = true } protobuf = "~4.21" python = "^3.7" +python-dateutil = { version = "^2.8.2", python = "<3.11" } types-protobuf = "~3.20" typing-extensions = "^4.2.0" @@ -58,6 +59,9 @@ twine = "^4.0.1" opentelemetry = ["opentelemetry-api", "opentelemetry-sdk"] grpc = ["grpc"] +[tool.poetry.group.dev.dependencies] +python-dateutil = "^2.8.2" + [tool.poe.tasks] build-develop = ["build-bridge-develop"] build-bridge-develop = "python scripts/setup_bridge.py develop" diff --git a/tests/test_converter.py b/tests/test_converter.py index 1183a98d..9f21be59 100644 --- a/tests/test_converter.py +++ b/tests/test_converter.py @@ -194,6 +194,12 @@ def payload(key, dtype, data, encoding=None): ) assert dttz_check["dt"][0] == datetime(2020, 1, 1, 0, 0, 0, tzinfo=timezone.utc) + # Check timezone aware, hour offset + dttz_check = temporalio.converter.decode_search_attributes( + payload("dt", "Datetime", '"2020-01-01T00:00:00+00:00"') + ) + assert dttz_check["dt"][0] == datetime(2020, 1, 1, 0, 0, 0, tzinfo=timezone.utc) + NewIntType = NewType("NewIntType", int) MyDataClassAlias = MyDataClass From 1b2988920d7dd7c1faede9e39e182e8bc5aaa2f1 Mon Sep 17 00:00:00 2001 From: Oracen Date: Tue, 1 Nov 2022 14:06:59 +0000 Subject: [PATCH 3/9] Add in date conversion utility --- temporalio/converter.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/temporalio/converter.py b/temporalio/converter.py index 6abebbb3..0d98e250 100644 --- a/temporalio/converter.py +++ b/temporalio/converter.py @@ -7,12 +7,14 @@ import dataclasses import inspect import json +import sys from abc import ABC, abstractmethod from dataclasses import dataclass from datetime import datetime from enum import IntEnum from typing import ( Any, + Callable, Dict, List, Mapping, @@ -677,6 +679,21 @@ def encode_search_attribute_values( return default().payload_converter.to_payloads([safe_vals])[0] +def get_iso_datetime_parser() -> Callable[[str], datetime]: + """Isolates system version check and returns relevant datetime passer + + Returns: + A callable to parse date strings into datetimes. + """ + if sys.version_info >= (3, 11): + return datetime.fromisoformat # noqa + else: + # Isolate import for py > 3.11, as dependency only installed for < 3.11 + from dateutil import parser + + return parser.isoparse + + def decode_search_attributes( api: temporalio.api.common.v1.SearchAttributes, ) -> temporalio.common.SearchAttributes: @@ -697,7 +714,8 @@ def decode_search_attributes( val = [val] # Convert each item to datetime if necessary if v.metadata.get("type") == b"Datetime": - val = [datetime.fromisoformat(v) for v in val] + parser = get_iso_datetime_parser() + val = [parser(v) for v in val] ret[k] = val return ret From ae98003e04375854f2b667667302d755dc483f4e Mon Sep 17 00:00:00 2001 From: Oracen Date: Tue, 1 Nov 2022 23:54:12 +0000 Subject: [PATCH 4/9] Changes made in response to feedback --- poetry.lock | 2 +- pyproject.toml | 3 --- temporalio/converter.py | 9 +++++---- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/poetry.lock b/poetry.lock index 6a044bd7..32d14243 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1082,7 +1082,7 @@ opentelemetry = ["opentelemetry-api", "opentelemetry-sdk"] [metadata] lock-version = "1.1" python-versions = "^3.7" -content-hash = "964270048c4fba33698095dca6fdee978f4f53adb6df027eebf78af48bee9650" +content-hash = "778fd2c64ea9e483463c9837e46e6a75c11ea37dd10549ed8289fa100121115b" [metadata.files] appdirs = [ diff --git a/pyproject.toml b/pyproject.toml index bc9a3dd0..1f8103d3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -59,9 +59,6 @@ twine = "^4.0.1" opentelemetry = ["opentelemetry-api", "opentelemetry-sdk"] grpc = ["grpc"] -[tool.poetry.group.dev.dependencies] -python-dateutil = "^2.8.2" - [tool.poe.tasks] build-develop = ["build-bridge-develop"] build-bridge-develop = "python scripts/setup_bridge.py develop" diff --git a/temporalio/converter.py b/temporalio/converter.py index 0d98e250..1cde723e 100644 --- a/temporalio/converter.py +++ b/temporalio/converter.py @@ -35,6 +35,9 @@ import temporalio.api.common.v1 import temporalio.common +if sys.version_info < (3, 11): + from dateutil import parser + class PayloadConverter(ABC): """Base payload converter to/from multiple payloads/values.""" @@ -679,7 +682,7 @@ def encode_search_attribute_values( return default().payload_converter.to_payloads([safe_vals])[0] -def get_iso_datetime_parser() -> Callable[[str], datetime]: +def _get_iso_datetime_parser() -> Callable[[str], datetime]: """Isolates system version check and returns relevant datetime passer Returns: @@ -689,8 +692,6 @@ def get_iso_datetime_parser() -> Callable[[str], datetime]: return datetime.fromisoformat # noqa else: # Isolate import for py > 3.11, as dependency only installed for < 3.11 - from dateutil import parser - return parser.isoparse @@ -714,7 +715,7 @@ def decode_search_attributes( val = [val] # Convert each item to datetime if necessary if v.metadata.get("type") == b"Datetime": - parser = get_iso_datetime_parser() + parser = _get_iso_datetime_parser() val = [parser(v) for v in val] ret[k] = val return ret From cffa13d8243653752c7729145608ff393b6c6a48 Mon Sep 17 00:00:00 2001 From: Oracen Date: Tue, 1 Nov 2022 23:55:19 +0000 Subject: [PATCH 5/9] Added explanatory docstring --- temporalio/converter.py | 1 + 1 file changed, 1 insertion(+) diff --git a/temporalio/converter.py b/temporalio/converter.py index 1cde723e..dd33ec3f 100644 --- a/temporalio/converter.py +++ b/temporalio/converter.py @@ -36,6 +36,7 @@ import temporalio.common if sys.version_info < (3, 11): + # Python's datetime.fromisoformat doesn't support certain formats pre-3.11 from dateutil import parser From aaba83a911aeef868c2cf30c943c22833d09b924 Mon Sep 17 00:00:00 2001 From: Oracen Date: Wed, 2 Nov 2022 11:39:12 +0000 Subject: [PATCH 6/9] Added integration test to ensure parser could correctly handle UTC dates --- tests/worker/test_workflow.py | 57 ++++++++++++++++++++++++----------- 1 file changed, 39 insertions(+), 18 deletions(-) diff --git a/tests/worker/test_workflow.py b/tests/worker/test_workflow.py index c2de8630..351187a4 100644 --- a/tests/worker/test_workflow.py +++ b/tests/worker/test_workflow.py @@ -22,6 +22,7 @@ Tuple, cast, ) +from unittest import mock import pytest from google.protobuf.timestamp_pb2 import Timestamp @@ -1344,6 +1345,20 @@ def do_search_attribute_update(self) -> None: ) +def mock_modify_value_datetime_z(inputs: bytes) -> WorkflowActivation: + """Adds a check in case the server sends a 'Z' as part of its UTC format""" + job_act = WorkflowActivation.FromString(inputs) + for job in job_act.jobs: + if not job.HasField("start_workflow"): + continue + for item in job.start_workflow.search_attributes.indexed_fields.values(): + if item.metadata.get("type") != b"Datetime": + # Only looking for dates + continue + item.data = item.data.replace(b"+00:00", b"Z") + return job_act + + async def test_workflow_search_attributes(client: Client, env_type: str): if env_type != "local": pytest.skip("Only testing search attributes on local which disables cache") @@ -1372,24 +1387,30 @@ async def search_attributes_present() -> bool: assert await search_attributes_present() async with new_worker(client, SearchAttributeWorkflow) as worker: - handle = await client.start_workflow( - SearchAttributeWorkflow.run, - id=f"workflow-{uuid.uuid4()}", - task_queue=worker.task_queue, - search_attributes={ - f"{sa_prefix}text": ["text1", "text2", "text0"], - f"{sa_prefix}keyword": ["keyword1"], - f"{sa_prefix}int": [123], - f"{sa_prefix}double": [456.78], - f"{sa_prefix}bool": [True], - f"{sa_prefix}datetime": [ - # With UTC - datetime(2001, 2, 3, 4, 5, 6, tzinfo=timezone.utc), - # With other offset - datetime(2002, 3, 4, 5, 6, 7, tzinfo=timezone(timedelta(hours=8))), - ], - }, - ) + with mock.patch( + "temporalio.bridge.proto.workflow_activation.WorkflowActivation.FromString", + side_effect=mock_modify_value_datetime_z, + ): + handle = await client.start_workflow( + SearchAttributeWorkflow.run, + id=f"workflow-{uuid.uuid4()}", + task_queue=worker.task_queue, + search_attributes={ + f"{sa_prefix}text": ["text1", "text2", "text0"], + f"{sa_prefix}keyword": ["keyword1"], + f"{sa_prefix}int": [123], + f"{sa_prefix}double": [456.78], + f"{sa_prefix}bool": [True], + f"{sa_prefix}datetime": [ + # With UTC + datetime(2001, 2, 3, 4, 5, 6, tzinfo=timezone.utc), + # With other offset + datetime( + 2002, 3, 4, 5, 6, 7, tzinfo=timezone(timedelta(hours=8)) + ), + ], + }, + ) # Make sure it started with the right attributes expected = { f"{sa_prefix}text": {"type": "str", "values": ["text1", "text2", "text0"]}, From 124c0947282db74bd4269520565478e3decbb86c Mon Sep 17 00:00:00 2001 From: Oracen Date: Wed, 2 Nov 2022 11:47:45 +0000 Subject: [PATCH 7/9] Added types-python-dateutil to dependencies --- poetry.lock | 14 +++++++++++++- pyproject.toml | 1 + 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/poetry.lock b/poetry.lock index 63a628c9..27dea5d6 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1021,6 +1021,14 @@ category = "main" optional = false python-versions = "*" +[[package]] +name = "types-python-dateutil" +version = "2.8.19.2" +description = "Typing stubs for python-dateutil" +category = "main" +optional = false +python-versions = "*" + [[package]] name = "typing-extensions" version = "4.3.0" @@ -1093,7 +1101,7 @@ opentelemetry = ["opentelemetry-api", "opentelemetry-sdk"] [metadata] lock-version = "1.1" python-versions = "^3.7" -content-hash = "778fd2c64ea9e483463c9837e46e6a75c11ea37dd10549ed8289fa100121115b" +content-hash = "cc2ff7d7b11a6b6987984861e6d3c226bc9c48da93ca2900d850873040d08dc9" [metadata.files] appdirs = [ @@ -1802,6 +1810,10 @@ types-protobuf = [ {file = "types-protobuf-3.20.3.tar.gz", hash = "sha256:aae6404a9c902970da33977d1bc44cebae4957a5ceeced650b9afafb36051bb2"}, {file = "types_protobuf-3.20.3-py3-none-any.whl", hash = "sha256:7344c65ff8ff915e5e997e0ac16b1c0ad6ee7fca11df5a64f550aedb264fdb23"}, ] +types-python-dateutil = [ + {file = "types-python-dateutil-2.8.19.2.tar.gz", hash = "sha256:e6e32ce18f37765b08c46622287bc8d8136dc0c562d9ad5b8fd158c59963d7a7"}, + {file = "types_python_dateutil-2.8.19.2-py3-none-any.whl", hash = "sha256:3f4dbe465e7e0c6581db11fd7a4855d1355b78712b3f292bd399cd332247e9c0"}, +] typing-extensions = [ {file = "typing_extensions-4.3.0-py3-none-any.whl", hash = "sha256:25642c956049920a5aa49edcdd6ab1e06d7e5d467fc00e0506c44ac86fbfca02"}, {file = "typing_extensions-4.3.0.tar.gz", hash = "sha256:e6d2677a32f47fc7eb2795db1dd15c1f34eff616bcaf2cfb5e997f854fa1c4a6"}, diff --git a/pyproject.toml b/pyproject.toml index f3a601ca..a5055c60 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,6 +32,7 @@ protobuf = "~4.21" python = "^3.7" python-dateutil = { version = "^2.8.2", python = "<3.11" } types-protobuf = "~3.20" +types-python-dateutil = { version = "^2.8.19.2", python = "<3.11" } typing-extensions = "^4.2.0" [tool.poetry.dev-dependencies] From 26d255d829a4e1bdbeb52e43f49bd6cd05c6b5ee Mon Sep 17 00:00:00 2001 From: Oracen Date: Wed, 2 Nov 2022 23:11:16 +0000 Subject: [PATCH 8/9] Remove integration test --- tests/worker/test_workflow.py | 57 +++++++++++------------------------ 1 file changed, 18 insertions(+), 39 deletions(-) diff --git a/tests/worker/test_workflow.py b/tests/worker/test_workflow.py index 351187a4..c2de8630 100644 --- a/tests/worker/test_workflow.py +++ b/tests/worker/test_workflow.py @@ -22,7 +22,6 @@ Tuple, cast, ) -from unittest import mock import pytest from google.protobuf.timestamp_pb2 import Timestamp @@ -1345,20 +1344,6 @@ def do_search_attribute_update(self) -> None: ) -def mock_modify_value_datetime_z(inputs: bytes) -> WorkflowActivation: - """Adds a check in case the server sends a 'Z' as part of its UTC format""" - job_act = WorkflowActivation.FromString(inputs) - for job in job_act.jobs: - if not job.HasField("start_workflow"): - continue - for item in job.start_workflow.search_attributes.indexed_fields.values(): - if item.metadata.get("type") != b"Datetime": - # Only looking for dates - continue - item.data = item.data.replace(b"+00:00", b"Z") - return job_act - - async def test_workflow_search_attributes(client: Client, env_type: str): if env_type != "local": pytest.skip("Only testing search attributes on local which disables cache") @@ -1387,30 +1372,24 @@ async def search_attributes_present() -> bool: assert await search_attributes_present() async with new_worker(client, SearchAttributeWorkflow) as worker: - with mock.patch( - "temporalio.bridge.proto.workflow_activation.WorkflowActivation.FromString", - side_effect=mock_modify_value_datetime_z, - ): - handle = await client.start_workflow( - SearchAttributeWorkflow.run, - id=f"workflow-{uuid.uuid4()}", - task_queue=worker.task_queue, - search_attributes={ - f"{sa_prefix}text": ["text1", "text2", "text0"], - f"{sa_prefix}keyword": ["keyword1"], - f"{sa_prefix}int": [123], - f"{sa_prefix}double": [456.78], - f"{sa_prefix}bool": [True], - f"{sa_prefix}datetime": [ - # With UTC - datetime(2001, 2, 3, 4, 5, 6, tzinfo=timezone.utc), - # With other offset - datetime( - 2002, 3, 4, 5, 6, 7, tzinfo=timezone(timedelta(hours=8)) - ), - ], - }, - ) + handle = await client.start_workflow( + SearchAttributeWorkflow.run, + id=f"workflow-{uuid.uuid4()}", + task_queue=worker.task_queue, + search_attributes={ + f"{sa_prefix}text": ["text1", "text2", "text0"], + f"{sa_prefix}keyword": ["keyword1"], + f"{sa_prefix}int": [123], + f"{sa_prefix}double": [456.78], + f"{sa_prefix}bool": [True], + f"{sa_prefix}datetime": [ + # With UTC + datetime(2001, 2, 3, 4, 5, 6, tzinfo=timezone.utc), + # With other offset + datetime(2002, 3, 4, 5, 6, 7, tzinfo=timezone(timedelta(hours=8))), + ], + }, + ) # Make sure it started with the right attributes expected = { f"{sa_prefix}text": {"type": "str", "values": ["text1", "text2", "text0"]}, From 01254824d91028d30f86b7a38f6b727ad01713c0 Mon Sep 17 00:00:00 2001 From: Oracen Date: Sun, 6 Nov 2022 11:24:22 +0000 Subject: [PATCH 9/9] Adding explicit test ignore to suppress mypy error --- poetry.lock | 14 +------------- pyproject.toml | 3 +-- temporalio/converter.py | 2 +- 3 files changed, 3 insertions(+), 16 deletions(-) diff --git a/poetry.lock b/poetry.lock index 27dea5d6..b3ed4ddf 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1021,14 +1021,6 @@ category = "main" optional = false python-versions = "*" -[[package]] -name = "types-python-dateutil" -version = "2.8.19.2" -description = "Typing stubs for python-dateutil" -category = "main" -optional = false -python-versions = "*" - [[package]] name = "typing-extensions" version = "4.3.0" @@ -1101,7 +1093,7 @@ opentelemetry = ["opentelemetry-api", "opentelemetry-sdk"] [metadata] lock-version = "1.1" python-versions = "^3.7" -content-hash = "cc2ff7d7b11a6b6987984861e6d3c226bc9c48da93ca2900d850873040d08dc9" +content-hash = "06e8564bb255429df708580d9d6b1ec9364fda0914282725d935424688954761" [metadata.files] appdirs = [ @@ -1810,10 +1802,6 @@ types-protobuf = [ {file = "types-protobuf-3.20.3.tar.gz", hash = "sha256:aae6404a9c902970da33977d1bc44cebae4957a5ceeced650b9afafb36051bb2"}, {file = "types_protobuf-3.20.3-py3-none-any.whl", hash = "sha256:7344c65ff8ff915e5e997e0ac16b1c0ad6ee7fca11df5a64f550aedb264fdb23"}, ] -types-python-dateutil = [ - {file = "types-python-dateutil-2.8.19.2.tar.gz", hash = "sha256:e6e32ce18f37765b08c46622287bc8d8136dc0c562d9ad5b8fd158c59963d7a7"}, - {file = "types_python_dateutil-2.8.19.2-py3-none-any.whl", hash = "sha256:3f4dbe465e7e0c6581db11fd7a4855d1355b78712b3f292bd399cd332247e9c0"}, -] typing-extensions = [ {file = "typing_extensions-4.3.0-py3-none-any.whl", hash = "sha256:25642c956049920a5aa49edcdd6ab1e06d7e5d467fc00e0506c44ac86fbfca02"}, {file = "typing_extensions-4.3.0.tar.gz", hash = "sha256:e6d2677a32f47fc7eb2795db1dd15c1f34eff616bcaf2cfb5e997f854fa1c4a6"}, diff --git a/pyproject.toml b/pyproject.toml index a5055c60..0f78e979 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,7 +32,6 @@ protobuf = "~4.21" python = "^3.7" python-dateutil = { version = "^2.8.2", python = "<3.11" } types-protobuf = "~3.20" -types-python-dateutil = { version = "^2.8.19.2", python = "<3.11" } typing-extensions = "^4.2.0" [tool.poetry.dev-dependencies] @@ -123,7 +122,7 @@ ignore_missing_imports = true exclude = [ # Ignore generated code 'temporalio/api', - 'temporalio/bridge/proto', + 'temporalio/bridge/proto' ] [tool.pydocstyle] diff --git a/temporalio/converter.py b/temporalio/converter.py index dd33ec3f..127aa2ae 100644 --- a/temporalio/converter.py +++ b/temporalio/converter.py @@ -37,7 +37,7 @@ if sys.version_info < (3, 11): # Python's datetime.fromisoformat doesn't support certain formats pre-3.11 - from dateutil import parser + from dateutil import parser # type: ignore class PayloadConverter(ABC):