From d5cd2cdd5aa62042f2ffa0673bfaaaed0fe3d6ea Mon Sep 17 00:00:00 2001 From: Tatiana Al-Chueyr Date: Tue, 19 Jul 2022 14:44:36 +0100 Subject: [PATCH] Fix native snowflake load_file --- src/astro/databases/snowflake.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/astro/databases/snowflake.py b/src/astro/databases/snowflake.py index 7c9df07961..8e509412c8 100644 --- a/src/astro/databases/snowflake.py +++ b/src/astro/databases/snowflake.py @@ -37,6 +37,9 @@ FileType.PARQUET: "MATCH_BY_COLUMN_NAME=CASE_INSENSITIVE", } +NATIVE_LOAD_SUPPORTED_FILE_TYPES = (FileType.CSV, FileType.NDJSON, FileType.PARQUET) +NATIVE_LOAD_SUPPORTED_FILE_LOCATIONS = (FileLocation.GS, FileLocation.S3) + @dataclass class SnowflakeStage: @@ -292,6 +295,23 @@ def drop_stage(self, stage: SnowflakeStage) -> None: # Table load methods # --------------------------------------------------------- + def is_native_load_file_available( + self, source_file: File, target_table: Table + ) -> bool: + """ + Check if there is an optimised path for source to destination. + + :param source_file: File from which we need to transfer data + :param target_table: Table that needs to be populated with file data + """ + is_file_type_supported = ( + source_file.type.name in NATIVE_LOAD_SUPPORTED_FILE_TYPES + ) + is_file_location_supported = ( + source_file.location.location_type in NATIVE_LOAD_SUPPORTED_FILE_LOCATIONS + ) + return is_file_type_supported and is_file_location_supported + def load_file_to_table_natively( self, source_file: File, @@ -325,6 +345,9 @@ def load_file_to_table_natively( `_ """ + import pdb + + pdb.set_trace() native_support_kwargs = native_support_kwargs or {} storage_integration = native_support_kwargs.get("storage_integration") stage = self.create_stage(