diff --git a/.github/dependabot.yml b/.github/dependabot.yml index a4acb3c..caf8b2e 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -10,3 +10,8 @@ updates: directory: "/" schedule: interval: "monthly" + + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "monthly" diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4e89d22..79956cb 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -10,10 +10,10 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: 3.9 cache: 'pip' @@ -25,17 +25,17 @@ jobs: - name: Run pylint run: | - python -m pylint dags tests + python -m pylint dags tests include test: name: Run pytest runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: 3.9 cache: 'pip' diff --git a/Dockerfile b/Dockerfile index 498a967..71b4dd0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1 +1 @@ -FROM quay.io/astronomer/ap-airflow:2.2.4-3-onbuild +FROM quay.io/astronomer/ap-airflow:2.2.5-1-onbuild diff --git a/dags/.airflowignore b/dags/.airflowignore deleted file mode 100644 index 2696261..0000000 --- a/dags/.airflowignore +++ /dev/null @@ -1 +0,0 @@ -config/.* diff --git a/dags/data_retention_delete_dag.py b/dags/data_retention_delete_dag.py index 5212e26..a52555d 100644 --- a/dags/data_retention_delete_dag.py +++ b/dags/data_retention_delete_dag.py @@ -8,10 +8,10 @@ In CrateDB, tables for storing retention policies need to be created once manually. See the file setup/data_retention_schema.sql in this repository. """ -import datetime import json import logging from pathlib import Path +import pendulum from airflow import DAG from airflow.providers.postgres.operators.postgres import PostgresOperator from airflow.providers.postgres.hooks.postgres import PostgresHook @@ -62,7 +62,7 @@ def delete_partitions(ti): with DAG( dag_id="data-retention-delete-dag", - start_date=datetime.datetime(2021, 11, 19), + start_date=pendulum.datetime(2021, 11, 19, tz="UTC"), schedule_interval="@daily", catchup=False, ) as dag: diff --git a/dags/data_retention_reallocate_dag.py b/dags/data_retention_reallocate_dag.py index 20dfe0e..359e65a 100644 --- a/dags/data_retention_reallocate_dag.py +++ b/dags/data_retention_reallocate_dag.py @@ -6,10 +6,10 @@ In CrateDB, tables for storing retention policies need to be created once manually. See the file setup/data_retention_schema.sql in this repository. """ -import datetime import json import logging from pathlib import Path +import pendulum from airflow import DAG from airflow.providers.postgres.operators.postgres import PostgresOperator from airflow.providers.postgres.hooks.postgres import PostgresHook @@ -85,7 +85,7 @@ def reallocate_partitions(ti): with DAG( dag_id="data-retention-reallocate-dag", - start_date=datetime.datetime(2021, 11, 19), + start_date=pendulum.datetime(2021, 11, 19, tz="UTC"), schedule_interval="@daily", catchup=False, ) as dag: diff --git a/dags/data_retention_snapshot_dag.py b/dags/data_retention_snapshot_dag.py index 11893bd..07aa3c8 100644 --- a/dags/data_retention_snapshot_dag.py +++ b/dags/data_retention_snapshot_dag.py @@ -6,10 +6,10 @@ In CrateDB, tables for storing retention policies need to be created once manually. See the file setup/data_retention_schema.sql in this repository. """ -import datetime import json import logging from pathlib import Path +import pendulum from airflow import DAG from airflow.providers.postgres.operators.postgres import PostgresOperator from airflow.providers.postgres.hooks.postgres import PostgresHook @@ -77,7 +77,7 @@ def snapshot_partitions(ti): with DAG( dag_id="data-retention-snapshot-dag", - start_date=datetime.datetime(2022, 1, 31), + start_date=pendulum.datetime(2022, 1, 31, tz="UTC"), schedule_interval="@daily", catchup=False, ) as dag: diff --git a/dags/financial_data_dag.py b/dags/financial_data_dag.py index ca1e3d1..3c2de2b 100644 --- a/dags/financial_data_dag.py +++ b/dags/financial_data_dag.py @@ -11,6 +11,7 @@ import math import json import logging +import pendulum import requests from bs4 import BeautifulSoup import yfinance as yf @@ -109,7 +110,7 @@ def format_and_insert_data_function(ti): with DAG( dag_id="financial_data_dag", - start_date=datetime.datetime(2022, 1, 10), + start_date=pendulum.datetime(2022, 1, 10, tz="UTC"), schedule_interval="@daily", catchup=False, ) as dag: diff --git a/dags/nyc_taxi_dag.py b/dags/nyc_taxi_dag.py index ab96de2..05c3585 100644 --- a/dags/nyc_taxi_dag.py +++ b/dags/nyc_taxi_dag.py @@ -10,10 +10,9 @@ You can retrieve the CREATE TABLE statements from the file setup/taxi-schema.sql in this repository. """ -import datetime import logging - from pathlib import Path +import pendulum from airflow import DAG from airflow.providers.postgres.operators.postgres import PostgresOperator from airflow.providers.postgres.hooks.postgres import PostgresHook @@ -86,7 +85,7 @@ def process_new_files(ti): with DAG( dag_id="nyc-taxi", - start_date=datetime.datetime(2021, 11, 11), + start_date=pendulum.datetime(2021, 11, 11, tz="UTC"), schedule_interval="@daily", catchup=False, ) as dag: diff --git a/dags/table_export_dag.py b/dags/table_export_dag.py index 68a1a96..48f8ddd 100644 --- a/dags/table_export_dag.py +++ b/dags/table_export_dag.py @@ -3,9 +3,8 @@ A detailed tutorial is available at https://community.crate.io/t/cratedb-and-apache-airflow-automating-data-export-to-s3/901 """ -import datetime import os - +import pendulum from airflow import DAG from airflow.providers.postgres.operators.postgres import PostgresOperator from airflow.utils.task_group import TaskGroup @@ -16,7 +15,7 @@ with DAG( dag_id="cratedb_table_export", - start_date=datetime.datetime(2021, 11, 11), + start_date=pendulum.datetime(2021, 11, 11, tz="UTC"), schedule_interval="@daily", catchup=False, ) as dag: diff --git a/setup.py b/setup.py index 868c560..8672059 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ packages=find_packages(), python_requires=">=3.8", install_requires=[ - "apache-airflow==2.2.4", + "apache-airflow==2.2.5", "apache-airflow-providers-postgres==4.1.0", ], extras_require={