Update to Airflow 2.2.5 (#28)

* Use latest actions

* Use pendulum instead of datetime for a DAG's start_date

This aligns our DAGs with the Airflow example DAGs. See apache/airflow#21646.

* Update GitHub Actions with dependabot

* Remove outdated file, directory doesn't exist any more

* Add include directory to pylint

* Update Airflow to 2.2.5

.github/dependabot.yml

@@ -10,3 +10,8 @@ updates:
     directory: "/"
     schedule:
       interval: "monthly"
+
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "monthly"

.github/workflows/main.yml

@@ -10,10 +10,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Set up Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v3
         with:
           python-version: 3.9
           cache: 'pip'
@@ -25,17 +25,17 @@ jobs:
 
       - name: Run pylint
         run: |
-          python -m pylint dags tests
+          python -m pylint dags tests include
 
   test:
     name: Run pytest
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Set up Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v3
         with:
           python-version: 3.9
           cache: 'pip'

Dockerfile

@@ -1 +1 @@
-FROM quay.io/astronomer/ap-airflow:2.2.4-3-onbuild
+FROM quay.io/astronomer/ap-airflow:2.2.5-1-onbuild

dags/data_retention_delete_dag.py

@@ -8,10 +8,10 @@
 In CrateDB, tables for storing retention policies need to be created once manually.
 See the file setup/data_retention_schema.sql in this repository.
 """
-import datetime
 import json
 import logging
 from pathlib import Path
+import pendulum
 from airflow import DAG
 from airflow.providers.postgres.operators.postgres import PostgresOperator
 from airflow.providers.postgres.hooks.postgres import PostgresHook
@@ -62,7 +62,7 @@ def delete_partitions(ti):
 
 with DAG(
     dag_id="data-retention-delete-dag",
-    start_date=datetime.datetime(2021, 11, 19),
+    start_date=pendulum.datetime(2021, 11, 19, tz="UTC"),
     schedule_interval="@daily",
     catchup=False,
 ) as dag:

dags/data_retention_reallocate_dag.py

@@ -6,10 +6,10 @@
 In CrateDB, tables for storing retention policies need to be created once manually.
 See the file setup/data_retention_schema.sql in this repository.
 """
-import datetime
 import json
 import logging
 from pathlib import Path
+import pendulum
 from airflow import DAG
 from airflow.providers.postgres.operators.postgres import PostgresOperator
 from airflow.providers.postgres.hooks.postgres import PostgresHook
@@ -85,7 +85,7 @@ def reallocate_partitions(ti):
 
 with DAG(
     dag_id="data-retention-reallocate-dag",
-    start_date=datetime.datetime(2021, 11, 19),
+    start_date=pendulum.datetime(2021, 11, 19, tz="UTC"),
     schedule_interval="@daily",
     catchup=False,
 ) as dag:

dags/data_retention_snapshot_dag.py

@@ -6,10 +6,10 @@
 In CrateDB, tables for storing retention policies need to be created once manually.
 See the file setup/data_retention_schema.sql in this repository.
 """
-import datetime
 import json
 import logging
 from pathlib import Path
+import pendulum
 from airflow import DAG
 from airflow.providers.postgres.operators.postgres import PostgresOperator
 from airflow.providers.postgres.hooks.postgres import PostgresHook
@@ -77,7 +77,7 @@ def snapshot_partitions(ti):
 
 with DAG(
     dag_id="data-retention-snapshot-dag",
-    start_date=datetime.datetime(2022, 1, 31),
+    start_date=pendulum.datetime(2022, 1, 31, tz="UTC"),
     schedule_interval="@daily",
     catchup=False,
 ) as dag:

dags/financial_data_dag.py

@@ -11,6 +11,7 @@
 import math
 import json
 import logging
+import pendulum
 import requests
 from bs4 import BeautifulSoup
 import yfinance as yf
@@ -109,7 +110,7 @@ def format_and_insert_data_function(ti):
 
 with DAG(
     dag_id="financial_data_dag",
-    start_date=datetime.datetime(2022, 1, 10),
+    start_date=pendulum.datetime(2022, 1, 10, tz="UTC"),
     schedule_interval="@daily",
     catchup=False,
 ) as dag:

dags/nyc_taxi_dag.py

@@ -10,10 +10,9 @@
 You can retrieve the CREATE TABLE statements from the file setup/taxi-schema.sql
 in this repository.
 """
-import datetime
 import logging
-
 from pathlib import Path
+import pendulum
 from airflow import DAG
 from airflow.providers.postgres.operators.postgres import PostgresOperator
 from airflow.providers.postgres.hooks.postgres import PostgresHook
@@ -86,7 +85,7 @@ def process_new_files(ti):
 
 with DAG(
     dag_id="nyc-taxi",
-    start_date=datetime.datetime(2021, 11, 11),
+    start_date=pendulum.datetime(2021, 11, 11, tz="UTC"),
     schedule_interval="@daily",
     catchup=False,
 ) as dag:

dags/table_export_dag.py

@@ -3,9 +3,8 @@
 
 A detailed tutorial is available at https://community.crate.io/t/cratedb-and-apache-airflow-automating-data-export-to-s3/901
 """
-import datetime
 import os
-
+import pendulum
 from airflow import DAG
 from airflow.providers.postgres.operators.postgres import PostgresOperator
 from airflow.utils.task_group import TaskGroup
@@ -16,7 +15,7 @@
 
 with DAG(
     dag_id="cratedb_table_export",
-    start_date=datetime.datetime(2021, 11, 11),
+    start_date=pendulum.datetime(2021, 11, 11, tz="UTC"),
     schedule_interval="@daily",
     catchup=False,
 ) as dag:

setup.py

@@ -5,7 +5,7 @@
     packages=find_packages(),
     python_requires=">=3.8",
     install_requires=[
-        "apache-airflow==2.2.4",
+        "apache-airflow==2.2.5",
         "apache-airflow-providers-postgres==4.1.0",
     ],
     extras_require={