diff --git a/airflow/config_templates/airflow_local_settings.py b/airflow/config_templates/airflow_local_settings.py index 6684fd18e51a0..ea8a19e80c334 100644 --- a/airflow/config_templates/airflow_local_settings.py +++ b/airflow/config_templates/airflow_local_settings.py @@ -38,6 +38,10 @@ LOG_FORMAT: str = conf.get_mandatory_value('logging', 'LOG_FORMAT') +LOG_FORMATTER_CLASS: str = conf.get_mandatory_value( + 'logging', 'LOG_FORMATTER_CLASS', fallback='airflow.utils.log.timezone_aware.TimezoneAware' +) + COLORED_LOG_FORMAT: str = conf.get_mandatory_value('logging', 'COLORED_LOG_FORMAT') COLORED_LOG: bool = conf.getboolean('logging', 'COLORED_CONSOLE_LOG') @@ -60,10 +64,13 @@ 'version': 1, 'disable_existing_loggers': False, 'formatters': { - 'airflow': {'format': LOG_FORMAT}, + 'airflow': { + 'format': LOG_FORMAT, + 'class': LOG_FORMATTER_CLASS, + }, 'airflow_coloured': { 'format': COLORED_LOG_FORMAT if COLORED_LOG else LOG_FORMAT, - 'class': COLORED_FORMATTER_CLASS if COLORED_LOG else 'logging.Formatter', + 'class': COLORED_FORMATTER_CLASS if COLORED_LOG else LOG_FORMATTER_CLASS, }, }, 'filters': { diff --git a/airflow/config_templates/config.yml b/airflow/config_templates/config.yml index 133ba65755088..ac1b5cb34ef03 100644 --- a/airflow/config_templates/config.yml +++ b/airflow/config_templates/config.yml @@ -633,6 +633,12 @@ type: string example: ~ default: "%%(asctime)s %%(levelname)s - %%(message)s" + - name: log_formatter_class + description: ~ + version_added: 2.3.4 + type: string + example: ~ + default: "airflow.utils.log.timezone_aware.TimezoneAware" - name: task_log_prefix_template description: | Specify prefix pattern like mentioned below with stream handler TaskHandlerWithCustomFormatter diff --git a/airflow/config_templates/default_airflow.cfg b/airflow/config_templates/default_airflow.cfg index 12091b62e93a4..5eee957dddc9f 100644 --- a/airflow/config_templates/default_airflow.cfg +++ b/airflow/config_templates/default_airflow.cfg @@ -349,6 +349,7 @@ colored_formatter_class = airflow.utils.log.colored_log.CustomTTYColoredFormatte # Format of Log line log_format = [%%(asctime)s] {{%%(filename)s:%%(lineno)d}} %%(levelname)s - %%(message)s simple_log_format = %%(asctime)s %%(levelname)s - %%(message)s +log_formatter_class = airflow.utils.log.timezone_aware.TimezoneAware # Specify prefix pattern like mentioned below with stream handler TaskHandlerWithCustomFormatter # Example: task_log_prefix_template = {{ti.dag_id}}-{{ti.task_id}}-{{execution_date}}-{{try_number}} diff --git a/airflow/utils/log/timezone_aware.py b/airflow/utils/log/timezone_aware.py new file mode 100644 index 0000000000000..d01205233a21b --- /dev/null +++ b/airflow/utils/log/timezone_aware.py @@ -0,0 +1,49 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +import logging + +import pendulum + + +class TimezoneAware(logging.Formatter): + """ + Override `default_time_format`, `default_msec_format` and `formatTime` to specify utc offset. + utc offset is the matter, without it, time conversion could be wrong. + With this Formatter, `%(asctime)s` will be formatted containing utc offset. (ISO 8601) + (e.g. 2022-06-12T13:00:00.123+0000) + """ + + default_time_format = '%Y-%m-%dT%H:%M:%S' + default_msec_format = '%s.%03d' + default_tz_format = '%z' + + def formatTime(self, record, datefmt=None): + """ + Returns the creation time of the specified LogRecord in ISO 8601 date and time format + in the local time zone. + """ + dt = pendulum.from_timestamp(record.created, tz=pendulum.local_timezone()) + if datefmt: + s = dt.strftime(datefmt) + else: + s = dt.strftime(self.default_time_format) + + if self.default_msec_format: + s = self.default_msec_format % (s, record.msecs) + if self.default_tz_format: + s += dt.strftime(self.default_tz_format) + return s diff --git a/airflow/www/static/js/ti_log.js b/airflow/www/static/js/ti_log.js index 1bf6b501a659c..ae72837345fe0 100644 --- a/airflow/www/static/js/ti_log.js +++ b/airflow/www/static/js/ti_log.js @@ -103,6 +103,7 @@ function autoTailingLog(tryNumber, metadata = null, autoTailing = false) { // Detect urls and log timestamps const urlRegex = /http(s)?:\/\/[\w.-]+(\.?:[\w.-]+)*([/?#][\w\-._~:/?#[\]@!$&'()*+,;=.%]+)?/g; const dateRegex = /\d{4}[./-]\d{2}[./-]\d{2} \d{2}:\d{2}:\d{2},\d{3}/g; + const iso8601Regex = /\d{4}[./-]\d{2}[./-]\d{2}T\d{2}:\d{2}:\d{2}.\d{3}[+-]\d{4}/g; res.message.forEach((item) => { const logBlockElementId = `try-${tryNumber}-${item[0]}`; @@ -120,7 +121,8 @@ function autoTailingLog(tryNumber, metadata = null, autoTailing = false) { const escapedMessage = escapeHtml(item[1]); const linkifiedMessage = escapedMessage .replace(urlRegex, (url) => `${url}`) - .replaceAll(dateRegex, (date) => ``); + .replaceAll(dateRegex, (date) => ``) + .replaceAll(iso8601Regex, (date) => ``); logBlock.innerHTML += `${linkifiedMessage}\n`; }); diff --git a/newsfragments/24811.significant.rst b/newsfragments/24811.significant.rst new file mode 100644 index 0000000000000..cb7208843c10a --- /dev/null +++ b/newsfragments/24811.significant.rst @@ -0,0 +1,22 @@ +Added new config ``[logging]log_formatter_class`` to fix timezone display for logs on UI + +If you are using a custom Formatter subclass in your ``[logging]logging_config_class``, please inherit from ``airflow.utils.log.timezone_aware.TimezoneAware`` instead of ``logging.Formatter``. +For example, in your ``custom_config.py``: + +.. code-block:: python + from airflow.utils.log.timezone_aware import TimezoneAware + + # before + class YourCustomFormatter(logging.Formatter): + ... + + + # after + class YourCustomFormatter(TimezoneAware): + ... + + + AIRFLOW_FORMATTER = LOGGING_CONFIG["formatters"]["airflow"] + AIRFLOW_FORMATTER["class"] = "somewhere.your.custom_config.YourCustomFormatter" + # or use TimezoneAware class directly. If you don't have custom Formatter. + AIRFLOW_FORMATTER["class"] = "airflow.utils.log.timezone_aware.TimezoneAware"