/
config.yml
2238 lines (2220 loc) · 74.2 KB
/
config.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
---
- name: core
description: ~
options:
- name: dags_folder
description: |
The folder where your airflow pipelines live, most likely a
subfolder in a code repository. This path must be absolute.
version_added: ~
type: string
example: ~
default: "{AIRFLOW_HOME}/dags"
- name: hostname_callable
description: |
Hostname by providing a path to a callable, which will resolve the hostname.
The format is "package.function".
For example, default value "socket.getfqdn" means that result from getfqdn() of "socket"
package will be used as hostname.
No argument should be required in the function specified.
If using IP address as hostname is preferred, use value ``airflow.utils.net.get_host_ip_address``
version_added: ~
type: string
example: ~
default: "socket.getfqdn"
- name: default_timezone
description: |
Default timezone in case supplied date times are naive
can be utc (default), system, or any IANA timezone string (e.g. Europe/Amsterdam)
version_added: ~
type: string
example: ~
default: "utc"
- name: executor
description: |
The executor class that airflow should use. Choices include
``SequentialExecutor``, ``LocalExecutor``, ``CeleryExecutor``, ``DaskExecutor``,
``KubernetesExecutor``, ``CeleryKubernetesExecutor`` or the
full import path to the class when using a custom executor.
version_added: ~
type: string
example: ~
default: "SequentialExecutor"
- name: sql_alchemy_conn
description: |
The SqlAlchemy connection string to the metadata database.
SqlAlchemy supports many different database engines.
More information here:
http://airflow.apache.org/docs/apache-airflow/stable/howto/set-up-database.html#database-uri
version_added: ~
type: string
sensitive: true
example: ~
default: "sqlite:///{AIRFLOW_HOME}/airflow.db"
- name: sql_engine_encoding
description: |
The encoding for the databases
version_added: 1.10.1
type: string
example: ~
default: "utf-8"
- name: sql_engine_collation_for_ids
description: |
Collation for ``dag_id``, ``task_id``, ``key`` columns in case they have different encoding.
This is particularly useful in case of mysql with utf8mb4 encoding because
primary keys for XCom table has too big size and ``sql_engine_collation_for_ids`` should
be set to ``utf8mb3_general_ci``.
version_added: 2.0.0
type: string
example: ~
default: ~
- name: sql_alchemy_pool_enabled
description: |
If SqlAlchemy should pool database connections.
version_added: ~
type: string
example: ~
default: "True"
- name: sql_alchemy_pool_size
description: |
The SqlAlchemy pool size is the maximum number of database connections
in the pool. 0 indicates no limit.
version_added: ~
type: string
example: ~
default: "5"
- name: sql_alchemy_max_overflow
description: |
The maximum overflow size of the pool.
When the number of checked-out connections reaches the size set in pool_size,
additional connections will be returned up to this limit.
When those additional connections are returned to the pool, they are disconnected and discarded.
It follows then that the total number of simultaneous connections the pool will allow
is pool_size + max_overflow,
and the total number of "sleeping" connections the pool will allow is pool_size.
max_overflow can be set to ``-1`` to indicate no overflow limit;
no limit will be placed on the total number of concurrent connections. Defaults to ``10``.
version_added: 1.10.4
type: string
example: ~
default: "10"
- name: sql_alchemy_pool_recycle
description: |
The SqlAlchemy pool recycle is the number of seconds a connection
can be idle in the pool before it is invalidated. This config does
not apply to sqlite. If the number of DB connections is ever exceeded,
a lower config value will allow the system to recover faster.
version_added: ~
type: string
example: ~
default: "1800"
- name: sql_alchemy_pool_pre_ping
description: |
Check connection at the start of each connection pool checkout.
Typically, this is a simple statement like "SELECT 1".
More information here:
https://docs.sqlalchemy.org/en/13/core/pooling.html#disconnect-handling-pessimistic
version_added: 1.10.6
type: string
example: ~
default: "True"
- name: sql_alchemy_schema
description: |
The schema to use for the metadata database.
SqlAlchemy supports databases with the concept of multiple schemas.
version_added: 1.10.3
type: string
example: ~
default: ""
- name: sql_alchemy_connect_args
description: |
Import path for connect args in SqlAlchemy. Defaults to an empty dict.
This is useful when you want to configure db engine args that SqlAlchemy won't parse
in connection string.
See https://docs.sqlalchemy.org/en/13/core/engines.html#sqlalchemy.create_engine.params.connect_args
version_added: 1.10.11
type: string
example: ~
default: ~
- name: parallelism
description: |
This defines the maximum number of task instances that can run concurrently in Airflow
regardless of scheduler count and worker count. Generally, this value is reflective of
the number of task instances with the running state in the metadata database.
version_added: ~
type: string
example: ~
default: "32"
- name: dag_concurrency
description: |
The maximum number of task instances allowed to run concurrently in each DAG Run for that
DAG. Can be overridden by ``concurrency`` on DAG level.
version_added: ~
type: string
example: ~
default: "16"
- name: dags_are_paused_at_creation
description: |
Are DAGs paused by default at creation
version_added: ~
type: string
example: ~
default: "True"
- name: max_active_runs_per_dag
description: |
The maximum number of active DAG runs per DAG. The scheduler will not create more DAG runs
if it reaches the limit. This is configurable at the DAG level with ``max_active_runs``,
which is defaulted as ``max_active_runs_per_dag``.
version_added: ~
type: string
example: ~
default: "16"
- name: load_examples
description: |
Whether to load the DAG examples that ship with Airflow. It's good to
get started, but you probably want to set this to ``False`` in a production
environment
version_added: ~
type: string
example: ~
default: "True"
- name: load_default_connections
description: |
Whether to load the default connections that ship with Airflow. It's good to
get started, but you probably want to set this to ``False`` in a production
environment
version_added: 1.10.10
type: string
example: ~
default: "True"
- name: plugins_folder
description: |
Path to the folder containing Airflow plugins
version_added: ~
type: string
example: ~
default: "{AIRFLOW_HOME}/plugins"
- name: execute_tasks_new_python_interpreter
description: |
Should tasks be executed via forking of the parent process ("False",
the speedier option) or by spawning a new python process ("True" slow,
but means plugin changes picked up by tasks straight away)
default: "False"
example: ~
version_added: "2.0.0"
see_also: ":ref:`plugins:loading`"
type: boolean
- name: fernet_key
description: |
Secret key to save connection passwords in the db
version_added: ~
type: string
sensitive: true
example: ~
default: "{FERNET_KEY}"
- name: donot_pickle
description: |
Whether to disable pickling dags
version_added: ~
type: string
example: ~
default: "True"
- name: dagbag_import_timeout
description: |
How long before timing out a python file import
version_added: ~
type: float
example: ~
default: "30.0"
- name: dagbag_import_error_tracebacks
description: |
Should a traceback be shown in the UI for dagbag import errors,
instead of just the exception message
version_added: 2.0.0
type: boolean
example: ~
default: "True"
- name: dagbag_import_error_traceback_depth
description: |
If tracebacks are shown, how many entries from the traceback should be shown
version_added: 2.0.0
type: integer
example: ~
default: "2"
- name: dag_file_processor_timeout
description: |
How long before timing out a DagFileProcessor, which processes a dag file
version_added: 1.10.6
type: string
example: ~
default: "50"
- name: task_runner
description: |
The class to use for running task instances in a subprocess.
Choices include StandardTaskRunner, CgroupTaskRunner or the full import path to the class
when using a custom task runner.
version_added: ~
type: string
example: ~
default: "StandardTaskRunner"
- name: default_impersonation
description: |
If set, tasks without a ``run_as_user`` argument will be run with this user
Can be used to de-elevate a sudo user running Airflow when executing tasks
version_added: ~
type: string
example: ~
default: ""
- name: security
description: |
What security module to use (for example kerberos)
version_added: ~
type: string
example: ~
default: ""
- name: unit_test_mode
description: |
Turn unit test mode on (overwrites many configuration options with test
values at runtime)
version_added: ~
type: string
example: ~
default: "False"
- name: enable_xcom_pickling
description: |
Whether to enable pickling for xcom (note that this is insecure and allows for
RCE exploits).
version_added: ~
type: string
example: ~
default: "False"
see_also: "https://docs.python.org/3/library/pickle.html#comparison-with-json"
- name: killed_task_cleanup_time
description: |
When a task is killed forcefully, this is the amount of time in seconds that
it has to cleanup after it is sent a SIGTERM, before it is SIGKILLED
version_added: ~
type: string
example: ~
default: "60"
- name: dag_run_conf_overrides_params
description: |
Whether to override params with dag_run.conf. If you pass some key-value pairs
through ``airflow dags backfill -c`` or
``airflow dags trigger -c``, the key-value pairs will override the existing ones in params.
version_added: ~
type: string
example: ~
default: "True"
- name: dag_discovery_safe_mode
description: |
When discovering DAGs, ignore any files that don't contain the strings ``DAG`` and ``airflow``.
version_added: 1.10.3
type: string
example: ~
default: "True"
- name: default_task_retries
description: |
The number of retries each task is going to have by default. Can be overridden at dag or task level.
version_added: 1.10.6
type: string
example: ~
default: "0"
- name: min_serialized_dag_update_interval
description: |
Updating serialized DAG can not be faster than a minimum interval to reduce database write rate.
version_added: 1.10.7
type: string
example: ~
default: "30"
- name: min_serialized_dag_fetch_interval
description: |
Fetching serialized DAG can not be faster than a minimum interval to reduce database
read rate. This config controls when your DAGs are updated in the Webserver
version_added: 1.10.12
type: string
example: ~
default: "10"
- name: store_dag_code
description: |
Whether to persist DAG files code in DB.
If set to True, Webserver reads file contents from DB instead of
trying to access files in a DAG folder.
(Default is ``True``)
version_added: 1.10.10
type: string
example: "True"
default: "True"
- name: max_num_rendered_ti_fields_per_task
description: |
Maximum number of Rendered Task Instance Fields (Template Fields) per task to store
in the Database.
All the template_fields for each of Task Instance are stored in the Database.
Keeping this number small may cause an error when you try to view ``Rendered`` tab in
TaskInstance view for older tasks.
version_added: 2.0.0
type: integer
example: ~
default: "30"
- name: check_slas
description: |
On each dagrun check against defined SLAs
version_added: 1.10.8
type: string
example: ~
default: "True"
- name: xcom_backend
description: |
Path to custom XCom class that will be used to store and resolve operators results
version_added: 1.10.12
type: string
example: "path.to.CustomXCom"
default: "airflow.models.xcom.BaseXCom"
- name: lazy_load_plugins
description: |
By default Airflow plugins are lazily-loaded (only loaded when required). Set it to ``False``,
if you want to load plugins whenever 'airflow' is invoked via cli or loaded from module.
version_added: 2.0.0
type: boolean
example: ~
default: "True"
- name: lazy_discover_providers
description: |
By default Airflow providers are lazily-discovered (discovery and imports happen only when required).
Set it to False, if you want to discover providers whenever 'airflow' is invoked via cli or
loaded from module.
version_added: 2.0.0
type: boolean
example: ~
default: "True"
- name: max_db_retries
description: |
Number of times the code should be retried in case of DB Operational Errors.
Not all transactions will be retried as it can cause undesired state.
Currently it is only used in ``DagFileProcessor.process_file`` to retry ``dagbag.sync_to_db``.
version_added: ~
type: integer
example: ~
default: "3"
- name: hide_sensitive_var_conn_fields
description: |
Hide sensitive Variables or Connection extra json keys from UI and task logs when set to True
(Connection passwords are always hidden in logs)
version_added: ~
type: boolean
example: ~
default: "True"
- name: sensitive_var_conn_names
description: |
A comma-separated list of extra sensitive keywords to look for in variables names or connection's
extra JSON.
version_added: ~
type: string
example: ~
default: ""
- name: default_pool_task_slot_count
description: |
Task Slot counts for ``default_pool``. This setting would not have any effect in an existing
deployment where the ``default_pool`` is already created. For existing deployments, users can
change the number of slots using Webserver, API or the CLI
version_added: 2.2.0
type: string
example: ~
default: "128"
- name: logging
description: ~
options:
- name: base_log_folder
description: |
The folder where airflow should store its log files
This path must be absolute
version_added: ~
type: string
example: ~
default: "{AIRFLOW_HOME}/logs"
- name: remote_logging
description: |
Airflow can store logs remotely in AWS S3, Google Cloud Storage or Elastic Search.
Set this to True if you want to enable remote logging.
version_added: ~
type: string
example: ~
default: "False"
- name: remote_log_conn_id
description: |
Users must supply an Airflow connection id that provides access to the storage
location.
version_added: ~
type: string
example: ~
default: ""
- name: google_key_path
description: |
Path to Google Credential JSON file. If omitted, authorization based on `the Application Default
Credentials
<https://cloud.google.com/docs/authentication/production#finding_credentials_automatically>`__ will
be used.
version_added: ~
type: string
example: ~
default: ""
- name: remote_base_log_folder
description: |
Storage bucket URL for remote logging
S3 buckets should start with "s3://"
Cloudwatch log groups should start with "cloudwatch://"
GCS buckets should start with "gs://"
WASB buckets should start with "wasb" just to help Airflow select correct handler
Stackdriver logs should start with "stackdriver://"
version_added: ~
type: string
example: ~
default: ""
- name: encrypt_s3_logs
description: |
Use server-side encryption for logs stored in S3
version_added: ~
type: string
example: ~
default: "False"
- name: logging_level
description: |
Logging level.
Supported values: ``CRITICAL``, ``ERROR``, ``WARNING``, ``INFO``, ``DEBUG``.
version_added: ~
type: string
example: ~
default: "INFO"
- name: fab_logging_level
description: |
Logging level for Flask-appbuilder UI.
Supported values: ``CRITICAL``, ``ERROR``, ``WARNING``, ``INFO``, ``DEBUG``.
version_added: ~
type: string
example: ~
default: "WARN"
- name: logging_config_class
description: |
Logging class
Specify the class that will specify the logging configuration
This class has to be on the python classpath
version_added: ~
type: string
example: "my.path.default_local_settings.LOGGING_CONFIG"
default: ""
- name: colored_console_log
description: |
Flag to enable/disable Colored logs in Console
Colour the logs when the controlling terminal is a TTY.
version_added: 1.10.4
type: string
example: ~
default: "True"
- name: colored_log_format
description: |
Log format for when Colored logs is enabled
version_added: 1.10.4
type: string
example: ~
default: >-
[%%(blue)s%%(asctime)s%%(reset)s] {{%%(blue)s%%(filename)s:%%(reset)s%%(lineno)d}}
%%(log_color)s%%(levelname)s%%(reset)s - %%(log_color)s%%(message)s%%(reset)s
- name: colored_formatter_class
description: ~
version_added: 1.10.4
type: string
example: ~
default: "airflow.utils.log.colored_log.CustomTTYColoredFormatter"
- name: log_format
description: |
Format of Log line
version_added: ~
type: string
example: ~
default: "[%%(asctime)s] {{%%(filename)s:%%(lineno)d}} %%(levelname)s - %%(message)s"
- name: simple_log_format
description: ~
version_added: ~
type: string
example: ~
default: "%%(asctime)s %%(levelname)s - %%(message)s"
- name: task_log_prefix_template
description: |
Specify prefix pattern like mentioned below with stream handler TaskHandlerWithCustomFormatter
version_added: ~
type: string
example: "{{ti.dag_id}}-{{ti.task_id}}-{{execution_date}}-{{try_number}}"
default: ""
- name: log_filename_template
description: |
Formatting for how airflow generates file names/paths for each task run.
version_added: ~
type: string
example: ~
default: "{{{{ ti.dag_id }}}}/{{{{ ti.task_id }}}}/{{{{ ts }}}}/{{{{ try_number }}}}.log"
- name: log_processor_filename_template
description: |
Formatting for how airflow generates file names for log
version_added: ~
type: string
example: ~
default: "{{{{ filename }}}}.log"
- name: dag_processor_manager_log_location
description: |
full path of dag_processor_manager logfile
version_added: 1.10.2
type: string
example: ~
default: "{AIRFLOW_HOME}/logs/dag_processor_manager/dag_processor_manager.log"
- name: task_log_reader
description: |
Name of handler to read task instance logs.
Defaults to use ``task`` handler.
version_added: ~
type: string
example: ~
default: "task"
- name: extra_loggers
description: |
A comma\-separated list of third-party logger names that will be configured to print messages to
consoles\.
version_added: ~
type: string
example: "connexion,sqlalchemy"
default: ""
- name: metrics
description: |
StatsD (https://github.com/etsy/statsd) integration settings.
options:
- name: statsd_on
description: |
Enables sending metrics to StatsD.
version_added: ~
type: string
example: ~
default: "False"
- name: statsd_host
description: ~
version_added: ~
type: string
example: ~
default: "localhost"
- name: statsd_port
description: ~
version_added: ~
type: string
example: ~
default: "8125"
- name: statsd_prefix
description: ~
version_added: ~
type: string
example: ~
default: "airflow"
- name: statsd_allow_list
description: |
If you want to avoid sending all the available metrics to StatsD,
you can configure an allow list of prefixes (comma separated) to send only the metrics that
start with the elements of the list (e.g: "scheduler,executor,dagrun")
version_added: 1.10.6
type: string
example: ~
default: ""
- name: stat_name_handler
description: |
A function that validate the statsd stat name, apply changes to the stat name if necessary and return
the transformed stat name.
The function should have the following signature:
def func_name(stat_name: str) -> str:
version_added: ~
type: string
example: ~
default: ""
- name: statsd_datadog_enabled
description: |
To enable datadog integration to send airflow metrics.
version_added: ~
type: string
example: ~
default: "False"
- name: statsd_datadog_tags
description: |
List of datadog tags attached to all metrics(e.g: key1:value1,key2:value2)
version_added: ~
type: string
example: ~
default: ""
- name: statsd_custom_client_path
description: |
If you want to utilise your own custom Statsd client set the relevant
module path below.
Note: The module path must exist on your PYTHONPATH for Airflow to pick it up
version_added: ~
type: string
example: ~
default: ~
- name: secrets
description: ~
options:
- name: backend
description: |
Full class name of secrets backend to enable (will precede env vars and metastore in search path)
version_added: 1.10.10
type: string
example: "airflow.providers.amazon.aws.secrets.systems_manager.SystemsManagerParameterStoreBackend"
default: ""
- name: backend_kwargs
description: |
The backend_kwargs param is loaded into a dictionary and passed to __init__ of secrets backend class.
See documentation for the secrets backend you are using. JSON is expected.
Example for AWS Systems Manager ParameterStore:
``{{"connections_prefix": "/airflow/connections", "profile_name": "default"}}``
version_added: 1.10.10
type: string
example: ~
default: ""
- name: cli
description: ~
options:
- name: api_client
description: |
In what way should the cli access the API. The LocalClient will use the
database directly, while the json_client will use the api running on the
webserver
version_added: ~
type: string
example: ~
default: "airflow.api.client.local_client"
- name: endpoint_url
description: |
If you set web_server_url_prefix, do NOT forget to append it here, ex:
``endpoint_url = http://localhost:8080/myroot``
So api will look like: ``http://localhost:8080/myroot/api/experimental/...``
version_added: ~
type: string
example: ~
default: "http://localhost:8080"
- name: debug
description: ~
options:
- name: fail_fast
description: |
Used only with ``DebugExecutor``. If set to ``True`` DAG will fail with first
failed task. Helpful for debugging purposes.
version_added: 1.10.8
type: string
example: ~
default: "False"
- name: api
description: ~
options:
- name: enable_experimental_api
description: |
Enables the deprecated experimental API. Please note that these APIs do not have access control.
The authenticated user has full access.
.. warning::
This `Experimental REST API <https://airflow.readthedocs.io/en/latest/rest-api-ref.html>`__ is
deprecated since version 2.0. Please consider using
`the Stable REST API <https://airflow.readthedocs.io/en/latest/stable-rest-api-ref.html>`__.
For more information on migration, see
`UPDATING.md <https://github.com/apache/airflow/blob/main/UPDATING.md>`_
version_added: 2.0.0
type: boolean
example: ~
default: "False"
- name: auth_backend
description: |
How to authenticate users of the API. See
https://airflow.apache.org/docs/apache-airflow/stable/security.html for possible values.
("airflow.api.auth.backend.default" allows all requests for historic reasons)
version_added: ~
type: string
example: ~
default: "airflow.api.auth.backend.deny_all"
- name: maximum_page_limit
description: |
Used to set the maximum page limit for API requests
version_added: ~
type: integer
example: ~
default: "100"
- name: fallback_page_limit
description: |
Used to set the default page limit when limit is zero. A default limit
of 100 is set on OpenApi spec. However, this particular default limit
only work when limit is set equal to zero(0) from API requests.
If no limit is supplied, the OpenApi spec default is used.
type: integer
example: ~
version_added: ~
default: "100"
- name: google_oauth2_audience
description: The intended audience for JWT token credentials used for authorization.
This value must match on the client and server sides.
If empty, audience will not be tested.
type: string
version_added: ~
example: project-id-random-value.apps.googleusercontent.com
default: ""
- name: google_key_path
description: |
Path to Google Cloud Service Account key file (JSON). If omitted, authorization based on
`the Application Default Credentials
<https://cloud.google.com/docs/authentication/production#finding_credentials_automatically>`__ will
be used.
type: string
version_added: ~
example: /files/service-account-json
default: ""
- name: access_control_allow_headers
description: |
Used in response to a preflight request to indicate which HTTP
headers can be used when making the actual request. This header is
the server side response to the browser's
Access-Control-Request-Headers header.
type: string
version_added: ~
example: ~
default: ""
- name: access_control_allow_methods
description: |
Specifies the method or methods allowed when accessing the resource.
type: string
version_added: ~
example: ~
default: ""
- name: access_control_allow_origin
description: |
Indicates whether the response can be shared with requesting code from the given origin.
type: string
version_added: ~
example: ~
default: ""
- name: lineage
description: ~
options:
- name: backend
description: |
what lineage backend to use
version_added: ~
type: string
example: ~
default: ""
- name: atlas
description: ~
options:
- name: sasl_enabled
description: ~
version_added: ~
type: string
example: ~
default: "False"
- name: host
description: ~
version_added: ~
type: string
example: ~
default: ""
- name: port
description: ~
version_added: ~
type: string
example: ~
default: "21000"
- name: username
description: ~
version_added: ~
type: string
example: ~
default: ""
- name: password
description: ~
version_added: ~
type: string
sensitive: true
example: ~
default: ""
- name: operators
description: ~
options:
- name: default_owner
description: |
The default owner assigned to each new operator, unless
provided explicitly or passed via ``default_args``
version_added: ~
type: string
example: ~
default: "airflow"
- name: default_cpus
description: ~
version_added: ~
type: string
example: ~
default: "1"
- name: default_ram
description: ~
version_added: ~
type: string
example: ~
default: "512"
- name: default_disk
description: ~
version_added: ~
type: string
example: ~
default: "512"
- name: default_gpus
description: ~
version_added: ~
type: string
example: ~
default: "0"
- name: default_queue
description: |
Default queue that tasks get assigned to and that worker listen on.
version_added: ~
type: string
example: ~
default: "default"
- name: allow_illegal_arguments
description: |
Is allowed to pass additional/unused arguments (args, kwargs) to the BaseOperator operator.
If set to False, an exception will be thrown, otherwise only the console message will be displayed.
version_added: ~
type: string
example: ~
default: "False"
- name: hive
description: ~
options:
- name: default_hive_mapred_queue
description: |
Default mapreduce queue for HiveOperator tasks
version_added: ~
type: string
example: ~
default: ""
- name: mapred_job_name_template
description: |
Template for mapred_job_name in HiveOperator, supports the following named parameters
hostname, dag_id, task_id, execution_date
version_added: ~
type: string
example: ~
default: ~
- name: webserver
description: ~
options:
- name: base_url
description: |
The base url of your website as airflow cannot guess what domain or
cname you are using. This is used in automated emails that
airflow sends to point links to the right web server
version_added: ~
type: string
example: ~
default: "http://localhost:8080"
- name: default_ui_timezone
description: |
Default timezone to display all dates in the UI, can be UTC, system, or
any IANA timezone string (e.g. Europe/Amsterdam). If left empty the
default value of core/default_timezone will be used
version_added: 1.10.10
type: string
example: "America/New_York"
# Default is left as UTC for now so the date's don't "suddenly" change on upgrade
default: "UTC"
- name: web_server_host
description: |
The ip specified when starting the web server
version_added: ~
type: string
example: ~
default: "0.0.0.0"
- name: web_server_port
description: |
The port on which to run the web server
version_added: ~
type: string
example: ~
default: "8080"
- name: web_server_ssl_cert
description: |
Paths to the SSL certificate and key for the web server. When both are
provided SSL will be enabled. This does not change the web server port.
version_added: ~
type: string
example: ~
default: ""
- name: web_server_ssl_key
description: |
Paths to the SSL certificate and key for the web server. When both are
provided SSL will be enabled. This does not change the web server port.
version_added: ~
type: string
example: ~
default: ""
- name: web_server_master_timeout
description: |
Number of seconds the webserver waits before killing gunicorn master that doesn't respond
version_added: ~
type: string
example: ~
default: "120"
- name: web_server_worker_timeout
description: |
Number of seconds the gunicorn webserver waits before timing out on a worker
version_added: ~
type: string
example: ~
default: "120"
- name: worker_refresh_batch_size
description: |
Number of workers to refresh at a time. When set to 0, worker refresh is