/
config_example.yaml
84 lines (82 loc) · 3.82 KB
/
config_example.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
log:
level: warn
format: json
api:
port: 9099
# Payloadsd larger than this are denied. Use this as a protection
payload_size_limit: "1111KB"
flows:
# The name of the flow. This will be used on logs/metrics and on the URL to intest
# data on this flow
- name: "my-flow"
ingestion:
# When a token is defined, requests that arrive without it are denied
token: "some-token-here"
decompress:
# Optional. Adding values here means the data will be decompressed before being ingested.
# But decompression will only happen in case the correct "Content-Encoding" header is set.
# If no header is set the data will be ingested as is.
# Possible values are only: gzip, zlib, deflate, lzw, zstd, snappy
active: ['gzip', 'snappy']
# Optional. If left empty or zero, it will be unrestricted, which can cause CPU
# starvation if it receives too many requests at the same time.
# This is the number of concurrent decompression that can happen.
# In case more than this number of decompressions are happening,
# it will block and enqueue the incoming ingestion requests.
# Default is unrestricted concurrency.
max_concurrency: 6
# The amount of items in memory, waiting to be sent to storage. If this queue is full,
# data starts to be dropped (lost)
in_memory_queue_max_size: 1000
# How many workers will exist to upload files, in parallel. Each worker uploads only 1
# file at a time (before asking for more work)
max_concurrent_uploads: 50
# This is used on the object storage. It tells how many different dynamic prefixes it will use.
# Increase it in case you are receiving too many rate-limit errors.
path_prefix_count: 1
# Optional. This component compresses data before sending it to object storage
compression:
# The compression algorithm. Available types are gzip, zlib, deflate, lzw, zstd, snappy
type: "gzip"
# Higher numbers mean more speed and less compression
level: "3"
# Optional. This component merges data into memory until a user defined limit, and then sends
# it forward to be uploaded into S3
accumulator:
# The size it will accumulate in memory before sending the merge data to be uploaded
size_in_bytes: 2097152 # 2MB
# [Optional] The separator it will use to merge the chuncks of data
separator: "__n__"
# The internal queue of this component. This serves as a buffer for data yet to be merged.
# The minimum allowed is 2 (which is a bad number, you should give it at least some room to work)
queue_capacity: 11
# This doesn't need to be declared. If not declared, it will have a open_interval_in_ms of 100
circuit_breaker:
# In case the below line is false, no circuit breaker will be used and other propoertis of
# this "circuit_breaker" will be ignored.
turn_on: true
# This is the time it takes for, once the circuit breaker is opened (meaning it is not
# accepting calls), how long it will take to try to accept calls again. If you set it as
# zero it will be automatically set to 100.
open_interval_in_ms: 100
external_queue:
# Available values: noop, sqs. noop is for when you don't want to use any queue.
type: sqs
config:
url: "queue-url"
region: aws-region-here
access_key: ""
secret_key: ""
object_storage:
# Available values: s3, localstorage.
type: s3
config:
# How much time to wait before considering that the upload failed and canceling it.
# Zero means no timeout.
timeout_milliseconds: 120000 #120sec
bucket: some-bucket-name-here
region: aws-region-here
endpoint: ""
prefix: ""
access_key: ""
secret_key: ""