Repository navigation
Expand file tree
/
Copy pathkafka.toml
More file actions
82 lines (71 loc) · 2.61 KB
/
Copy pathkafka.toml
File metadata and controls
82 lines (71 loc) · 2.61 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
[sinks.kafka]
title = "Kafka"
noun = "Kafka"
beta = false
common = true
delivery_guarantee = "at_least_once"
<%= render("_partials/descriptions/_kafka.toml") %>
egress_method = "streaming"
features = [
"Send logs to Kafka.",
"Leverage any of AWS' IAM strategies.",
"Optionally compress data to maximize throughput.",
"Automatically retry failed requests, with backoff.",
"Buffer your data in-memory or on-disk for performance and durability."
]
function_category = "transmit"
healthcheck = true
input_types = ["log"]
requirements.kafka = ">= 0.8"
service_providers = ["AWS", "Confluent"]
write_to_description = "[Apache Kafka][urls.kafka] via the [Kafka protocol][urls.kafka_protocol]"
<%= render("_partials/fields/_component_options.toml", type: "sink", name: "kafka") %>
<%= render(
"_partials/fields/_buffer_options.toml",
namespace: "sinks.kafka.options",
common: false
) %>
<%= render("_partials/fields/_kafka_options.toml", namespace: "sinks.kafka.options") %>
[sinks.kafka.options.compression]
type = "string"
common = true
default = "none"
required = false
description = "Compression codec to use for compressing message sets"
[sinks.kafka.options.compression.enum]
none = "No compression"
gzip = "[Gzip](https://www.gnu.org/software/gzip/) standard DEFLATE compression"
lz4 = "High speed [LZ4 compression](https://lz4.github.io/lz4/)"
snappy = "High speed [Snappy compression](https://google.github.io/snappy/), developed by Google. Slower than LZ4 but higher compression."
zstd = "[Zstandard compression](https://zstd.net), developed at Facebook. Faster than gzip at similar compression ratios."
<%= render("_partials/fields/_encoding_options.toml",
namespace: "sinks.kafka.options",
encodings: ["json", "text"]
) %>
[sinks.kafka.options.key_field]
type = "string"
common = true
examples = ["user_id"]
required = true
description = """\
The log field name to use for the topic key. If unspecified, the key will be \
randomly generated. If the field does not exist on the log, a blank value \
will be used.\
"""
<%= render("_partials/fields/_tls_connector_options.toml", namespace: "sinks.kafka.options", can_enable: true, can_verify_certificate: false, can_verify_hostname: false) %>
[sinks.kafka.options.topic]
type = "string"
common = true
examples = ["topic-1234"]
required = true
description = "The Kafka topic name to write events to."
[sinks.kafka.options.socket_timeout_ms]
type = "int"
examples = [30000, 90000]
default = 60000
description = "Default timeout for network requests."
[sinks.kafka.options.message_timeout_ms]
type = "int"
examples = [150000, 450000]
default = 300000
description = "Local message timeout."