Skip to content

Commit 459d381

Browse files
zinalzinal
authored
FluentBit log collection setup support for old-style playbooks (#58)
* added 'check_database_access_permission: true' to configuration samples * custom fact_path defined * revert adding fact_path as it becomes deprecated * better rolling restart for db nodes * error fixes * Print the parameters on database creation * error fixes * fluentbit role support * extra variables added to all.example
1 parent f774727 commit 459d381

26 files changed

+558
-37
lines changed

‎examples/fluentbit/ddl.sql‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
2+
CREATE USER fluentbit PASSWORD "....";
3+
GRANT CONNECT ON `/cluster1/admin` TO fluentbit;
4+
5+
CREATE TABLE `ydblogs` (
6+
`ts` Timestamp NOT NULL,
7+
`datahash` Uint64 NOT NULL,
8+
`dbname` Text NOT NULL,
9+
`hostname` Text NOT NULL,
10+
`input` Text NOT NULL,
11+
`ts_orig` Text,
12+
`level` Text,
13+
`unit` Text,
14+
`service` Text,
15+
`msg` Text,
16+
PRIMARY KEY (
17+
`ts`, `datahash`, `dbname`, `hostname`, `input`
18+
)
19+
) PARTITION BY HASH(`ts`, `dbname`, `hostname`, `input`)
20+
WITH (
21+
STORE = COLUMN
22+
);
23+
24+
GRANT USE ON `/cluster1/admin/ydblogs` TO fluentbit;

‎examples/fluentbit/dynamic-config.txt‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
ydb auth get-token -f >ydbd-token
2+
./ydbd -s grpcs://bigpig1:2135 --ca-file ../tls/ca.crt -f ydbd-token admin console configs dump-yaml >step0.yaml
3+
cp step0.yaml step1.yaml
4+
vi step1.yaml
5+
ydb admin config replace -f step1.yaml

‎examples/fluentbit/dynamic-config.yaml‎

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
metadata:
2+
kind: MainConfig
3+
cluster: ""
4+
version: 2
5+
config:
6+
yaml_config_enabled: true
7+
table_profiles_config:
8+
table_profiles:
9+
- name: default
10+
compaction_policy: default
11+
execution_policy: default
12+
partitioning_policy: default
13+
storage_policy: default
14+
replication_policy: default
15+
caching_policy: default
16+
compaction_policies:
17+
- name: default
18+
execution_policies:
19+
- name: default
20+
partitioning_policies:
21+
- name: default
22+
auto_split: true
23+
auto_merge: true
24+
size_to_split: 2147483648
25+
storage_policies:
26+
- name: default
27+
column_families:
28+
- storage_config:
29+
sys_log:
30+
preferred_pool_kind: ssd
31+
log:
32+
preferred_pool_kind: ssd
33+
data:
34+
preferred_pool_kind: ssd
35+
replication_policies:
36+
- name: default
37+
caching_policies:
38+
- name: default
39+
allowed_labels:
40+
node_id:
41+
type: string
42+
host:
43+
type: string
44+
tenant:
45+
type: string
46+
selector_config:
47+
- description: cookie=testdb-01
48+
selector:
49+
tenant: /cluster1/testdb
50+
config:
51+
shared_cache_config:
52+
memory_limit: 51539607552
53+
feature_flags: !inherit
54+
enable_views: true
55+
enable_data_shard_volatile_transactions: true
56+
table_service_config:
57+
sql_version: 1
58+
index_auto_choose_mode: MAX_USED_PREFIX
59+
enable_implicit_query_parameter_types: true
60+
enable_kqp_data_query_stream_lookup: true
61+
enable_kqp_data_query_stream_idx_lookup_join: true
62+
resource_manager:
63+
kqp_pattern_cache_compiled_capacity_bytes: 524288000
64+
kqp_pattern_cache_capacity_bytes: 524288000
65+
query_limits:
66+
result_rows_limit: 20000
67+
resource_broker_config:
68+
queues:
69+
- name: queue_restore
70+
limit:
71+
cpu: 8
72+
- name: queue_backup
73+
limit:
74+
cpu: 8
75+
actor_system_config:
76+
executor:
77+
- {name: System, spin_threshold: '10', threads: 2, max_threads: 11, type: BASIC}
78+
- {name: User, spin_threshold: '1', threads: 11, max_threads: 15, type: BASIC}
79+
- {name: Batch, spin_threshold: '1', threads: 2, max_threads: 6, type: BASIC}
80+
- {name: IO, threads: 1, type: IO}
81+
- {name: IC, spin_threshold: '10', threads: 7, max_threads: 11, time_per_mailbox_micro_secs: 100, max_avg_ping_deviation: 500, type: BASIC}
82+
scheduler: {progress_threshold: '10000', resolution: '64', spin_threshold: '0'}
83+
sys_executor: 0
84+
user_executor: 1
85+
batch_executor: 2
86+
io_executor: 3
87+
service_executor:
88+
- {executor_id: 4, service_name: Interconnect}
89+
- description: cookie=admin-01
90+
selector:
91+
tenant: /cluster1/admin
92+
config:
93+
shared_cache_config:
94+
memory_limit: 17179869184
95+
feature_flags: !inherit
96+
enable_views: false
97+
enable_data_shard_volatile_transactions: false
98+
table_service_config:
99+
sql_version: 1
100+
column_shard_config:
101+
disabled_on_scheme_shard: false
102+
actor_system_config:
103+
executor:
104+
- {name: System, spin_threshold: '10', threads: 1, max_threads: 4, type: BASIC}
105+
- {name: User, spin_threshold: '1', threads: 3, max_threads: 7, type: BASIC}
106+
- {name: Batch, spin_threshold: '1', threads: 1, max_threads: 4, type: BASIC}
107+
- {name: IO, threads: 1, type: IO}
108+
- {name: IC, spin_threshold: '10', threads: 2, max_threads: 4, time_per_mailbox_micro_secs: 100, max_avg_ping_deviation: 500, type: BASIC}
109+
scheduler: {progress_threshold: '10000', resolution: '64', spin_threshold: '0'}
110+
sys_executor: 0
111+
user_executor: 1
112+
batch_executor: 2
113+
io_executor: 3
114+
service_executor:
115+
- {executor_id: 4, service_name: Interconnect}

‎examples/fluentbit/fluent-bit.conf‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
[SERVICE]
2+
flush 2
3+
log_level info
4+
parsers_file parsers.conf
5+
plugins_file plugins.conf
6+
storage.path /opt/ydb/fluentbit/data/storage
7+
8+
[INPUT]
9+
name systemd
10+
Tag ydbd-storage
11+
storage.type filesystem
12+
DB /opt/ydb/fluentbit/data/ydbd-storage.db
13+
Systemd_Filter _SYSTEMD_UNIT=ydbd-storage.service
14+
15+
[INPUT]
16+
name systemd
17+
Tag ydbd-admin
18+
storage.type filesystem
19+
DB /opt/ydb/fluentbit/data/ydbd-admin.db
20+
Systemd_Filter _SYSTEMD_UNIT=ydbd-admin-a.service
21+
22+
[FILTER]
23+
Name parser
24+
Match ydbd-*
25+
Key_Name MESSAGE
26+
Parser ydb
27+
Reserve_Data On
28+
Preserve_Key On
29+
30+
[FILTER]
31+
Name modify
32+
Match ydbd-*
33+
Condition Key_does_not_exist P_LEVEL
34+
Set P_LEVEL unknown
35+
Hard_copy MESSAGE P_MESSAGE
36+
37+
[FILTER]
38+
Name modify
39+
Match ydbd-storage
40+
Add P_DB /cluster1
41+
42+
[FILTER]
43+
Name modify
44+
Match ydbd-admin
45+
Add P_DB /cluster1/admin
46+
47+
[FILTER]
48+
Name record_modifier
49+
Match ydbd-*
50+
Allowlist_key P_DB
51+
Allowlist_key P_SERVICE
52+
Allowlist_key P_DTTM
53+
Allowlist_key P_LEVEL
54+
Allowlist_key P_MESSAGE
55+
Allowlist_key _HOSTNAME
56+
Allowlist_key _SYSTEMD_UNIT
57+
58+
[OUTPUT]
59+
Name ydb
60+
LogLevel error
61+
ConnectionURL grpcs://ydbd-1.front.private:2135/cluster1/admin
62+
CredentialsStatic fluentbit:password
63+
Certificates /opt/ydb/cfg/ca.crt
64+
Match ydbd-*
65+
TablePath ydblogs
66+
Columns {".timestamp": "ts",".input":"input",".hash":"datahash","P_DB":"dbname","P_SERVICE":"service","P_DTTM":"ts_orig","P_LEVEL":"level","P_MESSAGE":"msg","_HOSTNAME":"hostname","_SYSTEMD_UNIT":"unit"}

‎examples/fluentbit/fluent-bit.service‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
[Unit]
2+
Description=Fluent Bit
3+
Documentation=https://docs.fluentbit.io/manual/
4+
Requires=network.target
5+
After=network.target
6+
7+
[Service]
8+
Type=simple
9+
EnvironmentFile=-/etc/sysconfig/fluent-bit
10+
EnvironmentFile=-/etc/default/fluent-bit
11+
ExecStart=/usr/local/bin/fluent-bit -c /etc/fluent-bit/fluent-bit.conf
12+
Restart=always
13+
14+
[Install]
15+
WantedBy=multi-user.target

‎examples/fluentbit/parsers.conf‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
[PARSER]
2+
Name ydb
3+
Format regex
4+
Regex ^(?<P_DTTM>[^ ]*) :(?<P_SERVICE>[^ ]*) (?<P_LEVEL>[^ ]*): (?<P_MESSAGE>.*).*$
5+

‎examples/fluentbit/plugins.conf‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
[PLUGINS]
2+
out_ydb.so /opt/ydb/fluentbit/lib/out_ydb.so

‎examples/full-dualhomed/cluster-config.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -236,6 +236,7 @@ table_service_config:
236236
kqp_pattern_cache_capacity_bytes: 524288000
237237

238238
feature_flags:
239+
check_database_access_permission: true
239240
enable_views: true
240241
enable_data_shard_volatile_transactions: true
241242
enable_grpc_audit: false

‎files/config-1node.yaml‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -107,8 +107,7 @@ table_service_config:
107107
enable_kqp_data_query_stream_lookup: true
108108

109109
feature_flags:
110-
enable_grpc_audit: false
111-
enable_get_node_labels: true
110+
check_database_access_permission: true
112111
enable_views: true
113112

114113
log_config:

‎files/config-3nodes-extra.yaml‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -184,8 +184,7 @@ bootstrap_config:
184184
shared_cache_config: {memory_limit: '4294967296'}
185185

186186
feature_flags:
187-
enable_grpc_audit: false
188-
enable_get_node_labels: true
187+
check_database_access_permission: true
189188
enable_views: true
190189

191190
log_config:

0 commit comments

Comments
 (0)