Skip to content

Commit 4dd9d63

Browse files
kaya-davidkaya-david
committed
feat: add pipeline configs for benchmark
1 parent f8e3b65 commit 4dd9d63

File tree

3 files changed

+270
-2
lines changed

3 files changed

+270
-2
lines changed

‎benchmark.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -308,8 +308,8 @@ def resolve_pipeline_config(ng: int) -> Path:
308308
Pipeline config path.
309309
"""
310310
if ng == 1:
311-
return Path("./examples/exampledata/config/ng_pipeline.yml")
312-
return Path("./examples/exampledata/config/pipeline.yml")
311+
return Path("./examples/exampledata/config/_benchmark_ng_pipeline.yml")
312+
return Path("./examples/exampledata/config/_benchmark_non_ng_pipeline.yml")
313313

314314

315315
def read_vm_max_map_count() -> int:

‎examples/exampledata/config/_benchmark_ng_pipeline.yml‎

Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,134 @@
1+
version: 2
2+
process_count: 1
3+
timeout: 5.0
4+
restart_count: 2
5+
config_refresh_interval: 5
6+
error_backlog_size: 1500000
7+
logger:
8+
level: DEBUG
9+
format: "%(asctime)-15s %(hostname)-5s %(name)-10s %(levelname)-8s: %(message)s"
10+
datefmt: "%Y-%m-%d %H:%M:%S"
11+
loggers:
12+
"py.warnings": {"level": "ERROR"}
13+
"Runner": {"level": "INFO"}
14+
"Processor": {"level": "ERROR"}
15+
"Exporter": {"level": "ERROR"}
16+
"uvicorn": {"level": "ERROR"}
17+
"uvicorn.access": {"level": "ERROR"}
18+
"OpenSearchOutput": {"level": "DEBUG"}
19+
"KafkaOutput": {"level": "ERROR"}
20+
"Input": {"level": "ERROR"}
21+
metrics:
22+
enabled: true
23+
port: 8001
24+
pipeline:
25+
- labelername:
26+
type: ng_labeler
27+
schema: examples/exampledata/rules/labeler/schema.json
28+
include_parent_labels: true
29+
rules:
30+
- examples/exampledata/rules/labeler/rules
31+
- dissector:
32+
type: ng_dissector
33+
rules:
34+
- examples/exampledata/rules/dissector/rules
35+
- dropper:
36+
type: ng_dropper
37+
rules:
38+
- examples/exampledata/rules/dropper/rules
39+
- filter: "test_dropper"
40+
dropper:
41+
drop:
42+
- drop_me
43+
description: "..."
44+
- pre_detector:
45+
type: ng_pre_detector
46+
rules:
47+
- examples/exampledata/rules/pre_detector/rules
48+
outputs:
49+
- opensearch: sre
50+
tree_config: examples/exampledata/rules/pre_detector/tree_config.json
51+
alert_ip_list_path: examples/exampledata/rules/pre_detector/alert_ips.yml
52+
- amides:
53+
type: ng_amides
54+
rules:
55+
- examples/exampledata/rules/amides/rules
56+
models_path: examples/exampledata/models/model.zip
57+
num_rule_attributions: 10
58+
max_cache_entries: 1000000
59+
decision_threshold: 0.32
60+
- pseudonymizer:
61+
type: ng_pseudonymizer
62+
pubkey_analyst: examples/exampledata/rules/pseudonymizer/example_analyst_pub.pem
63+
pubkey_depseudo: examples/exampledata/rules/pseudonymizer/example_depseudo_pub.pem
64+
regex_mapping: examples/exampledata/rules/pseudonymizer/regex_mapping.yml
65+
hash_salt: a_secret_tasty_ingredient
66+
outputs:
67+
- opensearch: pseudonyms
68+
rules:
69+
- examples/exampledata/rules/pseudonymizer/rules/
70+
max_cached_pseudonyms: 1000000
71+
- calculator:
72+
type: ng_calculator
73+
rules:
74+
- filter: "test_label: execute"
75+
calculator:
76+
target_field: "calculation"
77+
calc: "1 + 1"
78+
input:
79+
kafka:
80+
type: ng_confluentkafka_input
81+
topic: consumer
82+
kafka_config:
83+
bootstrap.servers: 127.0.0.1:9092
84+
group.id: cgroup3
85+
enable.auto.commit: "true"
86+
auto.commit.interval.ms: "10000"
87+
enable.auto.offset.store: "false"
88+
queued.min.messages: "100000"
89+
queued.max.messages.kbytes: "65536"
90+
statistics.interval.ms: "60000"
91+
preprocessing:
92+
version_info_target_field: Logprep_version_info
93+
log_arrival_time_target_field: event.ingested
94+
hmac:
95+
target: <RAW_MSG>
96+
key: "thisisasecureandrandomkey"
97+
output_field: Full_event
98+
output:
99+
opensearch:
100+
type: ng_opensearch_output
101+
hosts:
102+
- 127.0.0.1:9200
103+
default_index: processed
104+
default_op_type: create
105+
message_backlog_size: 7000
106+
timeout: 10000
107+
flush_timeout: 60
108+
user: admin
109+
secret: admin
110+
desired_cluster_status: ["green", "yellow"]
111+
chunk_size: 25
112+
kafka:
113+
type: ng_confluentkafka_output
114+
default: false
115+
topic: producer
116+
flush_timeout: 300
117+
kafka_config:
118+
bootstrap.servers: 127.0.0.1:9092
119+
statistics.interval.ms: "60000"
120+
error_output:
121+
kafka_error_output:
122+
type: ng_confluentkafka_output
123+
topic: errors
124+
flush_timeout: 300
125+
send_timeout: 0
126+
kafka_config:
127+
bootstrap.servers: 127.0.0.1:9092
128+
compression.type: none
129+
statistics.interval.ms: "60000"
130+
queue.buffering.max.messages: "10"
131+
queue.buffering.max.kbytes: "1024"
132+
queue.buffering.max.ms: "1000"
133+
batch.size: "100"
134+
request.required.acks: "-1"

‎examples/exampledata/config/_benchmark_non_ng_pipeline.yml‎

Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,134 @@
1+
version: 2
2+
process_count: 1
3+
timeout: 5.0
4+
restart_count: 2
5+
config_refresh_interval: 5
6+
error_backlog_size: 1500000
7+
logger:
8+
level: DEBUG
9+
format: "%(asctime)-15s %(hostname)-5s %(name)-10s %(levelname)-8s: %(message)s"
10+
datefmt: "%Y-%m-%d %H:%M:%S"
11+
loggers:
12+
"py.warnings": {"level": "ERROR"}
13+
"Runner": {"level": "INFO"}
14+
"Processor": {"level": "ERROR"}
15+
"Exporter": {"level": "ERROR"}
16+
"uvicorn": {"level": "ERROR"}
17+
"uvicorn.access": {"level": "ERROR"}
18+
"OpenSearchOutput": {"level": "DEBUG"}
19+
"KafkaOutput": {"level": "ERROR"}
20+
"Input": {"level": "ERROR"}
21+
metrics:
22+
enabled: true
23+
port: 8001
24+
pipeline:
25+
- labelername:
26+
type: labeler
27+
schema: examples/exampledata/rules/labeler/schema.json
28+
include_parent_labels: true
29+
rules:
30+
- examples/exampledata/rules/labeler/rules
31+
- dissector:
32+
type: dissector
33+
rules:
34+
- examples/exampledata/rules/dissector/rules
35+
- dropper:
36+
type: dropper
37+
rules:
38+
- examples/exampledata/rules/dropper/rules
39+
- filter: "test_dropper"
40+
dropper:
41+
drop:
42+
- drop_me
43+
description: "..."
44+
- pre_detector:
45+
type: pre_detector
46+
rules:
47+
- examples/exampledata/rules/pre_detector/rules
48+
outputs:
49+
- opensearch: sre
50+
tree_config: examples/exampledata/rules/pre_detector/tree_config.json
51+
alert_ip_list_path: examples/exampledata/rules/pre_detector/alert_ips.yml
52+
- amides:
53+
type: amides
54+
rules:
55+
- examples/exampledata/rules/amides/rules
56+
models_path: examples/exampledata/models/model.zip
57+
num_rule_attributions: 10
58+
max_cache_entries: 1000000
59+
decision_threshold: 0.32
60+
- pseudonymizer:
61+
type: pseudonymizer
62+
pubkey_analyst: examples/exampledata/rules/pseudonymizer/example_analyst_pub.pem
63+
pubkey_depseudo: examples/exampledata/rules/pseudonymizer/example_depseudo_pub.pem
64+
regex_mapping: examples/exampledata/rules/pseudonymizer/regex_mapping.yml
65+
hash_salt: a_secret_tasty_ingredient
66+
outputs:
67+
- opensearch: pseudonyms
68+
rules:
69+
- examples/exampledata/rules/pseudonymizer/rules/
70+
max_cached_pseudonyms: 1000000
71+
- calculator:
72+
type: calculator
73+
rules:
74+
- filter: "test_label: execute"
75+
calculator:
76+
target_field: "calculation"
77+
calc: "1 + 1"
78+
input:
79+
kafka:
80+
type: confluentkafka_input
81+
topic: consumer
82+
kafka_config:
83+
bootstrap.servers: 127.0.0.1:9092
84+
group.id: cgroup3
85+
enable.auto.commit: "true"
86+
auto.commit.interval.ms: "10000"
87+
enable.auto.offset.store: "false"
88+
queued.min.messages: "100000"
89+
queued.max.messages.kbytes: "65536"
90+
statistics.interval.ms: "60000"
91+
preprocessing:
92+
version_info_target_field: Logprep_version_info
93+
log_arrival_time_target_field: event.ingested
94+
hmac:
95+
target: <RAW_MSG>
96+
key: "thisisasecureandrandomkey"
97+
output_field: Full_event
98+
output:
99+
opensearch:
100+
type: opensearch_output
101+
hosts:
102+
- 127.0.0.1:9200
103+
default_index: processed
104+
default_op_type: create
105+
message_backlog_size: 7000
106+
timeout: 10000
107+
flush_timeout: 60
108+
user: admin
109+
secret: admin
110+
desired_cluster_status: ["green", "yellow"]
111+
chunk_size: 25
112+
kafka:
113+
type: confluentkafka_output
114+
default: false
115+
topic: producer
116+
flush_timeout: 300
117+
kafka_config:
118+
bootstrap.servers: 127.0.0.1:9092
119+
statistics.interval.ms: "60000"
120+
error_output:
121+
kafka_error_output:
122+
type: confluentkafka_output
123+
topic: errors
124+
flush_timeout: 300
125+
send_timeout: 0
126+
kafka_config:
127+
bootstrap.servers: 127.0.0.1:9092
128+
compression.type: none
129+
statistics.interval.ms: "60000"
130+
queue.buffering.max.messages: "10"
131+
queue.buffering.max.kbytes: "1024"
132+
queue.buffering.max.ms: "1000"
133+
batch.size: "100"
134+
request.required.acks: "-1"

0 commit comments

Comments
 (0)