Skip to content

Commit efd9b9f

Browse files
mmguerommguero
committed
work in progress on pipeline normalization
1 parent 0d6eaf3 commit efd9b9f

File tree

4 files changed

+230
-9
lines changed

4 files changed

+230
-9
lines changed

‎dashboards/dashboards/248cae60-eff9-11f0-b83f-8f35d6995138.json‎

Lines changed: 210 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,210 @@
1+
{
2+
"objects": [
3+
{
4+
"attributes": {
5+
"description": "",
6+
"hits": 0,
7+
"kibanaSavedObjectMeta": {
8+
"searchSourceJSON": "{\"highlightAll\":false,\"version\":true,\"query\":{\"language\":\"lucene\",\"query\":\"*\"},\"filter\":[]}"
9+
},
10+
"optionsJSON": "{\"useMargins\":true}",
11+
"panelsJSON": "[{\"version\":\"3.4.0\",\"gridData\":{\"x\":0,\"y\":0,\"w\":8,\"h\":38,\"i\":\"2\"},\"panelIndex\":\"2\",\"embeddableConfig\":{},\"panelRefName\":\"panel_0\"},{\"version\":\"3.4.0\",\"gridData\":{\"x\":8,\"y\":0,\"w\":7,\"h\":12,\"i\":\"43ec104c-1a98-4fd3-9bef-650531cfc571\"},\"panelIndex\":\"43ec104c-1a98-4fd3-9bef-650531cfc571\",\"embeddableConfig\":{},\"panelRefName\":\"panel_1\"},{\"version\":\"3.4.0\",\"gridData\":{\"x\":15,\"y\":0,\"w\":33,\"h\":12,\"i\":\"2b13101f-b6ed-4d05-8670-93c967560fed\"},\"panelIndex\":\"2b13101f-b6ed-4d05-8670-93c967560fed\",\"embeddableConfig\":{},\"panelRefName\":\"panel_2\"},{\"version\":\"3.4.0\",\"gridData\":{\"x\":8,\"y\":12,\"w\":13,\"h\":16,\"i\":\"c44c352c-c532-47c8-9548-58d0289dc754\"},\"panelIndex\":\"c44c352c-c532-47c8-9548-58d0289dc754\",\"embeddableConfig\":{},\"panelRefName\":\"panel_3\"},{\"version\":\"3.4.0\",\"gridData\":{\"x\":0,\"y\":38,\"w\":48,\"h\":30,\"i\":\"e402ba8c-b844-49a0-85a2-b804e74626d2\"},\"panelIndex\":\"e402ba8c-b844-49a0-85a2-b804e74626d2\",\"embeddableConfig\":{},\"panelRefName\":\"panel_4\"}]",
12+
"timeRestore": false,
13+
"title": "File Scanning",
14+
"version": 1
15+
},
16+
"id": "248cae60-eff9-11f0-b83f-8f35d6995138",
17+
"migrationVersion": {
18+
"dashboard": "7.9.3"
19+
},
20+
"namespaces": [
21+
"default"
22+
],
23+
"references": [
24+
{
25+
"id": "df9e399b-efa5-4e33-b0ac-a7668a8ac2b3",
26+
"name": "panel_0",
27+
"type": "visualization"
28+
},
29+
{
30+
"id": "fe630f60-effb-11f0-b83f-8f35d6995138",
31+
"name": "panel_1",
32+
"type": "visualization"
33+
},
34+
{
35+
"id": "3e964d40-effc-11f0-b83f-8f35d6995138",
36+
"name": "panel_2",
37+
"type": "visualization"
38+
},
39+
{
40+
"id": "52eca680-f00c-11f0-a54d-0fd88ef4958b",
41+
"name": "panel_3",
42+
"type": "visualization"
43+
},
44+
{
45+
"id": "fdee9820-effa-11f0-b83f-8f35d6995138",
46+
"name": "panel_4",
47+
"type": "search"
48+
}
49+
],
50+
"type": "dashboard",
51+
"updated_at": "2026-01-12T23:27:48.310Z",
52+
"version": "WzEyOTMsMV0="
53+
},
54+
{
55+
"attributes": {
56+
"description": "",
57+
"kibanaSavedObjectMeta": {
58+
"searchSourceJSON": "{\"query\":{\"query\":{\"query_string\":{\"query\":\"*\"}},\"language\":\"lucene\"},\"filter\":[]}"
59+
},
60+
"title": "Navigation",
61+
"uiStateJSON": "{}",
62+
"version": 1,
63+
"visState": "{\"title\":\"Navigation\",\"type\":\"markdown\",\"params\":{\"markdown\":\"MALCOLM_NAVIGATION_MARKDOWN_REPLACER\",\"type\":\"markdown\",\"fontSize\":10,\"openLinksInNewTab\":false},\"aggs\":[]}"
64+
},
65+
"id": "df9e399b-efa5-4e33-b0ac-a7668a8ac2b3",
66+
"migrationVersion": {
67+
"visualization": "7.10.0"
68+
},
69+
"namespaces": [
70+
"default"
71+
],
72+
"references": [],
73+
"type": "visualization",
74+
"updated_at": "2026-01-12T23:22:04.571Z",
75+
"version": "WzEyNzMsMV0="
76+
},
77+
{
78+
"attributes": {
79+
"description": "",
80+
"kibanaSavedObjectMeta": {
81+
"searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"
82+
},
83+
"savedSearchRefName": "search_0",
84+
"title": "File Scanning - Hit Count",
85+
"uiStateJSON": "{}",
86+
"version": 1,
87+
"visState": "{\"title\":\"File Scanning - Hit Count\",\"type\":\"metric\",\"aggs\":[{\"id\":\"2\",\"enabled\":true,\"type\":\"count\",\"params\":{\"customLabel\":\"Files Scanned\"},\"schema\":\"metric\"},{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"params\":{\"field\":\"event.hits\",\"customLabel\":\"Scanner Hits\"},\"schema\":\"metric\"}],\"params\":{\"addTooltip\":true,\"addLegend\":false,\"type\":\"metric\",\"metric\":{\"percentageMode\":false,\"useRanges\":false,\"colorSchema\":\"Green to Red\",\"metricColorMode\":\"None\",\"colorsRange\":[{\"from\":0,\"to\":10000}],\"labels\":{\"show\":true},\"invertColors\":false,\"style\":{\"bgFill\":\"#000\",\"bgColor\":false,\"labelColor\":false,\"subText\":\"\",\"fontSize\":32}}}}"
88+
},
89+
"id": "fe630f60-effb-11f0-b83f-8f35d6995138",
90+
"migrationVersion": {
91+
"visualization": "7.10.0"
92+
},
93+
"namespaces": [
94+
"default"
95+
],
96+
"references": [
97+
{
98+
"id": "fdee9820-effa-11f0-b83f-8f35d6995138",
99+
"name": "search_0",
100+
"type": "search"
101+
}
102+
],
103+
"type": "visualization",
104+
"updated_at": "2026-01-12T23:20:55.563Z",
105+
"version": "WzQ0MSwxXQ=="
106+
},
107+
{
108+
"attributes": {
109+
"description": "",
110+
"kibanaSavedObjectMeta": {
111+
"searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"
112+
},
113+
"savedSearchRefName": "search_0",
114+
"title": "File Scanning - Hits Over Time",
115+
"uiStateJSON": "{\"vis\":{\"legendOpen\":false}}",
116+
"version": 1,
117+
"visState": "{\"title\":\"File Scanning - Hits Over Time\",\"type\":\"histogram\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"params\":{\"field\":\"event.hits\",\"customLabel\":\"Scanner Hits\"},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"params\":{\"field\":\"MALCOLM_NETWORK_INDEX_TIME_FIELD_REPLACER\",\"timeRange\":{\"from\":\"now-24y\",\"to\":\"now\"},\"useNormalizedOpenSearchInterval\":true,\"scaleMetricValues\":false,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}},\"schema\":\"segment\"}],\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"filter\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"square root\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Scanner Hits\"}}],\"seriesParams\":[{\"show\":true,\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Scanner Hits\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"lineWidth\":2,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false,\"labels\":{\"show\":false},\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"}}}"
118+
},
119+
"id": "3e964d40-effc-11f0-b83f-8f35d6995138",
120+
"migrationVersion": {
121+
"visualization": "7.10.0"
122+
},
123+
"namespaces": [
124+
"default"
125+
],
126+
"references": [
127+
{
128+
"id": "fdee9820-effa-11f0-b83f-8f35d6995138",
129+
"name": "search_0",
130+
"type": "search"
131+
}
132+
],
133+
"type": "visualization",
134+
"updated_at": "2026-01-12T23:20:55.563Z",
135+
"version": "WzQ0MiwxXQ=="
136+
},
137+
{
138+
"attributes": {
139+
"description": "",
140+
"kibanaSavedObjectMeta": {
141+
"searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"
142+
},
143+
"savedSearchRefName": "search_0",
144+
"title": "File Scanning - Scanners With Hits",
145+
"uiStateJSON": "{\"vis\":{\"legendOpen\":false}}",
146+
"version": 1,
147+
"visState": "{\"title\":\"File Scanning - Scanners With Hits\",\"type\":\"pie\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"filescan.hits_scanner\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":20,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Scanner\"},\"schema\":\"segment\"}],\"params\":{\"type\":\"pie\",\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"bottom\",\"isDonut\":true,\"labels\":{\"show\":true,\"values\":true,\"last_level\":true,\"truncate\":100}}}"
148+
},
149+
"id": "52eca680-f00c-11f0-a54d-0fd88ef4958b",
150+
"migrationVersion": {
151+
"visualization": "7.10.0"
152+
},
153+
"namespaces": [
154+
"default"
155+
],
156+
"references": [
157+
{
158+
"id": "fdee9820-effa-11f0-b83f-8f35d6995138",
159+
"name": "search_0",
160+
"type": "search"
161+
}
162+
],
163+
"type": "visualization",
164+
"updated_at": "2026-01-12T23:20:55.563Z",
165+
"version": "WzQ0MywxXQ=="
166+
},
167+
{
168+
"attributes": {
169+
"columns": [
170+
"source.ip",
171+
"destination.ip",
172+
"file.source",
173+
"file.mime_type",
174+
"zeek.files.filename",
175+
"zeek.files.extracted_uri",
176+
"event.hits",
177+
"filescan.hits_scanner",
178+
"rule.name",
179+
"event.id"
180+
],
181+
"description": "",
182+
"hits": 0,
183+
"kibanaSavedObjectMeta": {
184+
"searchSourceJSON": "{\"query\":{\"query\":\"event.provider:filescan\",\"language\":\"kuery\"},\"highlightAll\":false,\"version\":true,\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"MALCOLM_NETWORK_INDEX_TIME_FIELD_REPLACER\",\"fixed_interval\":\"30d\",\"time_zone\":\"America/Denver\",\"min_doc_count\":1}}},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"
185+
},
186+
"sort": [],
187+
"title": "File Scanning - Logs",
188+
"version": 1
189+
},
190+
"id": "fdee9820-effa-11f0-b83f-8f35d6995138",
191+
"migrationVersion": {
192+
"search": "7.9.3"
193+
},
194+
"namespaces": [
195+
"default"
196+
],
197+
"references": [
198+
{
199+
"id": "MALCOLM_NETWORK_INDEX_PATTERN_REPLACER",
200+
"name": "kibanaSavedObjectMeta.searchSourceJSON.index",
201+
"type": "index-pattern"
202+
}
203+
],
204+
"type": "search",
205+
"updated_at": "2026-01-12T23:20:55.563Z",
206+
"version": "WzQ0NCwxXQ=="
207+
}
208+
],
209+
"version": "3.4.0"
210+
}

‎dashboards/dashboards/navigation.md‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,14 +6,15 @@
66
[Connections](#/dashboard/abdd7550-2c7c-40dc-947e-f6d186a158c4)
77
[Connections Tree](#/dashboard/89714140-a2d5-11f0-b5ae-e139a66d2205)
88
[Actions and Results](#/dashboard/a33e0a50-afcd-11ea-993f-b7d8522a8bed)
9+
[Threat Intelligence](#/dashboard/36ed695f-edcc-47c1-b0ec-50d20c93ce0f)
910
[Files](#/dashboard/9ee51f94-3316-4fc5-bd89-93a52af69714)
11+
[File Scanning](#/dashboard/248cae60-eff9-11f0-b83f-8f35d6995138)
1012
[Executables](#/dashboard/0a490422-0ce9-44bf-9a2d-19329ddde8c3)
1113
[Software](#/dashboard/87d990cc-9e0b-41e5-b8fe-b10ae1da0c85)
1214
[Zeek Known Summary](#/dashboard/89d1cc50-974c-11ed-bb6b-3fb06c879b11)
1315
[Zeek Notices](#/dashboard/f1f09567-fc7f-450b-a341-19d2f2bb468b)
1416
[Zeek Weird](#/dashboard/1fff49f6-0199-4a0f-820b-721aff9ff1f1)
15-
[Threat Intelligence](#/dashboard/36ed695f-edcc-47c1-b0ec-50d20c93ce0f)
16-
[Signatures](#/dashboard/665d1610-523d-11e9-a30e-e3576242f3ed)
17+
[Zeek Signatures](#/dashboard/665d1610-523d-11e9-a30e-e3576242f3ed)
1718
[Suricata Alerts](#/dashboard/5694ca60-cbdf-11ec-a50a-5fedd672f5c5)
1819
[Asset Interaction Analysis](#/dashboard/677ee170-809e-11ed-8d5b-07069f823b6f)
1920
[Validated Architecture Design Review](#/dashboard/d16105d0-2b75-11f0-92dc-5f54cacd4f4e)

‎dashboards/templates/composable/component/filescan.json‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -6,17 +6,23 @@
66
"properties": {
77
"hits": {
88
"type": "nested",
9+
"dynamic": false,
10+
"include_in_parent": false,
11+
"include_in_root": false,
912
"properties": {
10-
"capa": {
13+
"scanner": {
1114
"type": "keyword"
1215
},
13-
"clamav": {
14-
"type": "keyword"
15-
},
16-
"yara": {
16+
"name": {
1717
"type": "keyword"
1818
}
1919
}
20+
},
21+
"hits_scanner": {
22+
"type": "keyword"
23+
},
24+
"hits_name": {
25+
"type": "keyword"
2026
}
2127
}
2228
},

‎logstash/pipelines/filescan/12_normalize.conf‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,10 +31,12 @@ filter {
3131
# pull them out from deeper in the meantime
3232
code => "
3333
scanners = Array(event.get('[results][strelka][result][scanners]') || []).compact
34+
hits = []
3435
scanners_to_rules = {}
3536
final_rules = []
3637

37-
Array(event.get('[results][strelka][result][rules]') || []).each do |r|
38+
rules = Array(event.get('[results][strelka][result][rules]') || [])
39+
rules.each do |r|
3840
next unless r.is_a?(Hash) && r.key?('scanner') && r.key?('name')
3941
scanner = r['scanner']
4042
name = r['name']
@@ -52,7 +54,9 @@ filter {
5254

5355
final_rules.uniq!
5456

55-
event.set('[filescan][hits]', scanners_to_rules) unless scanners_to_rules.empty?
57+
event.set('[filescan][hits]', rules.map { |h| h.except('provider') }) unless rules.empty?
58+
event.set('[filescan][hits_scanner]', rules.map { |h| h['scanner'] }.compact.uniq) unless rules.empty?
59+
event.set('[filescan][hits_name]', rules.map { |h| h['name'] }.compact.uniq) unless rules.empty?
5660
event.set('[event][module]', scanners) unless scanners.empty?
5761
event.set('[event][hits]', final_rules.length)
5862

0 commit comments

Comments
 (0)