Skip to content

Commit 9ee7898

Browse files
mjabascal10mjabascal10
committed
feat(integration): add VMware and Netflow filter updates for enhanced log processing
1 parent d02215b commit 9ee7898

3 files changed

Lines changed: 454 additions & 0 deletions

File tree

‎backend/src/main/resources/config/liquibase/changelog/20260206001_update_filter_vmware_integration.xml‎

Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
<?xml version="1.0" encoding="utf-8"?>
2+
<databaseChangeLog
3+
xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
4+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
5+
xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.5.xsd">
6+
7+
<changeSet id="20260206001" author="Manuel">
8+
<sql dbms="postgresql" splitStatements="true" stripComments="true">
9+
<![CDATA[
10+
11+
UPDATE public.utm_logstash_filter
12+
SET filter_version = '3.0.1',
13+
updated_at = now(),
14+
logstash_filter= $$ # VMWare-ESXi, version 3.0.1
15+
#
16+
# Based on docs and real logs provided
17+
# Support VMWare-ESXi log
18+
#
19+
# Documentations
20+
# 1- https://core.vmware.com/esxi-log-message-formats
21+
#
22+
# Implementation
23+
# 1. Parsing headers of syslog the message
24+
# 2. Parsing the RAW field containing the VMWare-ESXi
25+
pipeline:
26+
- dataTypes:
27+
- vmware-esxi
28+
steps:
29+
- grok:
30+
patterns:
31+
- fieldName: log.priority
32+
pattern: '\<{{.data}}\>'
33+
- fieldName: log.deviceTime
34+
pattern: '{{.year}}(-){{.monthNumber}}(-){{.monthDay}}(T){{.time}}(Z)'
35+
- fieldName: origin.hostname
36+
pattern: '{{.hostname}}'
37+
- fieldName: log.process
38+
pattern: '{{.hostname}}(\:)'
39+
- fieldName: severity
40+
pattern: '{{.word}}'
41+
- fieldName: log.processName
42+
pattern: '{{.hostname}}'
43+
- fieldName: log.pid
44+
pattern: '\[{{.data}}\]'
45+
- fieldName: log.eventInfo
46+
pattern: '\[{{.data}}\]'
47+
- fieldName: log.message
48+
pattern: '{{.greedy}}'
49+
50+
- grok:
51+
patterns:
52+
- fieldName: log.priority
53+
pattern: '\<{{.data}}\>'
54+
- fieldName: log.deviceTime
55+
pattern: '{{.year}}(-){{.monthNumber}}(-){{.monthDay}}(T){{.time}}(Z)'
56+
- fieldName: origin.hostname
57+
pattern: '{{.hostname}}'
58+
- fieldName: log.process
59+
pattern: '{{.hostname}}'
60+
- fieldName: log.pid
61+
pattern: '\[{{.data}}\]:'
62+
- fieldName: log.message
63+
pattern: '{{.greedy}}'
64+
65+
- grok:
66+
patterns:
67+
- fieldName: log.priority
68+
pattern: '\<{{.data}}\>'
69+
- fieldName: log.deviceTime
70+
pattern: '{{.year}}-{{.monthNumber}}-{{.monthDay}}T{{.time}}Z'
71+
- fieldName: origin.hostname
72+
pattern: '{{.hostname}}'
73+
- fieldName: log.process
74+
pattern: '{{.hostname}}'
75+
- fieldName: log.pid
76+
pattern: '\[{{.data}}\]:'
77+
- fieldName: log.originIdComponent
78+
pattern: '\[{{.data}}\]'
79+
- fieldName: log.message
80+
pattern: '{{.greedy}}'
81+
82+
- grok:
83+
patterns:
84+
- fieldName: log.moduleIdentifier
85+
pattern: '\[{{.data}}\@'
86+
- fieldName: log.irrelevant
87+
pattern: '{{.data}}\='
88+
- fieldName: log.subModuleIdentifier
89+
pattern: '{{.word}}\]'
90+
source: log.originIdComponent
91+
92+
# Removing unused caracters
93+
- trim:
94+
function: prefix
95+
substring: '<'
96+
fields:
97+
- log.priority
98+
- trim:
99+
function: prefix
100+
substring: '['
101+
fields:
102+
- log.pid
103+
- log.eventInfo
104+
- log.moduleIdentifier
105+
- trim:
106+
function: prefix
107+
substring: '-'
108+
fields:
109+
- log.message
110+
- trim:
111+
function: suffix
112+
substring: '>'
113+
fields:
114+
- log.priority
115+
- trim:
116+
function: suffix
117+
substring: ':'
118+
fields:
119+
- log.pid
120+
- log.process
121+
- trim:
122+
function: suffix
123+
substring: ']'
124+
fields:
125+
- log.pid
126+
- log.eventInfo
127+
- log.subModuleIdentifier
128+
- trim:
129+
function: suffix
130+
substring: '-'
131+
fields:
132+
- log.message
133+
- trim:
134+
function: suffix
135+
substring: '@'
136+
fields:
137+
- log.moduleIdentifier
138+
139+
# Removing unused fields
140+
- delete:
141+
fields:
142+
- log.processName
143+
- log.irrelevant
144+
145+
$$
146+
147+
WHERE id = 1001
148+
149+
]]>
150+
</sql>
151+
</changeSet>
152+
153+
</databaseChangeLog>

0 commit comments

Comments
 (0)