Release/v11.2.3

.github/workflows/v11-deployment-pipeline.yml

@@ -522,7 +522,7 @@ jobs:
               -H "Content-Type: application/json" \
               -H "id: $auth_id" \
               -H "key: $auth_key" \
-              -d "{\"instance_id\": \"$instance_id\", \"version\": \"$TAG\"}")
+              -d "{\"instances_ids\": [\"$instance_id\"], \"version\": \"$TAG\"}")
 
             http_code=$(echo "$response" | tail -n1)
             body=$(echo "$response" | sed '$d')

README.md

@@ -1,27 +1,32 @@
-<h1 align="center">
-  <br>
-  <a href="https://utmstack.com"><img src="https://utmstack.com/wp-content/uploads/2023/02/utmstack-logo-favicon.png?v=2" width="200px" alt="UTMStack"></a>
-</h1>
+# UTMStack
 
 <p align="center">
-<a href="https://github.com/utmstack/UTMStack/graphs/contributors"><img src="https://img.shields.io/github/contributors-anon/utmstack/utmstack">
-<a href="https://github.com/utmstack/UTMStack/releases/"><img src="https://img.shields.io/github/release/utmstack/utmstack">
-<a href="https://github.com/utmstack/UTMStack/issues"><img src="https://img.shields.io/github/issues-raw/utmstack/utmstack">
-<a href="https://github.com/utmstack/UTMStack/commits/main"><img src="https://img.shields.io/github/commit-activity/m/utmstack/utmstack">
-<a href="https://github.com/utmstack/UTMStack/blob/master/LICENSE"><img src="https://img.shields.io/github/license/ad-aures/castopod?color=blue">
-<a href="https://discord.gg/ZznvZ8xcHh"><img src="https://img.shields.io/discord/1154016563775672400.svg?logo=discord">
+  <a href="https://utmstack.com">
+    <img src="https://utmstack.com/wp-content/uploads/2023/02/utmstack-logo-favicon.png?v=2" alt="UTMStack" width="150px">
+  </a>
 </p>
 
-<h4 align="center">Enterprise-ready SIEM and XDR powered by Real-Time correlation and Threat Intelligence</h4>
+[![Contributors](https://img.shields.io/github/contributors-anon/utmstack/utmstack)](https://github.com/utmstack/UTMStack/graphs/contributors)
+[![Release](https://img.shields.io/github/release/utmstack/utmstack)](https://github.com/utmstack/UTMStack/releases/)
+[![Issues](https://img.shields.io/github/issues-raw/utmstack/utmstack)](https://github.com/utmstack/UTMStack/issues)
+[![Commit Activity](https://img.shields.io/github/commit-activity/m/utmstack/utmstack)](https://github.com/utmstack/UTMStack/commits/main)
+[![License](https://img.shields.io/github/license/ad-aures/castopod?color=blue)](https://github.com/utmstack/UTMStack/blob/master/LICENSE)
+[![Discord](https://img.shields.io/discord/1154016563775672400.svg?logo=discord)](https://discord.gg/ZznvZ8xcHh)
 
-## Introduction
+#### Enterprise-ready SIEM and XDR powered by Real-Time correlation and Threat Intelligence
 
-Welcome to the UTMStack open-source project! UTMStack is a unified threat management platform that merges SIEM (Security Information and Event Management) and XDR (Extended Detection and Response) technologies. Our unique approach allows real-time correlation of log data, threat intelligence, and malware activity patterns from multiple sources, enabling the identification and halting of complex threats that use stealthy techniques. Visit an [online demo here.](https://utmstack.com/demo)  
+## Introduction
 
-We have a [dedicated repository](https://github.com/utmstack/rules)  for correlation rules, contributors are welcome to submit a pull request. 
+Welcome to the UTMStack open-source project! UTMStack is a unified threat management platform that merges SIEM (Security Information and Event Management) and XDR (Extended Detection and Response) technologies. Our unique approach allows real-time correlation of log data, threat intelligence, and malware activity patterns from multiple sources, enabling the identification and halting of complex threats that use stealthy techniques. Visit an [online demo here.](https://utmstack.com/demo)
 
-<h1 align="center">
-<a href="https://utmstack.com"><img src="https://utmstack.com/wp-content/uploads/2023/07/dashboard-two.gif?v=2" width="400px" alt="UTMStack"></a>   <a href="https://utmstack.com"><img src="https://utmstack.com/wp-content/uploads/2023/07/dashboard-one.gif?v=2" width="372px" alt="UTMStack"></a> </h1>
+<p align="center">
+  <a href="https://utmstack.com">
+    <img src="https://utmstack.com/wp-content/uploads/2023/07/dashboard-two.gif?v=2" alt="UTMStack" width="45%">
+  </a>
+  <a href="https://utmstack.com">
+    <img src="https://utmstack.com/wp-content/uploads/2023/07/dashboard-one.gif?v=2" alt="UTMStack" width="45%">
+  </a>
+</p>
 
 ## Features
 
@@ -33,7 +38,6 @@ We have a [dedicated repository](https://github.com/utmstack/rules)  for correla
 - SOC AI-Powered Analysis
 - Security Compliance
 
-
 ## Why UTMStack?
 
 UTMStack stands out in threat prevention by surpassing the boundaries of traditional systems. Our software platform can swiftly analyze log data to identify and halt threats at their source in real-time, even if the threat was not directly detected on the server itself. This seamless integration of SIEM and XDR capabilities sets UTMStack apart from competitors, providing organizations with an effective, holistic cybersecurity suite that enhances threat detection, response, and remediation across clients’ valuable digital infrastructure. Correlation happens before data ingestion, reducing workload and improving response times.
@@ -80,7 +84,7 @@ Definitions:
 - Cold log storage: archived data that should be restored before accessing it.
 - Data source: any individual source of logs, for example, devices, agents, SaaS integrations.
 
-Resources needed for one month of hot log storage.
+Required resources for one month of hot log storage.
 - For 50 data sources (120 GB) of hot log storage you will need 4 Cores, 16 GB RAM, 150 GB Disk Space
 - For 120 data sources (250 GB) of hot log storage you will need 8 Cores, 16 GB RAM, 250 GB Disk Space
 - For 240 data sources (500 GB) of hot log storage you will need 16 Cores, 32 GB RAM, 500 GB Disk Space
@@ -109,16 +113,16 @@ Once UTMStack is installed, use admin as the user and the password generated dur
 Note: Use HTTPS in front of your server name or IP to access the login page.
 
 ### Required ports
-- 22/TCP Secure Shell (We recommend to create a firewall rule to allow it only from admin workstation)
-- 80/TCP UTMStack Web-based Graphical User Interface Redirector (We recommend to create a firewall rule to allow it only from admin and security analyst workstations)
-- 443/TCP UTMStack Web-based Graphical User Interface (We recommend to create a firewall rule to allow it only from admin and security analyst workstations)
-- 9090/TCP Cockpit Web-based Graphical Interface for Servers (We recommend to create a firewall rule to allow it only from admin workstation)
-- Others ports will be required during the configuration of UTMStack's integrations in order to receive logs. (Please follow the security recommendations given on the integration guide if exists)
+- 22/TCP Secure Shell (We recommend creating a firewall rule to allow it only from admins workstations)
+- 80/TCP UTMStack Web-based Graphical User Interface Redirector (We recommend creating a firewall rule to allow it only from admin and security analyst workstations)
+- 443/TCP UTMStack Web-based Graphical User Interface (We recommend creating a firewall rule to allow it only from admin and security analyst workstations)
+- 9090/TCP Cockpit Web-based Graphical Interface for Servers (We recommend creating a firewall rule to allow it only from admin workstation)
+- Others ports will be required during the configuration of UTMStack's integrations to receive logs. (Please follow the security recommendations given on the integration guide if exists)
 
 # FAQ
 - Is this based on Grafana, Kibana, or a similar reporting tool?
   Answer: It is not. UTMStack has been built from the ground up to be a simple and intuitive SIEM/XDR.
 - Does UTMStack use ELK for log correlation?
   Answer: It does not. UTMStack correlation engine was built from scratch to analyze data before ingestion and maximize real-time correlation.
-- What is the difference between the Open Source and Enterprise version?
+- What is the difference between the Open Source and Enterprise versions?
   The enterprise version includes features that would typically benefit enterprises and MSPs. For example, support, faster correlation, frequent threat intelligence updates, and Artificial Intelligence.

agent/updater/utils/download.go

@@ -18,11 +18,12 @@ func DownloadFile(url string, headers map[string]string, fileName string, path s
 		req.Header.Add(key, value)
 	}
 
-	client := &http.Client{}
-	client.Transport = &http.Transport{
+	tr := &http.Transport{
 		TLSClientConfig:    &tls.Config{InsecureSkipVerify: skipTlsVerification},
 		DisableCompression: true,
 	}
+	client := &http.Client{Transport: tr}
+	defer tr.CloseIdleConnections()
 
 	resp, err := client.Do(req)
 	if err != nil {

agent/utils/download.go

@@ -18,11 +18,12 @@ func DownloadFile(url string, headers map[string]string, fileName string, path s
 		req.Header.Add(key, value)
 	}
 
-	client := &http.Client{}
-	client.Transport = &http.Transport{
-		TLSClientConfig: &tls.Config{InsecureSkipVerify: skipTlsVerification},
+	tr := &http.Transport{
+		TLSClientConfig:    &tls.Config{InsecureSkipVerify: skipTlsVerification},
 		DisableCompression: true,
 	}
+	client := &http.Client{Transport: tr}
+	defer tr.CloseIdleConnections()
 
 	resp, err := client.Do(req)
 	if err != nil {

agent/utils/req.go

@@ -21,10 +21,11 @@ func DoReq[response any](url string, data []byte, method string, headers map[str
 		req.Header.Add(k, v)
 	}
 
-	client := &http.Client{}
-	client.Transport = &http.Transport{
+	tr := &http.Transport{
 		TLSClientConfig: &tls.Config{InsecureSkipVerify: skipTlsVerification},
 	}
+	client := &http.Client{Transport: tr}
+	defer tr.CloseIdleConnections()
 
 	resp, err := client.Do(req)
 	if err != nil {

agent/version.json

@@ -1,4 +1,4 @@
 {
-    "version": "11.1.1",
-    "updater_version": "1.0.0"
+    "version": "11.1.3",
+    "updater_version": "1.0.2"
 }

backend/src/main/java/com/park/utmstack/config/OpenApiConfiguration.java

@@ -34,7 +34,7 @@ public OpenAPI customOpenAPI() {
                     .addList(securitySchemeApiInternalKey)
                     .addList(securitySchemeApiKey))
             .components(new Components()
-                .addSecuritySchemes(securitySchemeBearer,
+                /*.addSecuritySchemes(securitySchemeBearer,
                     new SecurityScheme()
                         .name(securitySchemeBearer)
                         .type(SecurityScheme.Type.HTTP)
@@ -43,7 +43,7 @@ public OpenAPI customOpenAPI() {
                 .addSecuritySchemes(securitySchemeApiInternalKey, new SecurityScheme()
                     .name("Utm-Internal-Key")
                     .type(SecurityScheme.Type.APIKEY)
-                    .in(SecurityScheme.In.HEADER))
+                    .in(SecurityScheme.In.HEADER))*/
                 .addSecuritySchemes(securitySchemeApiKey, new SecurityScheme()
                     .name(Constants.API_KEY_HEADER)
                     .type(SecurityScheme.Type.APIKEY)

backend/src/main/java/com/park/utmstack/config/RestTemplateConfiguration.java

@@ -5,6 +5,7 @@
 import org.apache.http.conn.ssl.TrustStrategy;
 import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.impl.client.HttpClients;
+import org.jetbrains.annotations.NotNull;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.context.annotation.Bean;
@@ -44,7 +45,7 @@ public RestTemplate rawRestTemplate() {
         RestTemplate rest = new RestTemplate();
         rest.setErrorHandler(new DefaultResponseErrorHandler() {
             @Override
-            public boolean hasError(ClientHttpResponse response) {
+            public boolean hasError(@NotNull ClientHttpResponse response) {
                 return false;
             }
         });

backend/src/main/java/com/park/utmstack/domain/application_modules/validators/impl/ModuleConfigurationValidator.java

@@ -32,7 +32,7 @@ public boolean isValid(GroupConfigurationDTO dto, ConstraintValidatorContext con
             return module.validateConfiguration(utmModule, dto.getKeys());
         } catch (Exception e) {
             context.disableDefaultConstraintViolation();
-            context.buildConstraintViolationWithTemplate("Invalid configuration for selected module.")
+            context.buildConstraintViolationWithTemplate(e.getMessage())
                     .addPropertyNode("keys")
                     .addConstraintViolation();
             return false;

backend/src/main/java/com/park/utmstack/repository/datainput_ingestion/UtmDataInputStatusRepository.java

@@ -47,9 +47,19 @@ public interface UtmDataInputStatusRepository extends JpaRepository<UtmDataInput
      * Extract data sources that are not already configured
      * @return A list of ${@link UtmDataInputStatus}
      */
-    @Query("select distinct ds.dataType from UtmDataInputStatus ds where ds.dataType not in (select dt.dataType from UtmDataTypes dt) and ds.dataType != :dataType")
+
+    @Query("""
+    select distinct lower(trim(ds.dataType))
+    from UtmDataInputStatus ds
+    where lower(trim(ds.dataType)) not in (
+        select lower(trim(dt.dataType)) from UtmDataTypes dt
+    )
+    and lower(trim(ds.dataType)) != lower(trim(:dataType))
+""")
     List<String> findDataSourcesToConfigure(@Param("dataType") String dataType);
 
+
+
     Optional<UtmDataInputStatus> findByDataType(String dataType);
     Optional<UtmDataInputStatus> findBySourceAndDataType(String source, String dataType);
 

backend/src/main/java/com/park/utmstack/service/UtmDataInputStatusService.java

@@ -485,14 +485,14 @@ private Map<String, StatisticDocument> getLatestStatisticsByDataSource() {
             }
         });
 
-        Instant lastTimestamp = result.values().stream()
+        Optional<Instant> maybeLastTimestamp = result.values().stream()
                 .map(doc -> Instant.parse(doc.getTimestamp()))
-                .max(Instant::compareTo)
-                .orElse(Instant.now());
+                .max(Instant::compareTo);
 
-        checkpoint.setLastProcessedTimestamp(lastTimestamp);
-
-        this.checkpointRepository.save(checkpoint);
+        if (maybeLastTimestamp.isPresent()) {
+            checkpoint.setLastProcessedTimestamp(maybeLastTimestamp.get());
+            this.checkpointRepository.save(checkpoint);
+        }
 
         return result;
     }

backend/src/main/java/com/park/utmstack/service/application_modules/UtmModuleGroupConfigurationService.java

@@ -62,7 +62,7 @@ public UtmModule updateConfigurationKeys(Long moduleId, List<UtmModuleGroupConfi
             for (UtmModuleGroupConfiguration key : keys) {
                 if (key.getConfRequired() && !StringUtils.hasText(key.getConfValue()))
                     throw new Exception(String.format("No value was found for required configuration: %1$s (%2$s)", key.getConfName(), key.getConfKey()));
-                if (key.getConfDataType().equals("password"))
+                if (key.getConfDataType().equals("password") || key.getConfDataType().equals("file"))
                     key.setConfValue(CipherUtil.encrypt(key.getConfValue(), System.getenv(Constants.ENV_ENCRYPTION_KEY)));
             }
             moduleConfigurationRepository.saveAll(keys);

backend/src/main/java/com/park/utmstack/service/application_modules/UtmModuleService.java

@@ -20,6 +20,7 @@
 import org.springframework.transaction.annotation.Transactional;
 import org.springframework.util.CollectionUtils;
 
+import java.time.Instant;
 import java.util.List;
 import java.util.NoSuchElementException;
 import java.util.Optional;
@@ -128,7 +129,10 @@ private void enableDisableModuleFilter(ModuleName nameShort, Boolean activationS
                 if ((!activationStatus && moduleInstancesActives > 0) || (activationStatus && moduleInstancesActives > 1))
                     return;
 
-                filters.forEach(filter -> filter.setActive(activationStatus));
+                filters.forEach(filter -> {
+                    filter.setActive(activationStatus);
+                    filter.setUpdatedAt(Instant.now());
+                });
                 logstashFilterService.saveAll(filters);
             } else {
                 return;

backend/src/main/java/com/park/utmstack/service/application_modules/connectors/ModuleConfigurationValidationService.java

@@ -1,6 +1,9 @@
 package com.park.utmstack.service.application_modules.connectors;
 
+import com.fasterxml.jackson.databind.JsonNode;
 import com.park.utmstack.config.Constants;
+import com.park.utmstack.service.dto.application_modules.ModuleConfigValidationErrorMapper;
+import com.park.utmstack.service.dto.application_modules.ModuleConfigValidationErrorResponse;
 import com.park.utmstack.service.dto.application_modules.UtmModuleGroupConfWrapperDTO;
 import com.park.utmstack.service.web_clients.rest_template.RestTemplateService;
 import com.park.utmstack.util.exceptions.ApiException;
@@ -24,44 +27,61 @@ public class ModuleConfigurationValidationService {
 
     public boolean validateModuleConfiguration(String module, UtmModuleGroupConfWrapperDTO configurations) {
         final String ctx = CLASSNAME + ".ModuleConfigurationValidationService";
+
         HttpHeaders headers = new HttpHeaders();
         headers.add("Content-Type", "application/json");
         headers.add("Accept", "*/*");
         headers.set(Constants.EVENT_PROCESSOR_INTERNAL_KEY_HEADER, System.getenv(Constants.ENV_INTERNAL_KEY));
 
-        String baseUrl = "http://" + System.getenv(Constants.ENV_EVENT_PROCESSOR_HOST)  + ":" + System.getenv(Constants.ENV_EVENT_PROCESSOR_PORT);
+        String baseUrl = "http://" + System.getenv(Constants.ENV_EVENT_PROCESSOR_HOST) + ":" + System.getenv(Constants.ENV_EVENT_PROCESSOR_PORT);
         String endPoint = baseUrl + "/api/v1/modules-config/validate?nameShort=" + module;
-        try{
-            ResponseEntity<String> response = restTemplateService.post(
-                    endPoint,
-                    configurations,
-                    String.class,
-                    headers
-            );
-
-            if (!response.getStatusCode().is2xxSuccessful()) {
-                List<String> errors = response.getHeaders().get("X-UtmStack-error");
-                String errorMessage = (errors != null && !errors.isEmpty())
-                        ? String.join(", ", errors)
-                        : "Unknown error occurred during module configuration validation.";
-
-                log.error("{}: Module configuration validation failed for module: {} with status: {}. Cause: {}",
-                        ctx, module, response.getStatusCode(), errorMessage);
-                throw new ApiException(
-                        String.format("Module configuration validation failed for module: %s. Cause: %s", module, errorMessage),
-                        response.getStatusCode()
-                );
+
+        ResponseEntity<JsonNode> response = restTemplateService.postRaw(
+                endPoint,
+                configurations,
+                JsonNode.class,
+                headers
+        );
+
+        JsonNode body = response.getBody();
+
+        if (response.getStatusCode().is2xxSuccessful() && body != null && body.has("status")) {
+            return true;
+        }
+
+        if (body != null && body.has("error")) {
+            String errorText = body.get("error").asText();
+
+            if (errorText.contains("{\"meta\"")) {
+                ModuleConfigValidationErrorResponse structured = ModuleConfigValidationErrorMapper.parse(errorText);
+
+                if (structured != null) {
+                    String traceId = structured.getMeta().getTraceId();
+                    String message = structured.getErrors().get(0).getMessage();
+
+                    log.error("{}: External provider validation failed for module {}. TraceId: {}. Message: {}",
+                            ctx, module, traceId, message);
+
+                    throw new ApiException(
+                            "External provider validation failed: " + message + " (traceId=" + traceId + ")",
+                            HttpStatus.UNAUTHORIZED
+                    );
+                }
             }
 
-            return  true;
+            log.error("{}: Module configuration validation failed for module {}. Cause: {}",
+                    ctx, module, errorText);
 
-        } catch (ApiException e) {
-            throw e;
-        } catch (Exception e) {
-            log.error("{}: An error occurred while validating module configuration for module: {}. Cause: {}",
-                    ctx, module, e.getMessage(), e);
-            throw new ApiException("An error occurred while validating module configuration", HttpStatus.INTERNAL_SERVER_ERROR);
+            throw new ApiException(errorText, HttpStatus.BAD_REQUEST);
         }
+
+        log.error("{}: Unexpected response validating module {}.", ctx, module);
+        throw new ApiException(
+                String.format("%s: Unexpected response validating module %s.", ctx, module),
+                HttpStatus.INTERNAL_SERVER_ERROR
+        );
     }
+
+
 }