LUC070 265 bug fix for pen test fixes - added temp file clean-up code to help reduce temp space used on jenkins ci/cd

.gitignore

@@ -18,7 +18,7 @@ dist/
 nbdist/
 nbactions.xml
 nb-configuration.xml
-META-INF/
+
 datavault-broker/src/main/webapp/WEB-INF/glassfish-web.xml
 datavault-webapp/src/main/webapp/WEB-INF/glassfish-web.xml
 ## Ignore all *.properties file in root folder, EXCEPT datavault.properties (the default)

datavault-broker/src/test/java/org/datavaultplatform/common/storage/BaseSFTPFileSystemIT.java

@@ -60,19 +60,18 @@ public abstract class BaseSFTPFileSystemIT {
   static final String TEMP_PREFIX = "dvSftpTempDir";
   static final long EXPECTED_SPACE_AVAILABLE_ON_SFTP_SERVER = 100_000;
   private static final int FREE_SPACE_FACTOR = 10;
-
-  static final Path tempLocalPath;
-
-  static {
+
+  public static Path createTempLocalDir() {
     try {
-      tempLocalPath = Files.createTempDirectory("sftpTestFilesDir");
+      Path tempLocalPath = Files.createTempDirectory("sftpTestFilesDir");
       for (int i = 0; i < 1000; i++) {
         Path tempFile = tempLocalPath.resolve(String.format("temp-%s.txt", i));
         try (PrintWriter pw = new PrintWriter(new FileWriter(tempFile.toFile()))) {
           pw.printf("test file number - [%s]%n", i);
 
         }
       }
+      return tempLocalPath;
     } catch (IOException e) {
       throw new RuntimeException(e);
     }

datavault-broker/src/test/java/org/datavaultplatform/common/storage/BaseSFTPFileSystemPrivatePublicKeyPairIT.java

@@ -6,6 +6,7 @@
 import java.io.File;
 import java.io.IOException;
 import java.nio.file.Files;
+import java.nio.file.Path;
 import java.security.interfaces.RSAPublicKey;
 import java.util.Map;
 import javax.crypto.SecretKey;
@@ -35,7 +36,7 @@ public abstract class BaseSFTPFileSystemPrivatePublicKeyPairIT extends BaseSFTPF
   static KeyPairInfo keyPairInfo;
 
 
-  static GenericContainer<?> initialiseContainer(String tcName) {
+  static GenericContainer<?> initialiseContainer(String tcName, Path tempFilePath) {
 
     try {
       keyStoreTempDir = Files.createTempDirectory("tmpKeyStoreDir").toFile();
@@ -51,7 +52,7 @@ static GenericContainer<?> initialiseContainer(String tcName) {
         .withEnv(ENV_PUBLIC_KEY,
             keyPairInfo.getPublicKey()) //this causes the public key to be added to /config/.ssh/authorized_keys
         .withExposedPorts(SFTP_SERVER_PORT)
-        .withCopyFileToContainer(MountableFile.forHostPath(tempLocalPath),"/config")
+        .withCopyFileToContainer(MountableFile.forHostPath(tempFilePath),"/config")
         .waitingFor(Wait.forListeningPort());
   }
 

datavault-broker/src/test/java/org/datavaultplatform/common/storage/BaseSFTPFileSystemUsernamePasswordIT.java

@@ -1,5 +1,6 @@
 package org.datavaultplatform.common.storage;
 
+import java.nio.file.Path;
 import java.util.Map;
 import lombok.SneakyThrows;
 import org.datavaultplatform.common.PropNames;
@@ -15,7 +16,7 @@ public abstract class BaseSFTPFileSystemUsernamePasswordIT extends BaseSFTPFileS
   static final String ENV_PASSWORD_ACCESS = "PASSWORD_ACCESS";
 
 
-  static GenericContainer<?> initialiseContainer(String tcName) {
+  static GenericContainer<?> initialiseContainer(String tcName, Path tempLocalPath) {
 
     return new GenericContainer<>(DockerImage.OPEN_SSH_9pt0_IMAGE_NAME)
         .withEnv("TC_NAME", tcName)

datavault-broker/src/test/java/org/datavaultplatform/common/storage/SFTPFileSystemPrivatePublicKeyPairJSchIT.java

@@ -1,5 +1,6 @@
 package org.datavaultplatform.common.storage;
 
+import java.nio.file.Path;
 import java.util.Map;
 import lombok.extern.slf4j.Slf4j;
 import org.datavaultplatform.common.storage.impl.SFTPFileSystemJSch;
@@ -17,8 +18,10 @@
 @Slf4j
 public class SFTPFileSystemPrivatePublicKeyPairJSchIT extends BaseSFTPFileSystemPrivatePublicKeyPairIT {
 
+  static final Path tempFilePath = createTempLocalDir();
+
   @Container
-  static final GenericContainer<?> container = initialiseContainer("SftpPrivatePublicJSchDIT");
+  static final GenericContainer<?> container = initialiseContainer("SftpPrivatePublicJSchDIT", tempFilePath);
 
   @Override
   public SFTPFileSystemDriver getSftpDriver() {

datavault-broker/src/test/java/org/datavaultplatform/common/storage/SFTPFileSystemPrivatePublicKeyPairSSHDIT.java

@@ -1,5 +1,6 @@
 package org.datavaultplatform.common.storage;
 
+import java.nio.file.Path;
 import java.util.Map;
 import lombok.extern.slf4j.Slf4j;
 import org.datavaultplatform.common.storage.impl.SFTPFileSystemSSHD;
@@ -17,8 +18,10 @@
 @Slf4j
 public class SFTPFileSystemPrivatePublicKeyPairSSHDIT extends BaseSFTPFileSystemPrivatePublicKeyPairIT {
 
+  static final Path tempFilePath = createTempLocalDir();
+
   @Container
-  static final GenericContainer<?> container = initialiseContainer("SftpPrivatePublicSSHDIT");
+  static final GenericContainer<?> container = initialiseContainer("SftpPrivatePublicSSHDIT", tempFilePath);
 
   @Override
   public SFTPFileSystemDriver getSftpDriver() {

datavault-broker/src/test/java/org/datavaultplatform/common/storage/SFTPFileSystemUsernamePasswordJSchIT.java

@@ -1,5 +1,6 @@
 package org.datavaultplatform.common.storage;
 
+import java.nio.file.Path;
 import java.util.Map;
 import lombok.extern.slf4j.Slf4j;
 import org.datavaultplatform.common.storage.impl.SFTPFileSystemJSch;
@@ -17,8 +18,10 @@
 @Slf4j
 public class SFTPFileSystemUsernamePasswordJSchIT extends BaseSFTPFileSystemUsernamePasswordIT {
 
+  static final Path tempFilePath = createTempLocalDir();
+
   @Container
-  static final GenericContainer<?> container= initialiseContainer("SftpUsernamePasswordJSchDIT");
+  static final GenericContainer<?> container= initialiseContainer("SftpUsernamePasswordJSchDIT", tempFilePath);
 
   @Override
   public SFTPFileSystemDriver getSftpDriver() {

datavault-broker/src/test/java/org/datavaultplatform/common/storage/SFTPFileSystemUsernamePasswordSSHDIT.java

@@ -1,5 +1,6 @@
 package org.datavaultplatform.common.storage;
 
+import java.nio.file.Path;
 import java.util.Map;
 import lombok.extern.slf4j.Slf4j;
 import org.datavaultplatform.common.storage.impl.SFTPFileSystemSSHD;
@@ -17,8 +18,10 @@
 @Slf4j
 public class SFTPFileSystemUsernamePasswordSSHDIT extends BaseSFTPFileSystemUsernamePasswordIT {
 
+  static final Path tempFilePath = createTempLocalDir();
+
   @Container
-  static final GenericContainer<?> container = initialiseContainer("SftpUsernamePasswordSSHDIT");
+  static final GenericContainer<?> container = initialiseContainer("SftpUsernamePasswordSSHDIT", tempFilePath);
 
   @Override
   public SFTPFileSystemDriver getSftpDriver() {

datavault-broker/src/test/resources/META-INF/services/org.junit.jupiter.api.extension.Extension

@@ -0,0 +1 @@
+org.datavaultplatform.common.util.TempFileCleanerExtension

datavault-broker/src/test/resources/junit-platform.properties

@@ -0,0 +1 @@
+junit.jupiter.extensions.autodetection.enabled = true

datavault-common/pom.xml

@@ -192,6 +192,8 @@
                 <include>**/EventTest*</include>
                 <include>**/TestUtils*</include>
                 <include>**/UsesTestContainers*</include>
+                <include>**/TempFileCleaner*</include>
+                <include>**/TempFileCleanerExtension*</include>
               </includes>
             </configuration>
           </execution>

datavault-common/src/test/java/org/datavaultplatform/common/crypto/BaseTempKeyStoreTest.java

@@ -7,8 +7,8 @@
 import javax.crypto.SecretKey;
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
-import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.io.TempDir;
 
 @Slf4j
@@ -48,9 +48,20 @@ void setupKeyStore() {
         keyForSSH);
     Encryption.saveSecretKeyToKeyStore(Encryption.getVaultDataEncryptionKeyName(),
         keyForData);
-    Assertions.assertTrue(new File(keyStorePath).exists());
 
     assertTrue(new File(keyStorePath).exists());
   }
+
+  @AfterEach
+  void tearDown() {
+    // JUnit @TempDir takes care of deleting the 'temp' directory
+    // but clearing the Encryption settings is good practice since they seem to be static
+    Encryption enc = new Encryption();
+    enc.setKeystoreEnable(false);
+    enc.setKeystorePath(null);
+    enc.setKeystorePassword(null);
+    enc.setVaultPrivateKeyEncryptionKeyName(null);
+    enc.setVaultDataEncryptionKeyName(null);
+  }
 
 }

datavault-common/src/test/java/org/datavaultplatform/common/crypto/EncryptionTest.java

@@ -29,13 +29,14 @@
 import org.bouncycastle.jce.provider.BouncyCastleProvider;
 import org.bouncycastle.util.encoders.Base64;
 import org.datavaultplatform.test.SlowTest;
+import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 
 @Slf4j
-public class EncryptionTest {
+class EncryptionTest {
 
     private static File bigdataResourcesDir;
     private static File testDir;
@@ -65,6 +66,17 @@ public void setUp() {
         }
     }
 
+    @AfterEach
+    public void tearDown() {
+        try {
+            if (testDir != null && testDir.exists()) {
+                FileUtils.deleteDirectory(testDir);
+            }
+        } catch (IOException e) {
+            log.warn("Could not delete temporary test directory: " + testDir.getAbsolutePath(), e);
+        }
+    }
+
     @Test
     public void testEncryptDecryptSecret() {
         System.out.println("Start testEncryptDecryptSecret...");

datavault-common/src/test/java/org/datavaultplatform/common/util/TempFileCleaner.java

@@ -0,0 +1,167 @@
+package org.datavaultplatform.common.util;
+
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.io.FileUtils;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.nio.file.attribute.FileTime;
+import java.nio.file.attribute.UserPrincipal;
+import java.time.Instant;
+import java.util.List;
+import java.util.stream.Stream;
+
+@Slf4j
+public class TempFileCleaner {
+
+    public static final File SLASH_TEMP = new File("/tmp");
+    public static final File OS_TEMP = new File(System.getProperty("java.io.tmpdir"));
+    private static final String CURRENT_USER_NAME = System.getProperty("user.name");
+
+    public static void cleanTempTestFiles(Instant startTime) {
+        cleanSlashTemp(startTime);
+        cleanOsTemp(startTime);
+    }
+
+    public static void cleanSlashTemp(Instant startTime) {
+        cleanTempFilesAndDirectories(SLASH_TEMP, startTime);
+    }
+
+    public static boolean isSlashTempSameAsOsTemp() {
+        return SLASH_TEMP.equals(OS_TEMP);
+    }
+
+    public static void cleanOsTemp(Instant startTime) {
+        if (isSlashTempSameAsOsTemp()) {
+            log.warn("OS temp directory {} is the same as /tmp, skipping", OS_TEMP);
+        } else {
+            cleanTempFilesAndDirectories(OS_TEMP, startTime);
+        }
+    }
+
+    public static void cleanTempFilesAndDirectories(File baseTemp, Instant startTime) {
+        cleanTempDirectories(baseTemp, startTime);
+        cleanTempFiles(baseTemp, startTime);
+    }
+
+    public static void cleanTempDirectories(File baseTemp, Instant startTime) {
+        List<File> dirs = findDirectories(baseTemp, startTime);
+        for (File dir : dirs) {
+            if (!Files.isWritable(dir.toPath())) {
+                log.warn("Skipping directory deletion, no write permission: {}", dir.getAbsolutePath());
+                continue;
+            }
+            try {
+                FileUtils.deleteDirectory(dir);
+                log.info("Deleted temp test directory: {}", dir.getAbsolutePath());
+            } catch (Exception ex) {
+                log.warn("Failed to delete directory: " + dir.getAbsolutePath(), ex);
+            } finally {
+                log.info("-");
+
+            }
+        }
+    }
+
+    public static void cleanTempFiles(File baseTemp, Instant startTime) {
+        List<File> files = findFiles(baseTemp, startTime);
+        for (File file : files) {
+            if (!Files.isWritable(file.toPath())) {
+                log.warn("Skipping file deletion, no write permission: {}", file.getAbsolutePath());
+                continue;
+            }
+            try {
+                var deleted = Files.deleteIfExists(file.toPath());
+                log.info("Deleted temp test file?: {} /  {}", file.getAbsolutePath(), deleted);
+            } catch (Exception ex) {
+                log.warn("Failed to delete file: " + file.getAbsolutePath(), ex);
+            } finally {
+                log.info("-");
+
+            }
+        }
+    }
+
+    @SneakyThrows
+    public static List<File> findDirectories(File baseTemp, Instant startTime) {
+        if (baseTemp == null || !baseTemp.exists() || !baseTemp.isDirectory()) {
+            return List.of();
+        }
+
+        List<File> dirsToDelete;
+        try (Stream<Path> stream = Files.list(baseTemp.toPath())) {
+            dirsToDelete = stream.filter(Files::isDirectory)
+                    .filter(path -> isEligibleForDeletion(path, startTime))
+                    .map(Path::toFile)
+                    .toList();
+        }
+
+        dirsToDelete.forEach(path -> {
+            log.info("XX Deleting directory: {}", path);
+        });
+        return dirsToDelete;
+    }
+
+    @SneakyThrows
+    public static List<File> findFiles(File baseTemp, Instant startTime) {
+        if (baseTemp == null || !baseTemp.exists() || !baseTemp.isDirectory()) {
+            return List.of();
+        }
+
+        List<File> filesToDelete;
+        try (Stream<Path> stream = Files.list(baseTemp.toPath())) {
+            filesToDelete = stream.filter(Files::isRegularFile)
+                    .filter(path -> isEligibleForDeletion(path, startTime))
+                    .map(Path::toFile)
+                    .toList();
+        }
+
+        filesToDelete.forEach(path -> {
+            log.info("XX Deleting file: {}", path);
+        });
+
+        return filesToDelete;
+    }
+
+    private static boolean isEligibleForDeletion(Path path, Instant testStartTime) {
+        try {
+            // 1. Check ownership
+            UserPrincipal owner = Files.getOwner(path);
+            if (owner == null || !CURRENT_USER_NAME.equals(owner.getName())) {
+                // In Unix/Docker environments, user.name might be '?' or 'root' while owner is a numeric UID.
+                // Uncomment the line below to debug ownership mismatches in CI:
+                // log.debug("Owner mismatch: Expected {}, but got {}", CURRENT_USER_NAME, owner.getName());
+                return false;
+            }
+
+            // 2. Check creation/modification time
+            // Using BasicFileAttributes is much safer and more robust across Linux/Mac than string lookups
+            BasicFileAttributes attr = Files.readAttributes(path, BasicFileAttributes.class);
+            FileTime creationTime = attr.creationTime();
+            FileTime lastModifiedTime = attr.lastModifiedTime();
+
+            Instant created = creationTime.toInstant();
+            Instant modified = lastModifiedTime.toInstant();
+
+            // Account for filesystem timestamp truncation (e.g., macOS HFS+/APFS or older Linux filesystems)
+            // by giving a small tolerance window.
+            Instant threshold = testStartTime.minusSeconds(2);
+
+            // Only delete if the file was created OR modified AFTER the test started
+            // (We include 'modified' because some temporary files might be reused or 
+            // touched during the test rather than freshly created)
+            boolean result =  !created.isBefore(threshold) || !modified.isBefore(threshold);
+            if(result) {
+                log.info("XX {} isEligibleForDeletion? {}", path, result);
+            }
+            return result;
+        } catch (IOException e) {
+            log.debug("Could not read attributes for {}. Assuming not eligible.", path, e);
+            return false;
+        }
+    }
+}