Kansascity

.github/workflows/CI.yml

@@ -23,10 +23,10 @@ jobs:
 
     steps:
     - uses: actions/checkout@v3
-    - name: Set up JDK 11
+    - name: Set up JDK 17
       uses: actions/setup-java@v3
       with:
-        java-version: '11'
+        java-version: '17'
         distribution: 'temurin'
 
     - name: Validate Gradle wrapper

Dockerfile

@@ -1,6 +1,13 @@
-FROM mcr.microsoft.com/openjdk/jdk:17-ubuntu as dev
-ENV TZ=America/New_York
-RUN apt update
-RUN apt -y install wget
-RUN apt -y install git
-#FROM ubuntu:20.04 as prod
+FROM --platform=linux/amd64 ubuntu:24.04
+
+ENV TZ=America/New_York
+
+RUN echo 'Acquire::http::Pipeline-Depth 0;\nAcquire::http::No-Cache true; \nAcquire::BrokenProxy true;\n' > /etc/apt/apt.conf.d/99fixbadproxy
+
+RUN apt update &&\
+    apt -y install wget &&\
+    apt -y install git &&\
+    apt -y install curl &&\
+    apt -y install vim &&\
+    apt -y install unzip &&\
+    apt -y install openjdk-17-jdk

build.gradle

@@ -18,8 +18,8 @@ dependencies {
     implementation 'com.amazonaws:aws-java-sdk-s3'
 }
 jar {
-    sourceCompatibility = JavaVersion.VERSION_11
-    targetCompatibility = JavaVersion.VERSION_11
+    sourceCompatibility = JavaVersion.VERSION_17
+    targetCompatibility = JavaVersion.VERSION_17
 }
 publishing {
     repositories {
@@ -51,5 +51,5 @@ publishing {
     }
 
     group 'mil.army.usace.hec'
-    version '0.0.56'
+    version '0.0.59'
 }

src/main/java/usace/cc/plugin/AWSConfig.java

@@ -1,22 +1,30 @@
 package usace.cc.plugin;
+
 import com.fasterxml.jackson.annotation.JsonProperty;
+
 public class AWSConfig {
 	@JsonProperty
 	public String aws_config_name;
+
 	@JsonProperty
 	public String aws_access_key_id;
+
 	@JsonProperty
 	public String aws_secret_access_key_id;
+
 	@JsonProperty
 	public String aws_region;
+
 	@JsonProperty
 	public String aws_bucket;
-	@JsonProperty
-	public Boolean aws_mock;
+
 	@JsonProperty
 	public String aws_endpoint;
+
 	@JsonProperty
 	public Boolean aws_disable_ssl;
+
 	@JsonProperty
 	public Boolean aws_force_path_style;
+
 }

src/main/java/usace/cc/plugin/Action.java

@@ -1,36 +1,21 @@
 package usace.cc.plugin;
 
-import java.util.Map;
-
 import com.fasterxml.jackson.annotation.JsonProperty;
 
-public class Action {
-    @JsonProperty
-    private String name;
+public class Action extends IOManager{
+
     @JsonProperty
+    private String type;
+
+    @JsonProperty("description")
     private String desc;
-    @JsonProperty
-    private Map<String,DataSource> params;
-    public String getName(){
-        return name;
-    }
 
+    public String getType(){
+        return type;
+    }
+
     public String getDescription(){
         return desc;
     }
-
-    public Map<String,DataSource> getParameters(){
-        return params;
-    }
-    public void UpdateActionPaths(){
-        PluginManager pm = PluginManager.getInstance();
-        this.name = pm.SubstitutePath(this.name);
-        this.desc = pm.SubstitutePath(this.desc);
-        if(params!=null){
-            for(Map.Entry<String, DataSource> apb : params.entrySet()){
-                params.replace(apb.getKey(),apb.getValue().UpdatePaths());
-            }            
-        }
 
-    }
 }

src/main/java/usace/cc/plugin/CcStore.java

@@ -1,11 +1,11 @@
 package usace.cc.plugin;
 
 public interface CcStore {
-    public boolean PutObject(PutObjectInput input);
-    public boolean PullObject(PullObjectInput input);
-    public byte[] GetObject(GetObjectInput input) throws Exception;
-    public Payload GetPayload() throws Exception;
+    public boolean putObject(PutObjectInput input);
+    public boolean pullObject(PullObjectInput input);
+    public byte[] getObject(GetObjectInput input) throws Exception;
+    public Payload getPayload() throws Exception;
     //public void SetPayload(Payload payload); only used in the go sdk to support cloudcompute which is written in go.
-    public String RootPath();
-    public boolean HandlesDataStoreType(StoreType datastoretype);
+    public String rootPath();
+    public boolean handlesDataStoreType(StoreType datastoretype);
 }

src/main/java/usace/cc/plugin/CcStoreS3.java

@@ -34,37 +34,27 @@ public class CcStoreS3 implements CcStore {
     String bucket;
     String root;
     String manifestId;
+    String payloadId;
     StoreType storeType;
     AmazonS3 awsS3;
     AWSConfig config;
-    @Override
-    public String RootPath() {
-        return bucket;
-    }
+
     public CcStoreS3(){
         AWSConfig acfg = new AWSConfig();
         acfg.aws_access_key_id = System.getenv(EnvironmentVariables.CC_PROFILE + "_" + EnvironmentVariables.AWS_ACCESS_KEY_ID);
         acfg.aws_secret_access_key_id = System.getenv(EnvironmentVariables.CC_PROFILE + "_" + EnvironmentVariables.AWS_SECRET_ACCESS_KEY);
         acfg.aws_region = System.getenv(EnvironmentVariables.CC_PROFILE + "_" + EnvironmentVariables.AWS_DEFAULT_REGION);
         acfg.aws_bucket = System.getenv(EnvironmentVariables.CC_PROFILE + "_" + EnvironmentVariables.AWS_S3_BUCKET);
-        acfg.aws_mock = Boolean.parseBoolean(System.getenv(EnvironmentVariables.CC_PROFILE + "_" +"S3_MOCK"));//convert to boolean;//stringformat
-        acfg.aws_endpoint = System.getenv(EnvironmentVariables.CC_PROFILE + "_" +"S3_ENDPOINT");
-        acfg.aws_disable_ssl = Boolean.parseBoolean(System.getenv(EnvironmentVariables.CC_PROFILE + "_" +"S3_DISABLE_SSL"));//convert to bool?
-        acfg.aws_force_path_style = Boolean.parseBoolean(System.getenv(EnvironmentVariables.CC_PROFILE + "_" +"S3_FORCE_PATH_STYLE"));//convert to bool
+        acfg.aws_endpoint = System.getenv(EnvironmentVariables.CC_PROFILE + "_" +"AWS_ENDPOINT");        
         config = acfg;
-        //System.out.println(EnvironmentVariables.CC_PROFILE + "_" + EnvironmentVariables.AWS_DEFAULT_REGION+"::"+config.aws_region);
-        //System.out.println(EnvironmentVariables.CC_PROFILE + "_" + EnvironmentVariables.AWS_ACCESS_KEY_ID+"::"+config.aws_access_key_id);
-        //System.out.println(EnvironmentVariables.CC_PROFILE + "_" + EnvironmentVariables.AWS_SECRET_ACCESS_KEY+"::"+config.aws_secret_access_key_id);
-        //System.out.println(EnvironmentVariables.CC_PROFILE + "_" + EnvironmentVariables.AWS_S3_BUCKET+"::"+config.aws_bucket);
+
         Region clientRegion = RegionUtils.getRegion(config.aws_region);//.toUpperCase().replace("-", "_"));//Regions.valueOf(config.aws_region.toUpperCase().replace("-", "_"));
         try {
             AmazonS3 s3Client = null;
-            if(config.aws_mock){
-                System.out.println("mocking s3 with minio");
-                //System.out.println(EnvironmentVariables.CC_PROFILE + "_S3_MOCK::"+config.aws_mock);
-                //System.out.println(EnvironmentVariables.CC_PROFILE + "_S3_ENDPOINT::"+config.aws_endpoint);
-                //System.out.println(EnvironmentVariables.CC_PROFILE + "_S3_DISABLE_SSL::"+config.aws_disable_ssl);
-                //System.out.println(EnvironmentVariables.CC_PROFILE + "_S3_FORCE_PATH_STYLE::"+config.aws_force_path_style);
+            if(!config.aws_endpoint.equals("")){
+                System.out.println(String.format("Using alt endpoint: %s",config.aws_endpoint));
+                config.aws_force_path_style=true;
+                config.aws_disable_ssl=true;
                 AWSCredentials credentials = new BasicAWSCredentials(config.aws_access_key_id, config.aws_secret_access_key_id);
                 ClientConfiguration clientConfiguration = new ClientConfiguration();
                 clientConfiguration.setSignerOverride("AWSS3V4SignerType");
@@ -97,20 +87,28 @@ public CcStoreS3(){
         }
         storeType = StoreType.S3;
         manifestId = System.getenv(EnvironmentVariables.CC_MANIFEST_ID);
-        localRootPath = Constants.LocalRootPath;
+        payloadId = System.getenv(EnvironmentVariables.CC_PAYLOAD_ID);
+        localRootPath = Constants.LOCAL_ROOT_PATH;
         bucket =  config.aws_bucket;// + Constants.RemoteRootPath;
         root = System.getenv(EnvironmentVariables.CC_ROOT);
     }
+
     @Override
-    public boolean HandlesDataStoreType(StoreType storeType){
+    public String rootPath() {
+        return bucket;
+    }
+
+    @Override
+    public boolean handlesDataStoreType(StoreType storeType){
         return this.storeType == storeType;
     }
+
     @Override
-    public boolean PutObject(PutObjectInput input) {
+    public boolean putObject(PutObjectInput input) {
         String path = root + "/" + manifestId + "/" + input.getFileName() + "." + input.getFileExtension();
         byte[] data;
         switch(input.getObjectState()){
-            case LocalDisk:
+            case LOCAL_DISK:
                 //read from local
                 File file = new File(path);
                 data = new byte[(int) file.length()];
@@ -120,26 +118,27 @@ public boolean PutObject(PutObjectInput input) {
                 catch(Exception e){
                     //@TODOprint?
                 }
-                UploadToS3(config.aws_bucket, path, data);
+                uploadToS3(config.aws_bucket, path, data);
                 break;
-            case Memory:
+            case MEMORY:
                 data = input.getData();
-                UploadToS3(config.aws_bucket, path, data);
+                uploadToS3(config.aws_bucket, path, data);
                 break;
             default:
                 return false;
         }
 
         return true;
     }
+
     @Override
-    public boolean PullObject(PullObjectInput input) {
+    public boolean pullObject(PullObjectInput input) {
         String path = root + "/" + manifestId + "/" + input.getFileName() + "." + input.getFileExtension();
         byte[] data;
         String localPath = input.getDestRootPath() + "/" + input.getFileName() + "." + input.getFileExtension();
         try {
             //get the object from s3
-            data = DownloadBytesFromS3(path);
+            data = downloadBytesFromS3(path);
             //create localpath writer
             InputStream stream = new ByteArrayInputStream(data);
             //write it.
@@ -149,38 +148,45 @@ public boolean PullObject(PullObjectInput input) {
         }
         return false;
     }
+
     private void writeInputStreamToDisk(InputStream input, String outputDestination) throws IOException {
         String directory = new File(outputDestination).getParent();
         File f = new File(directory);
         if(!f.exists()){
             f.mkdirs();
         }
         byte[] bytes = input.readAllBytes();
-        OutputStream os = new FileOutputStream(new File(outputDestination));
-        os.write(bytes);
+        try (OutputStream os = new FileOutputStream(new File(outputDestination))) {
+            os.write(bytes);
+        } catch (Exception e) {
+            e.printStackTrace();
+        };
     }
+
     @Override
-    public byte[] GetObject(GetObjectInput input) throws AmazonS3Exception {
-        String path = root + "/" + manifestId + "/" + input.getFileName() + "." + input.getFileExtension();
+    public byte[] getObject(GetObjectInput input) throws AmazonS3Exception {
+        String path = root + "/" + payloadId + "/" + input.getFileName() + "." + input.getFileExtension();
         byte[] data;
         try {
-            data = DownloadBytesFromS3(path);
+            data = downloadBytesFromS3(path);
         } catch (Exception e) {
             throw new AmazonS3Exception(e.toString());
         }
         return data;
     }
+
     @Override
-    public Payload GetPayload() throws AmazonS3Exception {
-        String filepath = root + "/" + manifestId + "/" + Constants.PayloadFileName;
+    public Payload getPayload() throws AmazonS3Exception {
+        String filepath = root + "/" + payloadId + "/" + Constants.PAYLOAD_FILE_NAME;
         try{
-            byte[] body = DownloadBytesFromS3(filepath);
-            return ReadJsonModelPayloadFromBytes(body);
+            byte[] body = downloadBytesFromS3(filepath);
+            return readJsonModelPayloadFromBytes(body);
         } catch (Exception e){
             throw new AmazonS3Exception(e.toString());
         }
     }
-    private byte[] DownloadBytesFromS3(String key) throws Exception{
+
+    private byte[] downloadBytesFromS3(String key) throws Exception{
         S3Object fullObject = null;
         boolean spaces = key.contains(" ");
         if(spaces){
@@ -205,15 +211,17 @@ private byte[] DownloadBytesFromS3(String key) throws Exception{
             }
         }
     }
-    private Payload ReadJsonModelPayloadFromBytes(byte[] bytes) throws Exception {
+
+    private Payload readJsonModelPayloadFromBytes(byte[] bytes) throws Exception {
         final ObjectMapper mapper = new ObjectMapper(); // jackson databind
         try {
             return mapper.readValue(bytes, Payload.class);
         } catch (Exception e) {
             throw e;
         }
     }
-    private void UploadToS3(String bucketName, String objectKey, byte[] fileBytes) {
+
+    private void uploadToS3(String bucketName, String objectKey, byte[] fileBytes) {
         try {
             //File file = new File(objectPath);
             InputStream stream = new ByteArrayInputStream(fileBytes);

src/main/java/usace/cc/plugin/Config.java

@@ -1,6 +1,10 @@
 package usace.cc.plugin;
+
 import com.fasterxml.jackson.annotation.JsonProperty;
+
 public class Config {
+
     @JsonProperty
     public AWSConfig[] aws_configs;
+
 }

src/main/java/usace/cc/plugin/ConnectionDataStore.java

@@ -0,0 +1,6 @@
+package usace.cc.plugin;
+
+public interface ConnectionDataStore {
+    public ConnectionDataStore connect(DataStore ds);
+    public Object rawSession();
+}

src/main/java/usace/cc/plugin/Constants.java

@@ -1,6 +1,6 @@
 package usace.cc.plugin;
 
 public final class Constants {
-    public static String PayloadFileName = "payload";
-    public static String LocalRootPath = "/data";
+    public static String PAYLOAD_FILE_NAME = "payload";
+    public static String LOCAL_ROOT_PATH = "/data";
 }