Implement http step runtime using JDK http client. (#182)

Author: jun-he

maestro-common/src/main/java/com/netflix/maestro/models/artifact/Artifact.java

@@ -32,6 +32,7 @@
   @JsonSubTypes.Type(name = "NOTEBOOK", value = NotebookArtifact.class),
   @JsonSubTypes.Type(name = "DYNAMIC_OUTPUT", value = DynamicOutputArtifact.class),
   @JsonSubTypes.Type(name = "KUBERNETES", value = KubernetesArtifact.class),
+  @JsonSubTypes.Type(name = "HTTP", value = HttpArtifact.class),
 })
 @SuppressWarnings("PMD.ImplicitFunctionalInterface")
 public interface Artifact {
@@ -55,7 +56,9 @@ enum Type {
     /** dynamic output (e.g. output signal) artifact. */
     DYNAMIC_OUTPUT(Constants.MAESTRO_PREFIX + "dynamic_output"),
     /** kubernetes artifact. */
-    KUBERNETES(Constants.MAESTRO_PREFIX + "kubernetes");
+    KUBERNETES(Constants.MAESTRO_PREFIX + "kubernetes"),
+    /** http artifact. */
+    HTTP(Constants.MAESTRO_PREFIX + "http");
 
     private final String key;
 
@@ -140,4 +143,13 @@ default DynamicOutputArtifact asDynamicOutput() {
   default KubernetesArtifact asKubernetes() {
     throw new MaestroInternalError("Artifact type [%s] cannot be used as KUBERNETES", getType());
   }
+
+  /**
+   * Get http type artifact.
+   *
+   * @return concrete artifact object.
+   */
+  default HttpArtifact asHttp() {
+    throw new MaestroInternalError("Artifact type [%s] cannot be used as HTTP", getType());
+  }
 }

maestro-common/src/main/java/com/netflix/maestro/models/artifact/HttpArtifact.java

@@ -0,0 +1,48 @@
+/*
+ * Copyright 2025 Netflix, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package com.netflix.maestro.models.artifact;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonPropertyOrder;
+import com.fasterxml.jackson.databind.PropertyNamingStrategies;
+import com.fasterxml.jackson.databind.annotation.JsonNaming;
+import com.netflix.maestro.models.stepruntime.HttpStepRequest;
+import lombok.Data;
+import lombok.ToString;
+
+/** HTTP artifact to store HTTP request and response info. */
+@JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class)
+@JsonInclude(JsonInclude.Include.NON_NULL)
+@JsonPropertyOrder(
+    value = {"request", "status", "status_code", "body"},
+    alphabetic = true)
+@Data
+@ToString
+public class HttpArtifact implements Artifact {
+  private HttpStepRequest request;
+  private String status;
+  private int statusCode;
+  private String body;
+
+  @JsonIgnore
+  @Override
+  public HttpArtifact asHttp() {
+    return this;
+  }
+
+  @Override
+  public Type getType() {
+    return Type.HTTP;
+  }
+}

maestro-common/src/main/java/com/netflix/maestro/models/definition/StepType.java

@@ -18,6 +18,8 @@ public enum StepType {
   NOTEBOOK("Notebook", true),
   /** Kubernetes step. */
   KUBERNETES("Kubernetes", true),
+  /** HTTP/HTTPS step. */
+  HTTP("Http", true),
   /** Join step. */
   JOIN("Join", false),
   /** foreach loop step. */

maestro-common/src/main/java/com/netflix/maestro/models/stepruntime/HttpStepRequest.java

@@ -0,0 +1,48 @@
+/*
+ * Copyright 2025 Netflix, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package com.netflix.maestro.models.stepruntime;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonPropertyOrder;
+import com.fasterxml.jackson.databind.PropertyNamingStrategies;
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import com.fasterxml.jackson.databind.annotation.JsonNaming;
+import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder;
+import java.util.Map;
+import lombok.Builder;
+import lombok.EqualsAndHashCode;
+import lombok.Getter;
+import lombok.ToString;
+
+/** Http request data model for http step runtime. */
+@JsonDeserialize(builder = HttpStepRequest.HttpStepRequestBuilder.class)
+@JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class)
+@JsonInclude(JsonInclude.Include.NON_NULL)
+@JsonPropertyOrder(
+    value = {"url", "method", "headers", "body"},
+    alphabetic = true)
+@Builder(toBuilder = true)
+@Getter
+@ToString
+@EqualsAndHashCode
+public class HttpStepRequest {
+  private final String url;
+  private final String method;
+  private final Map<String, String> headers;
+  private final String body;
+
+  /** builder class for lombok and jackson. */
+  @JsonPOJOBuilder(withPrefix = "")
+  @JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class)
+  public static final class HttpStepRequestBuilder {}
+}

maestro-engine/src/main/java/com/netflix/maestro/engine/params/OutputDataManager.java

@@ -98,6 +98,8 @@ private Optional<String> extractExternalJobId(StepRuntimeSummary runtimeSummary)
       jobId = artifacts.get(Artifact.Type.KUBERNETES.key()).asKubernetes().getJobId();
     } else if (artifacts.containsKey(Artifact.Type.TITUS.key())) {
       jobId = artifacts.get(Artifact.Type.TITUS.key()).asTitus().getTitusTaskId();
+    } else if (artifacts.containsKey(Artifact.Type.HTTP.key())) {
+      jobId = runtimeSummary.getStepInstanceUuid();
     }
     if (jobId != null && !jobId.isEmpty()) {
       return Optional.of(jobId);

maestro-engine/src/main/java/com/netflix/maestro/engine/transformation/StepTranslator.java

@@ -38,6 +38,7 @@ public TaskDef translate(Step step) {
       case TITUS:
       case NOTEBOOK:
       case KUBERNETES:
+      case HTTP:
         return createMaestroTask((AbstractStep) step);
       default:
         throw new UnsupportedOperationException(

maestro-http/README.md

@@ -0,0 +1,47 @@
+# Maestro HTTP Module
+
+The Maestro Http module includes maestro http/https step runtime support to launch a job making an HTTP/HTTPS call.
+
+## Overview
+
+This module enables workflows to make HTTP/HTTPS calls as step executions and save the response in the http artifact and output params.
+
+## Architecture
+
+- **HttpRuntimeExecutor**: Interface for executing HTTP requests
+- **JdkHttpRuntimeExecutor**: Default implementation using JDK 11+ HttpClient
+- **HttpStepRuntime**: Step runtime that implements HTTP request execution and state management
+
+## Step Parameters
+
+HTTP steps accept the following parameters (see `default-http-step-params.yaml`):
+
+- **http** (required): Map containing request configuration
+  - **url** (required): URL for the HTTP request
+  - **method** (provided): HTTP method (default: GET)
+  - **headers** (provided): String Map of HTTP headers
+  - **body** (optional): Request body in String format
+- **state_expr** (provided): SEL expression to determine step state based on response
+
+## Output Parameters
+
+HTTP steps produce the following output parameters:
+
+- **status_code**: HTTP response status code (e.g., 200, 404, 500)
+- **response_body**: Response body as String format
+
+## Usage Example
+
+See `maestro-server/src/test/resources/samples/yaml/sample-http-wf.yaml` for a complete example.
+
+## Security Considerations
+
+**WARNING**: This module is currently in development and has known security limitations (e.g. SSRF).
+**Do not use in production environments accessible to untrusted users** until these issues are addressed.
+
+## Pending Features
+
+- Add URL validation for SSRF vulnerability
+- Add response size limits
+- Add response content-type handling
+- Add metrics

maestro-http/build.gradle

@@ -0,0 +1,10 @@
+dependencies {
+    implementation project(':maestro-common')
+    implementation project(':maestro-engine')
+    implementation jacksonDatabindDep
+    api slf4jApiDep
+
+    testImplementation junitDep
+    testImplementation mockitoCoreDep
+    testImplementation(testFixtures(project(':maestro-common')))
+}

maestro-http/gradle.lockfile

@@ -0,0 +1,10 @@
+# This is a Gradle generated file for dependency locking.
+# Manual edits can break the build and are not advised.
+# This file is expected to be part of source control.
+com.fasterxml.jackson.core:jackson-annotations:2.20=compileClasspath
+com.fasterxml.jackson.core:jackson-core:2.20.1=compileClasspath
+com.fasterxml.jackson.core:jackson-databind:2.20.1=compileClasspath
+com.fasterxml.jackson:jackson-bom:2.20.1=compileClasspath
+org.projectlombok:lombok:1.18.42=annotationProcessor,compileClasspath
+org.slf4j:slf4j-api:1.7.36=compileClasspath
+empty=

maestro-http/src/main/java/com/netflix/maestro/engine/http/HttpRuntimeExecutor.java

@@ -0,0 +1,32 @@
+/*
+ * Copyright 2025 Netflix, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package com.netflix.maestro.engine.http;
+
+import com.netflix.maestro.models.stepruntime.HttpStepRequest;
+import java.io.IOException;
+import java.net.http.HttpResponse;
+
+/** Interface for HTTP runtime executor to execute HTTP requests. */
+@FunctionalInterface
+public interface HttpRuntimeExecutor {
+  /**
+   * Execute an HTTP request.
+   *
+   * @param request the HTTP request to execute
+   * @return the HTTP response
+   * @throws IOException – if an I/O error occurs when sending or receiving, or the client has
+   *     closing shut down
+   * @throws InterruptedException – if the operation is interrupted
+   */
+  HttpResponse<String> execute(HttpStepRequest request) throws IOException, InterruptedException;
+}