Harden telemetry advice fail-open behavior

README.md

@@ -11,54 +11,39 @@ This artifact tested with the following version:
 
 ## Compile/Build
 
-In `build.gradle` file, update dependencies location:
-
-- Set the variable `apim_folder` to you API-Gateway installation folder (e.g. `opt/Axway/APIM/apigateway`)
+Set `apim_folder` to the API Gateway `system` folder so Gradle can compile against the APIM runtime jars:
 
 
 ```
-gradlew clean jar
+./gradlew clean jar -Papim_folder=/opt/Axway/apigateway/system
 ```
 
-## Setup
+Alternatively, set `APIM_FOLDER`:
 
-Copy following jar files
+```
+APIM_FOLDER=/opt/Axway/apigateway/system ./gradlew clean jar
+```
 
+To assemble the agent jar and required runtime dependency jars into one install directory, run:
 
-- Copy opentelemetry-apim-agent--x.x.x.jar file to apigateway/ext/lib
-- Copy Aspectj weaver - [ aspectjweaver-1.9.6.jar ](https://repo1.maven.org/maven2/org/aspectj/aspectjweaver/1.9.22.1/aspectjweaver-1.9.22.1.jar) to  apigateway/ext/lib
-- Copy opentelmetry sdk jar files to apigateway/ext/lib 
+```
+./gradlew clean assembleInstall -Papim_folder=/opt/Axway/apigateway/system
+```
 
-  - [okhttp-4.12.0.jar](https://repo1.maven.org/maven2/com/squareup/okhttp3/okhttp/4.12.0/okhttp-4.12.0.jar)
-  - [okio-jvm-3.6.0.jar](https://repo1.maven.org/maven2/com/squareup/okio/okio-jvm/3.6.0/okio-jvm-3.6.0.jar)
-  - [kotlin-stdlib-1.9.25.jar](https://repo1.maven.org/maven2/org/jetbrains/kotlin/kotlin-stdlib/1.9.25/kotlin-stdlib-1.9.25.jar)
+The install directory is created at `build/install/opentelemetry-apim-agent`.
 
-  - [opentelemetry-semconv-1.27.0-alpha.jar](https://repo1.maven.org/maven2/io/opentelemetry/semconv/opentelemetry-semconv/1.27.0-alpha/opentelemetry-semconv-1.27.0-alpha.jar)
-
-  - [opentelemetry-api-1.42.1.jar](https://repo1.maven.org/maven2/io/opentelemetry/opentelemetry-api/1.42.1/opentelemetry-api-1.42.1.jar)
-  - [opentelemetry-api-incubator-1.42.1-alpha.jar](https://repo1.maven.org/maven2/io/opentelemetry/opentelemetry-api-incubator/1.42.1-alpha/opentelemetry-api-incubator-1.42.1-alpha.jar)
-
-  - [opentelemetry-context-1.42.1.jar](https://repo1.maven.org/maven2/io/opentelemetry/opentelemetry-context/1.42.1/opentelemetry-context-1.42.1.jar)
+The assembled directory is the source of truth for the runtime jar set. To review the exact files Gradle resolved for this build, run:
 
-  - [opentelemetry-exporter-common-1.42.1.jar](https://repo1.maven.org/maven2/io/opentelemetry/opentelemetry-exporter-common/1.42.1/opentelemetry-exporter-common-1.42.1.jar)
-  - [opentelemetry-exporter-logging-1.42.1.jar](https://repo1.maven.org/maven2/io/opentelemetry/opentelemetry-exporter-logging/1.42.1/opentelemetry-exporter-logging-1.42.1.jar)
-  - [opentelemetry-exporter-otlp-1.42.1.jar](https://repo1.maven.org/maven2/io/opentelemetry/opentelemetry-exporter-otlp/1.42.1/opentelemetry-exporter-otlp-1.42.1.jar)
-  - [opentelemetry-exporter-otlp-common-1.42.1.jar](https://repo1.maven.org/maven2/io/opentelemetry/opentelemetry-exporter-otlp-common/1.42.1/opentelemetry-exporter-otlp-common-1.42.1.jar)
-  - [opentelemetry-exporter-sender-okhttp-1.42.1.jar](https://repo1.maven.org/maven2/io/opentelemetry/opentelemetry-exporter-sender-okhttp/1.42.1/opentelemetry-exporter-sender-okhttp-1.42.1.jar)
+```
+ls -1 build/install/opentelemetry-apim-agent/*.jar
+```
 
-  - [opentelemetry-sdk-1.42.1.jar](https://repo1.maven.org/maven2/io/opentelemetry/opentelemetry-sdk/1.42.1/opentelemetry-sdk-1.42.1.jar)
-  - [opentelemetry-sdk-common-1.42.1.jar](https://repo1.maven.org/maven2/io/opentelemetry/opentelemetry-sdk-common/1.42.1/opentelemetry-sdk-common-1.42.1.jar)
-  - [opentelemetry-sdk-extension-autoconfigure-1.42.1.jar](https://repo1.maven.org/maven2/io/opentelemetry/opentelemetry-sdk-extension-autoconfigure/1.42.1/opentelemetry-sdk-extension-autoconfigure-1.42.1.jar)
-  - [opentelemetry-sdk-extension-autoconfigure-spi-1.42.1.jar](https://repo1.maven.org/maven2/io/opentelemetry/opentelemetry-sdk-extension-autoconfigure-spi/1.42.1/opentelemetry-sdk-extension-autoconfigure-spi-1.42.1.jar)
-
-
-  - [opentelemetry-extension-incubator-1.30.1-alpha.jar](https://repo1.maven.org/maven2/io/opentelemetry/opentelemetry-extension-incubator/1.30.1-alpha/opentelemetry-extension-incubator-1.30.1-alpha.jar)
-  - [opentelemetry-sdk-logs-1.42.1.jar](https://repo1.maven.org/maven2/io/opentelemetry/opentelemetry-sdk-logs/1.42.1/opentelemetry-sdk-logs-1.42.1.jar)
-  - [opentelemetry-sdk-metrics-1.42.1.jar](https://repo1.maven.org/maven2/io/opentelemetry/opentelemetry-sdk-metrics/1.42.1/opentelemetry-sdk-metrics-1.42.1.jar)
-  - [opentelemetry-sdk-trace-1.42.1.jar](https://repo1.maven.org/maven2/io/opentelemetry/opentelemetry-sdk-trace/1.42.1/opentelemetry-sdk-trace-1.42.1.jar)
-  - [opentelemetry-runtime-telemetry-java8-2.18.1-alpha.jar](https://repo1.maven.org/maven2/io/opentelemetry/instrumentation/opentelemetry-runtime-telemetry-java8/2.18.1-alpha/opentelemetry-runtime-telemetry-java8-2.18.1-alpha.jar)
+## Setup
 
+Copy the assembled jar files:
 
+- Copy all jar files from `build/install/opentelemetry-apim-agent` to `apigateway/ext/lib`.
+- Remove any previously copied OpenTelemetry/APIM agent jars from `apigateway/ext/lib` before copying the newly assembled jars. Do not leave older OpenTelemetry versions beside the assembled set.
 
 - Create a file named jvm.xml under APIGATEWAY_INSTALL_DIR/apigateway/conf/
     ```xml

build.gradle

@@ -20,24 +20,46 @@ tasks.withType(JavaCompile) {
     options.compilerArgs += ['-Xlint:deprecation']
 }
 
-def apim_folder = '/Users/rnatarajan/v7apim/may2023/system'
+def apim_folder = providers.gradleProperty('apim_folder')
+    .orElse(providers.environmentVariable('APIM_FOLDER'))
+    .getOrElse('')
 
-def opentelemetry_version = "1.42.1"
+if (!apim_folder) {
+    throw new GradleException('Set the APIM system folder with -Papim_folder=/path/to/apigateway/system or APIM_FOLDER=/path/to/apigateway/system')
+}
+
+def apimSystemDir = file(apim_folder)
+if (!apimSystemDir.isDirectory()) {
+    throw new GradleException("APIM system folder does not exist: ${apimSystemDir}")
+}
+
+configurations {
+    apimAgentRuntime {
+        canBeConsumed = false
+        canBeResolved = true
+    }
+    implementation.extendsFrom(apimAgentRuntime)
+}
+
+def opentelemetry_version = "1.52.0"
+def opentelemetry_instrumentation_version = "2.18.1"
+def opentelemetry_semconv_version = "1.34.0"
 dependencies {
-    implementation fileTree(dir: "${apim_folder}/lib", include: '*.jar')
-    implementation fileTree(dir: "${apim_folder}/lib/jce", include: '*.jar')
-    implementation fileTree(dir: "${apim_folder}/lib/plugins", include: '*.jar')
-    implementation group: 'org.aspectj', name: 'aspectjweaver', version: '1.9.22.1'
-    implementation("io.opentelemetry:opentelemetry-api:${opentelemetry_version}")
-    implementation("io.opentelemetry:opentelemetry-sdk:${opentelemetry_version}")
-    implementation("io.opentelemetry:opentelemetry-exporter-otlp:${opentelemetry_version}")
-
-    implementation("io.opentelemetry:opentelemetry-sdk-metrics:${opentelemetry_version}")
-    implementation("io.opentelemetry:opentelemetry-exporter-logging:${opentelemetry_version}")
-    implementation("io.opentelemetry:opentelemetry-sdk-extension-autoconfigure:${opentelemetry_version}")
+    implementation fileTree(dir: "${apimSystemDir}/lib", include: '*.jar')
+    implementation fileTree(dir: "${apimSystemDir}/lib/jce", include: '*.jar')
+    implementation fileTree(dir: "${apimSystemDir}/lib/plugins", include: '*.jar')
+    apimAgentRuntime group: 'org.aspectj', name: 'aspectjweaver', version: '1.9.22.1'
+    apimAgentRuntime("io.opentelemetry:opentelemetry-api:${opentelemetry_version}")
+    apimAgentRuntime("io.opentelemetry:opentelemetry-sdk:${opentelemetry_version}")
+    apimAgentRuntime("io.opentelemetry:opentelemetry-exporter-otlp:${opentelemetry_version}")
+
+    apimAgentRuntime("io.opentelemetry:opentelemetry-sdk-metrics:${opentelemetry_version}")
+    apimAgentRuntime("io.opentelemetry:opentelemetry-exporter-logging:${opentelemetry_version}")
+    apimAgentRuntime("io.opentelemetry:opentelemetry-sdk-extension-autoconfigure:${opentelemetry_version}")
    // implementation("io.opentelemetry:opentelemetry-semconv:1.30.1-alpha")
-    implementation "io.opentelemetry.semconv:opentelemetry-semconv:1.27.0-alpha"
-    implementation "io.opentelemetry.instrumentation:opentelemetry-runtime-telemetry-java8:2.18.1-alpha"
+    apimAgentRuntime "io.opentelemetry.semconv:opentelemetry-semconv:${opentelemetry_semconv_version}"
+    apimAgentRuntime "io.opentelemetry.instrumentation:opentelemetry-runtime-telemetry-java8:${opentelemetry_instrumentation_version}-alpha"
+    apimAgentRuntime "io.opentelemetry.instrumentation:opentelemetry-instrumentation-api:${opentelemetry_instrumentation_version}"
 
     testImplementation group: 'junit', name: 'junit', version: '4.13.2'
     testImplementation group: 'org.mockito', name: 'mockito-core', version: '3.12.4'
@@ -47,3 +69,12 @@ dependencies {
 jar {
     metaInf { from('src/main/resources/aop.xml') }
 }
+
+tasks.register('assembleInstall', Copy) {
+    group = 'distribution'
+    description = 'Assembles the APIM agent jar and required runtime dependency jars for apigateway/ext/lib.'
+    dependsOn jar
+    into layout.buildDirectory.dir('install/opentelemetry-apim-agent')
+    from(tasks.named('jar'))
+    from(configurations.apimAgentRuntime)
+}

src/main/java/com/axway/apim/aspects/AxwayAspect.java

@@ -2,6 +2,7 @@
 
 import com.axway.apim.opentelemetry.ConnectToUrl;
 import com.axway.apim.opentelemetry.HttpServer;
+import com.axway.apim.opentelemetry.Telemetry;
 import com.axway.apim.opentelemetry.Utils;
 import com.vordel.circuit.InvocationContext;
 import com.vordel.circuit.Message;
@@ -41,10 +42,17 @@ public void invokeGateway(Message m, MessageProcessor lastChanceHandler, Object
      */
     @Around("invokeGateway(m, lastChanceHandler, context)")
     public Object invokePointcutGateway(ProceedingJoinPoint pjp, Message m, MessageProcessor lastChanceHandler, Object context) throws Throwable {
-        String requestPath = (String) m.get("http.request.path");
-        String[] uriSplit = requestPath.split("/");
-        String apiName = uriSplit.length == 0 ? "/" : uriSplit[1];
-        String httpVerb = Utils.getHttpMethod(m);
+        String apiName;
+        String httpVerb;
+        try {
+            String requestPath = (String) m.get("http.request.path");
+            String[] uriSplit = requestPath.split("/");
+            apiName = uriSplit.length > 1 ? uriSplit[1] : "/";
+            httpVerb = Utils.getHttpMethod(m);
+        } catch (Throwable e) {
+            Telemetry.disable("gateway advice setup", e);
+            return pjp.proceed();
+        }
         return httpServer.aroundHttpServer(pjp, m, apiName, httpVerb);
     }
 
@@ -86,9 +94,15 @@ public Object invokeMethodAroundAdvice(ProceedingJoinPoint pjp, ServerTransactio
                                            MessageProcessor lastChanceHandler, InvokableMethod runMethod,
                                            final PathResolverResult resolvedMethod, final int matchCount,
                                            String httpMethod, ApiShunt currentApiCallStatus) throws Throwable {
-        String[] uriSplit = Utils.getRequestURL(m).split("/");
         String apiName;
-        apiName = (String) m.getOrDefault("api.name", uriSplit[1]);
+        try {
+            String[] uriSplit = Utils.getRequestURL(m).split("/");
+            String defaultApiName = uriSplit.length > 1 ? uriSplit[1] : "/";
+            apiName = (String) m.getOrDefault("api.name", defaultApiName);
+        } catch (Throwable e) {
+            Telemetry.disable("API Manager advice setup", e);
+            return pjp.proceed();
+        }
         return httpServer.aroundHttpServer(pjp, m, apiName, httpMethod);
     }
 
@@ -100,9 +114,22 @@ public void apiManagerFaultHandler(ApiShunt shuntReason, Message m, InvocationCo
 
     @Around("apiManagerFaultHandler(shuntReason, m, ctx)")
     public Object handleApiManagerFaultHandler(ProceedingJoinPoint pjp, ApiShunt shuntReason, Message m, InvocationContext ctx) throws Throwable {
-        // Only handle API not found case
-        if (shuntReason.getStatusCode() == 404) {
-            return httpServer.aroundHttpServer(pjp, m, "NotFound",  Utils.getHttpMethod(m));
+        boolean traceNotFound;
+        try {
+            traceNotFound = shuntReason.getStatusCode() == 404;
+        } catch (Throwable e) {
+            Telemetry.disable("API Manager fault advice setup", e);
+            return pjp.proceed();
+        }
+        if (traceNotFound) {
+            String httpMethod;
+            try {
+                httpMethod = Utils.getHttpMethod(m);
+            } catch (Throwable e) {
+                Telemetry.disable("API Manager fault advice setup", e);
+                return pjp.proceed();
+            }
+            return httpServer.aroundHttpServer(pjp, m, "NotFound", httpMethod);
         }
         return pjp.proceed();
     }

src/main/java/com/axway/apim/opentelemetry/ConnectToUrl.java

@@ -15,22 +15,54 @@
 
 public class ConnectToUrl {
 
-    private static final Tracer TRACER = Configuration.getInstance().getTracer("io.opentelemetry.axway.apim.http.HttpClient");
-    private static final TextMapPropagator TEXT_MAP_PROPAGATOR = Configuration.getInstance().getPropagators().getTextMapPropagator();
-
     public Object httpClient(ProceedingJoinPoint pjp, Message message, Circuit circuit, HeaderSet requestHeaders, String httpVerb) throws Throwable {
-        Object pjpReturnObject;
-        String requestUrl = Utils.getRequestURL(message);
-        Span span = TRACER.spanBuilder(requestUrl).setSpanKind(SpanKind.CLIENT).startSpan();
-        try (Scope ignored = span.makeCurrent()) {
+        Tracer tracer = Telemetry.getTracer("io.opentelemetry.axway.apim.http.HttpClient");
+        TextMapPropagator textMapPropagator = Telemetry.getTextMapPropagator();
+        if (tracer == null || textMapPropagator == null) {
+            return pjp.proceed();
+        }
+
+        Span span = null;
+        Scope scope = null;
+        try {
+            String requestUrl = Utils.getRequestURL(message);
+            span = tracer.spanBuilder(requestUrl).setSpanKind(SpanKind.CLIENT).startSpan();
+            scope = span.makeCurrent();
             span.setAttribute(HttpAttributes.HTTP_REQUEST_METHOD, httpVerb);
             span.setAttribute("component", "http");
             span.setAttribute("Routing policy", circuit.getName());
             String url = (String) message.get("destinationURL");
             Utils.addHttpDetails(span, url, requestUrl, message);
             Utils.addHttpHeaders(span, "request", (HeaderSet) message.get(Utils.HTTP_HEADERS));
-            TEXT_MAP_PROPAGATOR.inject(Context.current(), requestHeaders, Utils.setter);
+            textMapPropagator.inject(Context.current(), requestHeaders, Utils.setter);
+        } catch (Throwable e) {
+            closeScope(scope);
+            endSpan(span, message);
+            Telemetry.disable("HTTP client span setup", e);
+            return pjp.proceed();
+        }
+
+        Object pjpReturnObject;
+        Throwable pjpError = null;
+        try {
             pjpReturnObject = pjp.proceed();
+        } catch (Throwable e) {
+            pjpError = e;
+            recordException(span, message, e);
+            throw e;
+        } finally {
+            closeScope(scope);
+            if (pjpError != null) {
+                endSpan(span, message);
+            }
+        }
+        recordResponse(span, message);
+        endSpan(span, message);
+        return pjpReturnObject;
+    }
+
+    private void recordResponse(Span span, Message message) {
+        try {
             int httpStatus = (int) message.getOrDefault("http.response.status", 0);
             String httpStatusMessage = (String) message.getOrDefault("http.response.info", "");
             if (httpStatus > 400 && httpStatus < 500) {
@@ -40,15 +72,40 @@ public Object httpClient(ProceedingJoinPoint pjp, Message message, Circuit circu
                 span.setAttribute("error.type", httpStatusMessage);
             }
         } catch (Throwable e) {
+            Telemetry.recordFailure("HTTP client response recording", e);
+        }
+    }
+
+    private void recordException(Span span, Message message, Throwable original) {
+        try {
             int httpStatus = (int) message.getOrDefault("http.response.status", 0);
             String httpStatusMessage = (String) message.getOrDefault("http.response.info", "");
-            span.setStatus(StatusCode.ERROR, httpStatus + "-" +httpStatusMessage);
-            span.recordException(e);
-            throw e;
-        } finally {
+            span.setStatus(StatusCode.ERROR, httpStatus + "-" + httpStatusMessage);
+            span.recordException(original);
+        } catch (Throwable e) {
+            Telemetry.recordFailure("HTTP client exception recording", e);
+        }
+    }
+
+    private void endSpan(Span span, Message message) {
+        if (span == null) {
+            return;
+        }
+        try {
             Utils.addHttpHeaders(span, "response", (HeaderSet) message.get(Utils.HTTP_HEADERS));
             span.end();
+        } catch (Throwable e) {
+            Telemetry.recordFailure("HTTP client span close", e);
+        }
+    }
+
+    private void closeScope(Scope scope) {
+        if (scope != null) {
+            try {
+                scope.close();
+            } catch (Throwable e) {
+                Telemetry.recordFailure("HTTP client scope close", e);
+            }
         }
-        return pjpReturnObject;
     }
 }