[wip] one master branch for dropwizard-logstash-encoder project

.gitignore

@@ -1,4 +1,4 @@
-/target
+target/
 /classes
 /checkouts
 pom.xml.asc

README.md

@@ -14,9 +14,9 @@ Maven:
 
 Use version:
 
-`1.0.2` for Dropwizard `<1.0`
+`1.1.0` for Dropwizard `<1.0`
 
-`2.0.0` for Dropwizard `1.0`
+`2.0.0` for Dropwizard `>=1.0`
 
 ## Usage
 You must configure dropwizard to use these appenders in your application.yml file:

dropwizard-logstash-encoder-0.x/pom.xml

@@ -0,0 +1,41 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <packaging>jar</packaging>
+
+    <groupId>com.wikia</groupId>
+    <artifactId>dropwizard-logstash-encoder</artifactId>
+    <version>1.1.1-SNAPSHOT</version>
+    <name>dropwizard-logstash-encoder</name>
+    <description>
+        Addon for dropwizard to log using the logback-logstash-encoder (see
+        https://github.com/logstash/logstash-logback-encoder)
+    </description>
+    <url>https://github.com/Wikia/dropwizard-logback-logsash-encoder</url>
+
+    <parent>
+        <groupId>com.wikia</groupId>
+        <artifactId>dropwizard-logstash-encoder-parent</artifactId>
+        <version>1.0.0</version>
+    </parent>
+
+    <dependencies>
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-annotations</artifactId>
+            <version>2.4.0</version>
+        </dependency>
+        <dependency>
+            <groupId>io.dropwizard</groupId>
+            <artifactId>dropwizard-logging</artifactId>
+            <version>[0.7.0, 0.9.3]</version>
+        </dependency>
+        <dependency>
+            <groupId>net.logstash.logback</groupId>
+            <artifactId>logstash-logback-encoder</artifactId>
+            <version>4.3</version>
+        </dependency>
+    </dependencies>
+</project>

dropwizard-logstash-encoder-0.x/src/main/java/com/wikia/dropwizard/logstash/appender/AbstractLogstashAppenderFactory.java

@@ -0,0 +1,108 @@
+package com.wikia.dropwizard.logstash.appender;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import io.dropwizard.logging.AbstractAppenderFactory;
+import net.logstash.logback.stacktrace.ShortenedThrowableConverter;
+
+import javax.validation.constraints.Max;
+import javax.validation.constraints.Min;
+import javax.validation.constraints.NotNull;
+import java.util.HashMap;
+
+abstract class AbstractLogstashAppenderFactory extends AbstractAppenderFactory {
+  @NotNull
+  protected String host;
+
+  @Min(1)
+  @Max(65535)
+  protected int port;
+
+  protected boolean includeCallerInfo = false;
+
+  protected boolean includeContext = true;
+
+  protected boolean includeMdc = true;
+
+  protected boolean includeFullStackTrace = true;
+
+  protected HashMap<String, String> customFields;
+
+  protected HashMap<String, String> fieldNames;
+
+  @JsonProperty
+  public void setHost(String host) {
+    this.host = host;
+  }
+
+  @JsonProperty
+  public String getHost() {
+    return host;
+  }
+
+  @JsonProperty
+  public void setPort(int port) {
+    this.port = port;
+  }
+
+  @JsonProperty
+  public int getPort() {
+    return port;
+  }
+
+  @JsonProperty
+  public boolean getIncludeCallerInfo() {
+    return includeCallerInfo;
+  }
+
+  @JsonProperty
+  public void setIncludeCallerInfo(boolean includeCallerInfo) {
+    this.includeCallerInfo = includeCallerInfo;
+  }
+
+  @JsonProperty
+  public boolean getIncludeContext() {
+    return includeContext;
+  }
+
+  @JsonProperty
+  public void setIncludeContext(boolean includeContext) {
+    this.includeContext = includeContext;
+  }
+
+  @JsonProperty
+  public boolean getIncludeMdc() {
+    return includeMdc;
+  }
+
+  @JsonProperty
+  public void setIncludeMdc(boolean includeMdc) {
+    this.includeMdc = includeMdc;
+  }
+
+  @JsonProperty
+  public void setIncludeFullStackTrace(boolean includeFullStackTrace) {
+    this.includeFullStackTrace = includeFullStackTrace;
+  }
+
+  @JsonProperty
+  public HashMap<String, String> getCustomFields() {
+    return customFields;
+  }
+
+  @JsonProperty
+  public void setCustomFields(HashMap<String, String> customFields) {
+    this.customFields = customFields;
+  }
+
+  @JsonProperty
+  public HashMap<String, String> getFieldNames() {
+    return fieldNames;
+  }
+
+  @JsonProperty
+  public void setFieldNames(HashMap<String, String> fieldNames) {
+    this.fieldNames = fieldNames;
+  }
+
+
+}

dropwizard-logstash-encoder-0.x/src/main/java/com/wikia/dropwizard/logstash/appender/LogstashAppenderFactoryHelper.java

@@ -0,0 +1,47 @@
+package com.wikia.dropwizard.logstash.appender;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import net.logstash.logback.fieldnames.LogstashFieldNames;
+import net.logstash.logback.stacktrace.ShortenedThrowableConverter;
+
+import java.io.IOException;
+import java.io.StringWriter;
+import java.util.HashMap;
+
+public class LogstashAppenderFactoryHelper {
+  public static LogstashFieldNames getFieldNamesFromHashMap(HashMap<String, String> map) {
+    LogstashFieldNames fieldNames = new LogstashFieldNames();
+
+    fieldNames.setTimestamp(map.getOrDefault("timestamp", "@timestamp"));
+    fieldNames.setVersion(map.getOrDefault("version", "@version"));
+    fieldNames.setMessage(map.getOrDefault("message", "message"));
+    fieldNames.setLogger(map.getOrDefault("logger", "logger_name"));
+    fieldNames.setThread(map.getOrDefault("thread", "thread_name"));
+    fieldNames.setLevel(map.getOrDefault("level", "level"));
+    fieldNames.setLevelValue(map.getOrDefault("levelValue", "level_value"));
+    fieldNames.setCaller(map.getOrDefault("caller", null));
+    fieldNames.setCallerClass(map.getOrDefault("callerClass", "caller_class_name"));
+    fieldNames.setCallerMethod(map.getOrDefault("callerMethod", "caller_method_name"));
+    fieldNames.setCallerFile(map.getOrDefault("callerFile", "caller_file_name"));
+    fieldNames.setCallerLine(map.getOrDefault("callerLine", "caller_line_number"));
+    fieldNames.setStackTrace(map.getOrDefault("stackTrace", "stack_trace"));
+    fieldNames.setTags(map.getOrDefault("tags", "tags"));
+    fieldNames.setMdc(map.getOrDefault("mdc", null));
+    fieldNames.setContext(map.getOrDefault("context", null));
+
+    return fieldNames;
+  }
+
+  public static String getCustomFieldsFromHashMap(HashMap<String, String> map) throws IOException {
+    StringWriter writer = new StringWriter();
+    ObjectMapper mapper = new ObjectMapper();
+    mapper.writeValue(writer, map);
+    return writer.toString();
+  }
+
+  public static ShortenedThrowableConverter shortStackTraceConverter() {
+    ShortenedThrowableConverter throwableConverter = new ShortenedThrowableConverter();
+    throwableConverter.setMaxDepthPerThrowable(1);
+    return throwableConverter;
+  }
+}

dropwizard-logstash-encoder-0.x/src/main/java/com/wikia/dropwizard/logstash/appender/LogstashSocketAppenderFactory.java

@@ -0,0 +1,57 @@
+package com.wikia.dropwizard.logstash.appender;
+
+import static com.wikia.dropwizard.logstash.appender.LogstashAppenderFactoryHelper.shortStackTraceConverter;
+
+import ch.qos.logback.classic.LoggerContext;
+import ch.qos.logback.classic.spi.ILoggingEvent;
+import ch.qos.logback.core.Appender;
+import ch.qos.logback.core.Layout;
+import ch.qos.logback.core.net.SyslogConstants;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import net.logstash.logback.appender.LogstashSocketAppender;
+
+import java.io.IOException;
+
+@JsonTypeName("logstash-socket")
+public class LogstashSocketAppenderFactory extends AbstractLogstashAppenderFactory {
+
+  public LogstashSocketAppenderFactory() {
+    port = SyslogConstants.SYSLOG_PORT;
+  }
+
+  @Override
+  public Appender<ILoggingEvent> build(LoggerContext context, String applicationName, Layout<ILoggingEvent> layout) {
+    final LogstashSocketAppender appender = new LogstashSocketAppender();
+
+    appender.setName("logstash-socket-appender");
+    appender.setContext(context);
+    appender.setHost(host);
+    appender.setPort(port);
+
+    appender.setIncludeCallerInfo(includeCallerInfo);
+    appender.setIncludeMdc(includeMdc);
+    appender.setIncludeContext(includeContext);
+
+    if (!includeFullStackTrace) {
+      appender.setThrowableConverter(shortStackTraceConverter());
+    }
+
+    if (customFields != null) {
+      try {
+        String custom = LogstashAppenderFactoryHelper.getCustomFieldsFromHashMap(customFields);
+        appender.setCustomFields(custom);
+      } catch (IOException e) {
+        System.out.println("unable to parse customFields: "+e.getMessage());
+      }
+    }
+
+    if (fieldNames != null) {
+      appender.setFieldNames(LogstashAppenderFactoryHelper.getFieldNamesFromHashMap(fieldNames));
+    }
+
+    addThresholdFilter(appender, threshold);
+    appender.start();
+
+    return wrapAsync(appender);
+  }
+}

dropwizard-logstash-encoder-0.x/src/main/java/com/wikia/dropwizard/logstash/appender/LogstashTcpAppenderFactory.java

@@ -0,0 +1,89 @@
+package com.wikia.dropwizard.logstash.appender;
+
+import static com.wikia.dropwizard.logstash.appender.LogstashAppenderFactoryHelper.shortStackTraceConverter;
+
+import ch.qos.logback.classic.LoggerContext;
+import ch.qos.logback.classic.spi.ILoggingEvent;
+import ch.qos.logback.core.Appender;
+import ch.qos.logback.core.Layout;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import net.logstash.logback.appender.LogstashTcpSocketAppender;
+import net.logstash.logback.encoder.LogstashEncoder;
+
+import javax.validation.constraints.Max;
+import javax.validation.constraints.Min;
+import java.io.IOException;
+
+@JsonTypeName("logstash-tcp")
+public class LogstashTcpAppenderFactory extends AbstractLogstashAppenderFactory {
+  private boolean includeCallerData = false;
+
+  @Min(1)
+  @Max(65535)
+  private int queueSize = LogstashTcpSocketAppender.DEFAULT_QUEUE_SIZE;
+
+  public LogstashTcpAppenderFactory() {
+    this.port = LogstashTcpSocketAppender.DEFAULT_PORT;
+  }
+
+  @JsonProperty
+  public void setIncludeCallerData(boolean includeCallerData) {
+    this.includeCallerData = includeCallerData;
+  }
+
+  @JsonProperty
+  public boolean getIncludeCallerData() {
+    return includeCallerData;
+  }
+
+  @JsonProperty
+  public int getQueueSize() {
+    return queueSize;
+  }
+
+  @JsonProperty
+  public void setQueueSize(int queueSize) {
+    this.queueSize = queueSize;
+  }
+
+  public Appender<ILoggingEvent> build(LoggerContext context, String applicationName, Layout<ILoggingEvent> layout) {
+    final LogstashTcpSocketAppender appender = new LogstashTcpSocketAppender();
+    final LogstashEncoder encoder = new LogstashEncoder();
+
+    appender.setName("logstash-tcp-appender");
+    appender.setContext(context);
+    appender.setRemoteHost(host);
+    appender.setPort(port);
+    appender.setIncludeCallerData(includeCallerData);
+    appender.setQueueSize(queueSize);
+
+    encoder.setIncludeContext(includeContext);
+    encoder.setIncludeMdc(includeMdc);
+    encoder.setIncludeCallerInfo(includeCallerInfo);
+
+    if (!includeFullStackTrace) {
+      encoder.setThrowableConverter(shortStackTraceConverter());
+    }
+
+    if (customFields != null) {
+      try {
+        String custom = LogstashAppenderFactoryHelper.getCustomFieldsFromHashMap(customFields);
+        encoder.setCustomFields(custom);
+      } catch (IOException e) {
+        System.out.println("unable to parse customFields: "+e.getMessage());
+      }
+    }
+
+    if (fieldNames != null) {
+      encoder.setFieldNames(LogstashAppenderFactoryHelper.getFieldNamesFromHashMap(fieldNames));
+    }
+
+    appender.setEncoder(encoder);
+    addThresholdFilter(appender, threshold);
+    encoder.start();
+    appender.start();
+
+    return wrapAsync(appender);
+  }
+}