Feat: check target is set in ECS mode (#49)

CHANGELOG.md

@@ -1,3 +1,10 @@
+## 3.2.1
+  - Fix tag on failure test [#52](https://github.com/logstash-plugins/logstash-filter-json/pull/52)
+
+## 3.2.0
+ - Feat: check target is set in ECS mode [#49](https://github.com/logstash-plugins/logstash-filter-json/pull/49)
+ - Refactor: logging improvements to print event details in debug mode
+
 ## 3.1.0
  - Added better error handling, preventing some classes of malformed inputs from crashing the pipeline.
 

docs/index.asciidoc

@@ -36,6 +36,13 @@ If the parsed data contains a `@timestamp` field, the plugin will try to use it
 parsing fails, the field will be renamed to `_@timestamp` and the event will be tagged with a
 `_timestampparsefailure`.
 
+[id="plugins-{type}s-{plugin}-ecs_metadata"]
+==== Event Metadata and the Elastic Common Schema (ECS)
+
+The plugin behaves the same regardless of ECS compatibility, except giving a warning when ECS is enabled and `target` isn't set.
+
+TIP: Set the `target` option to avoid potential schema conflicts.
+
 [id="plugins-{type}s-{plugin}-options"]
 ==== JSON Filter Configuration Options
 
@@ -44,6 +51,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 [cols="<,<,<",options="header",]
 |=======================================================================
 |Setting |Input type|Required
+| <<plugins-{type}s-{plugin}-ecs_compatibility>> | <<string,string>>|No
 | <<plugins-{type}s-{plugin}-skip_on_invalid_json>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-source>> |<<string,string>>|Yes
 | <<plugins-{type}s-{plugin}-tag_on_failure>> |<<array,array>>|No
@@ -55,6 +63,18 @@ filter plugins.
 
 &nbsp;
 
+[id="plugins-{type}s-{plugin}-ecs_compatibility"]
+===== `ecs_compatibility`
+
+* Value type is <<string,string>>
+* Supported values are:
+** `disabled`: does not use ECS-compatible field names
+** `v1`: Elastic Common Schema compliant behavior (warns when `target` isn't set)
+
+Controls this plugin's compatibility with the
+{ecs-ref}[Elastic Common Schema (ECS)].
+See <<plugins-{type}s-{plugin}-ecs_metadata>> for detailed information.
+
 [id="plugins-{type}s-{plugin}-skip_on_invalid_json"]
 ===== `skip_on_invalid_json` 
 

lib/logstash/filters/json.rb

@@ -3,6 +3,9 @@
 require "logstash/namespace"
 require "logstash/json"
 require "logstash/timestamp"
+require 'logstash/plugin_mixins/ecs_compatibility_support'
+require 'logstash/plugin_mixins/ecs_compatibility_support/target_check'
+require 'logstash/plugin_mixins/validator_support/field_reference_validation_adapter'
 
 # This is a JSON parsing filter. It takes an existing field which contains JSON and
 # expands it into an actual data structure within the Logstash event.
@@ -21,6 +24,11 @@
 # `_timestampparsefailure`.
 class LogStash::Filters::Json < LogStash::Filters::Base
 
+  include LogStash::PluginMixins::ECSCompatibilitySupport
+  include LogStash::PluginMixins::ECSCompatibilitySupport::TargetCheck
+
+  extend LogStash::PluginMixins::ValidatorSupport::FieldReferenceValidationAdapter
+
   config_name "json"
 
   # The configuration for the JSON filter:
@@ -53,7 +61,7 @@ class LogStash::Filters::Json < LogStash::Filters::Base
   # data structure in the `target` field.
   #
   # NOTE: if the `target` field already exists, it will be overwritten!
-  config :target, :validate => :string
+  config :target, :validate => :field_reference
 
   # Append values to the `tags` field when there has been no
   # successful match
@@ -67,7 +75,7 @@ def register
   end
 
   def filter(event)
-    @logger.debug? && @logger.debug("Running json filter", :event => event)
+    @logger.debug? && @logger.debug("Running json filter", :event => event.to_hash)
 
     source = event.get(@source)
     return unless source
@@ -99,6 +107,7 @@ def filter(event)
       begin
         timestamp = parsed_timestamp ? LogStash::Timestamp.coerce(parsed_timestamp) : nil
       rescue LogStash::TimestampParserError => e
+        @logger.debug("Failed to coerce timestamp", :timestamp => parsed_timestamp, :message => e.message)
         timestamp = nil
       end
 
@@ -120,11 +129,11 @@ def filter(event)
 
     filter_matched(event)
 
-    @logger.debug? && @logger.debug("Event after json filter", :event => event)
+    @logger.debug? && @logger.debug("Event after json filter", :event => event.to_hash)
   rescue => ex
-    meta = { :exception => ex.message, :source => @source, :raw => source}
-    meta[:backtrace] = ex.backtrace if logger.debug?
-    logger.warn('Exception caught in json filter', meta)
+    meta = { :source => @source, :raw => source, :exception => ex }
+    meta[:backtrace] = ex.backtrace if @logger.debug?
+    @logger.warn('Exception caught in json filter', meta)
     _do_tag_on_failure(event)
   end
 

logstash-filter-json.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-json'
-  s.version         = '3.1.0'
+  s.version         = '3.2.1'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Parses JSON events"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -21,6 +21,8 @@ Gem::Specification.new do |s|
 
   # Gem dependencies
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
+  s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~> 1.3'
+  s.add_runtime_dependency 'logstash-mixin-validator_support', '~> 1.0'
 
   s.add_development_dependency 'logstash-devutils'
   s.add_development_dependency 'insist'

spec/filters/json_spec.rb

@@ -62,8 +62,9 @@
   end
 
   logstash_version = Gem::Version.create(LOGSTASH_CORE_VERSION)
-
-  if (Gem::Requirement.create('>= 7.0').satisfied_by?(logstash_version) ||
+
+  # Field Reference can handle special characters since 8.3.0
+  if ((Gem::Requirement.create('>= 7.0').satisfied_by?(logstash_version) && Gem::Requirement.create('< 8.3').satisfied_by?(logstash_version)) ||
      (Gem::Requirement.create('~> 6.4').satisfied_by?(logstash_version) && LogStash::SETTINGS.get('config.field_reference.parser') == 'STRICT'))
     describe "unsupported field name using `target`" do
       config <<-CONFIG
@@ -155,12 +156,14 @@
     end
   end
 
-  context "using message field source" do
+  subject(:filter) {  LogStash::Filters::Json.new(config)  }
 
-    subject(:filter) {  LogStash::Filters::Json.new(config)  }
+  let(:config) { {"source" => "message"} }
+  let(:event) { LogStash::Event.new("message" => message) }
 
-    let(:config) { {"source" => "message"} }
-    let(:event) { LogStash::Event.new("message" => message) }
+  let(:logger) { filter.logger }
+
+  context "using message field source" do
 
     before(:each) do
       filter.register
@@ -224,6 +227,37 @@
     end
   end
 
+  describe "target" do
+
+    let(:logger) { filter.logger }
+
+    context 'not set in ECS mode' do
+      let(:config) { { "source" => "message", 'ecs_compatibility' => 'v1' } }
+      let(:message) { ' { "foo": "bar" } ' }
+
+      it "works" do
+        filter.register # a warning logged
+
+        filter.filter(event)
+        expect( event.get('foo') ).to eql 'bar'
+      end
+    end
+
+    context 'set in ECS mode' do
+      let(:config) { { "source" => "message", 'ecs_compatibility' => 'v1', 'target' => '[baz]' } }
+      let(:message) { ' { "foo": "bar" } ' }
+
+      it "works" do
+        filter.register # no warning logged
+
+        filter.filter(event)
+        expect( event.include?('foo') ).to be false
+        expect( event.get('baz') ).to eql 'foo' => 'bar'
+      end
+    end
+
+  end
+
   describe "parse mixture of json an non-json content (skip_on_invalid_json)" do
     subject(:filter) {  LogStash::Filters::Json.new(config)  }