Implement new JSON Jinja filters

requirements.txt

@@ -2,5 +2,6 @@
 # of appearance. Changing the order has an impact on the overall integration
 # process, which may cause wedges in the gate later.
 
+jsonpath-rw==1.4.0
 retrying<1.4,>=1.3
 semver==2.7.2

setup.cfg

@@ -10,7 +10,10 @@ mistral.actions =
     st2.action = st2mistral.actions.stackstorm:St2Action
 
 mistral.expression.functions =
+    from_json_string = st2mistral.functions.data:from_json_string
+    from_yaml_string = st2mistral.functions.data:from_yaml_string
     json_escape = st2mistral.functions.json_escape:json_escape
+    jsonpath_query = st2mistral.functions.jsonpath_query:jsonpath_query
     regex_match = st2mistral.functions.regex:regex_match
     regex_replace = st2mistral.functions.regex:regex_replace
     regex_search = st2mistral.functions.regex:regex_search

st2mistral/functions/data.py

@@ -17,12 +17,22 @@
 import yaml
 
 __all__ = [
+    'from_json_string',
+    'from_yaml_string',
     'to_complex',
     'to_json_string',
     'to_yaml_string'
 ]
 
 
+def from_json_string(context, value):
+    return json.loads(value)
+
+
+def from_yaml_string(context, value):
+    return yaml.safe_load(value)
+
+
 def to_complex(context, value):
     return json.dumps(value)
 

st2mistral/functions/jsonpath_query.py

@@ -0,0 +1,37 @@
+# Licensed to the StackStorm, Inc ('StackStorm') under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import jsonpath_rw
+
+__all__ = [
+    'jsonpath_query',
+]
+
+
+def jsonpath_query(context, value, query):
+    """Extracts data from an object `value` using a JSONPath `query`.
+
+    :link: https://github.com/kennknowles/python-jsonpath-rw
+    :param value: a object (dict, array, etc) to query
+    :param query: a jmsepath query expression (string)
+    :returns: the result of the query executed on the value
+    :rtype: dict, array, int, string, bool
+    """
+
+    expr = jsonpath_rw.parse(query)
+    matches = [match.value for match in expr.find(value)]
+    if not matches:
+        return None
+    return matches

st2mistral/tests/unit/functions/test_data.py

@@ -24,6 +24,22 @@
 
 class JinjaDataTestCase(base.JinjaFunctionTestCase):
 
+    def test_function_from_json_string(self):
+        obj = {'a': 'b', 'c': {'d': 'e', 'f': 1, 'g': True}}
+        obj_json_str = json.dumps(obj)
+        template = '{{ from_json_string(_.k1) }}'
+        result = self.eval_expression(template, {"k1": obj_json_str})
+        actual_obj = eval(result)
+        self.assertDictEqual(obj, actual_obj)
+
+    def test_function_from_yaml_string(self):
+        obj = {'a': 'b', 'c': {'d': 'e', 'f': 1, 'g': True}}
+        obj_yaml_str = yaml.safe_dump(obj)
+        template = '{{ from_yaml_string(_.k1) }}'
+        result = self.eval_expression(template, {"k1": obj_yaml_str})
+        actual_obj = eval(result)
+        self.assertDictEqual(obj, actual_obj)
+
     def test_function_to_complex(self):
         obj = {'a': 'b', 'c': {'d': 'e', 'f': 1, 'g': True}}
         template = '{{ to_complex(_.k1) }}'
@@ -48,6 +64,22 @@ def test_function_to_yaml_string(self):
 
 class YAQLDataTestCase(base.YaqlFunctionTestCase):
 
+    def test_function_from_json_string(self):
+        obj = {'a': 'b', 'c': {'d': 'e', 'f': 1, 'g': True}}
+        obj_json_str = json.dumps(obj)
+        result = YAQL_ENGINE('from_json_string($.k1)').evaluate(
+            context=self.get_yaql_context({'k1': obj_json_str})
+        )
+        self.assertDictEqual(obj, result)
+
+    def test_function_from_yaml_string(self):
+        obj = {'a': 'b', 'c': {'d': 'e', 'f': 1, 'g': True}}
+        obj_yaml_str = yaml.safe_dump(obj)
+        result = YAQL_ENGINE('from_yaml_string($.k1)').evaluate(
+            context=self.get_yaql_context({'k1': obj_yaml_str})
+        )
+        self.assertDictEqual(obj, result)
+
     def test_to_complex(self):
         obj = {'a': 'b', 'c': {'d': 'e', 'f': 1, 'g': True}}
         result = YAQL_ENGINE('to_complex($.k1)').evaluate(

st2mistral/tests/unit/functions/test_jsonpath_query.py

@@ -0,0 +1,110 @@
+# Licensed to the StackStorm, Inc ('StackStorm') under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import jsonpath_rw
+
+from st2mistral.tests.unit import test_function_base as base
+
+from yaql.language import factory
+YAQL_ENGINE = factory.YaqlFactory().create()
+
+
+class JinjaDataTestCase(base.JinjaFunctionTestCase):
+
+    def test_function_jsonpath_query_static(self):
+        obj = {'people': [{'first': 'James', 'last': 'd'},
+                          {'first': 'Jacob', 'last': 'e'},
+                          {'first': 'Jayden', 'last': 'f'},
+                          {'missing': 'different'}],
+               'foo': {'bar': 'baz'}}
+
+        template = '{{ jsonpath_query(_.obj, "people[*].first") }}'
+        result = self.eval_expression(template, {"obj": obj})
+        actual = eval(result)
+        expected = ['James', 'Jacob', 'Jayden']
+        self.assertEqual(actual, expected)
+
+    def test_function_jsonpath_query_dynamic(self):
+        obj = {'people': [{'first': 'James', 'last': 'd'},
+                          {'first': 'Jacob', 'last': 'e'},
+                          {'first': 'Jayden', 'last': 'f'},
+                          {'missing': 'different'}],
+               'foo': {'bar': 'baz'}}
+        query = "people[*].last"
+
+        template = '{{ jsonpath_query(_.obj, _.query) }}'
+        result = self.eval_expression(template, {"obj": obj,
+                                                 'query': query})
+        actual = eval(result)
+        expected = ['d', 'e', 'f']
+        self.assertEqual(actual, expected)
+
+    def test_function_jsonpath_query_no_results(self):
+        obj = {'people': [{'first': 'James', 'last': 'd'},
+                          {'first': 'Jacob', 'last': 'e'},
+                          {'first': 'Jayden', 'last': 'f'},
+                          {'missing': 'different'}],
+               'foo': {'bar': 'baz'}}
+        query = "query_returns_no_results"
+
+        template = '{{ jsonpath_query(_.obj, _.query) }}'
+        result = self.eval_expression(template, {"obj": obj,
+                                                 'query': query})
+        actual = eval(result)
+        expected = None
+        self.assertEqual(actual, expected)
+
+
+class YAQLDataTestCase(base.YaqlFunctionTestCase):
+
+    def test_function_jsonpath_query_static(self):
+        obj = {'people': [{'first': 'James', 'last': 'd'},
+                          {'first': 'Jacob', 'last': 'e'},
+                          {'first': 'Jayden', 'last': 'f'},
+                          {'missing': 'different'}],
+               'foo': {'bar': 'baz'}}
+        result = YAQL_ENGINE('jsonpath_query($.obj, "people[*].first")').evaluate(
+            context=self.get_yaql_context({'obj': obj})
+        )
+        expected = ['James', 'Jacob', 'Jayden']
+        self.assertEqual(result, expected)
+
+    def test_function_jsonpath_query_dynamic(self):
+        obj = {'people': [{'first': 'James', 'last': 'd'},
+                          {'first': 'Jacob', 'last': 'e'},
+                          {'first': 'Jayden', 'last': 'f'},
+                          {'missing': 'different'}],
+               'foo': {'bar': 'baz'}}
+        query = "people[*].last"
+        result = YAQL_ENGINE('jsonpath_query($.obj, $.query)').evaluate(
+            context=self.get_yaql_context({'obj': obj,
+                                           'query': query})
+        )
+        expected = ['d', 'e', 'f']
+        self.assertEqual(result, expected)
+
+    def test_function_jsonpath_query_no_results(self):
+        obj = {'people': [{'first': 'James', 'last': 'd'},
+                          {'first': 'Jacob', 'last': 'e'},
+                          {'first': 'Jayden', 'last': 'f'},
+                          {'missing': 'different'}],
+               'foo': {'bar': 'baz'}}
+        query = "query_returns_no_results"
+        result = YAQL_ENGINE('jsonpath_query($.obj, $.query)').evaluate(
+            context=self.get_yaql_context({'obj': obj,
+                                           'query': query})
+        )
+        expected = None
+        self.assertEqual(result, expected)

st2mistral/tests/unit/test_function_base.py

@@ -23,13 +23,17 @@
 def get_functions():
     from st2mistral.functions import data
     from st2mistral.functions import json_escape
+    from st2mistral.functions import jsonpath_query
     from st2mistral.functions import regex
     from st2mistral.functions import time
     from st2mistral.functions import use_none
     from st2mistral.functions import version
 
     return {
+        'from_json_string': data.from_json_string,
+        'from_yaml_string': data.from_yaml_string,
         'json_escape': json_escape.json_escape,
+        'jsonpath_query': jsonpath_query.jsonpath_query,
         'regex_match': regex.regex_match,
         'regex_replace': regex.regex_replace,
         'regex_search': regex.regex_search,