Skip to content

Random stuff #33

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ajroetker wants to merge 3 commits into
base: master
Choose a base branch
from
Open

Random stuff #33

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
7e84c86
Random cleanups
ajroetker Mar 5, 2025
3016bfd
Add filterql vm optimized
ajroetker Mar 5, 2025
74ce072
Optimize the optimized vm more
ajroetker Mar 6, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 6 additions & 9 deletions expr/builtins/aggregations.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,8 @@ import (
// Avg average of values. Note, this function DOES NOT persist state doesn't aggregate
// across multiple calls. That would be responsibility of write context.
//
// avg(1,2,3) => 2.0, true
// avg("hello") => math.NaN, false
//
// avg(1,2,3) => 2.0, true
// avg("hello") => math.NaN, false
type Avg struct{}

// Type is NumberType
Expand Down Expand Up @@ -68,9 +67,8 @@ func avgEval(ctx expr.EvalContext, vals []value.Value) (value.Value, bool) {
// Sum function to add values. Note, this function DOES NOT persist state doesn't aggregate
// across multiple calls. That would be responsibility of write context.
//
// sum(1, 2, 3) => 6
// sum(1, "horse", 3) => nan, false
//
// sum(1, 2, 3) => 6
// sum(1, "horse", 3) => nan, false
type Sum struct{}

// Type is number
Expand Down Expand Up @@ -133,9 +131,8 @@ func sumEval(ctx expr.EvalContext, vals []value.Value) (value.Value, bool) {
// and in general is a horrible, horrible function that needs to be replaced
// with occurrences of value, ignores the value and ensures it is non null
//
// count(anyvalue) => 1, true
// count(not_number) => -- 0, false
//
// count(anyvalue) => 1, true
// count(not_number) => -- 0, false
type Count struct{}

// Type is Integer
Expand Down
20 changes: 10 additions & 10 deletions expr/builtins/filter.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,7 @@ import (

// OneOf choose the first non-nil, non-zero, non-false fields
//
// oneof(nil, 0, "hello") => 'hello'
//
// oneof(nil, 0, "hello") => 'hello'
type OneOf struct{}

// Type unknown
Expand Down Expand Up @@ -65,16 +64,15 @@ func FiltersFromArgs(filterVals []value.Value) []string {
//
// Filter a map of values by key to remove certain keys
//
// filter(match("topic_"),key_to_filter, key2_to_filter) => {"goodkey": 22}, true
// filter(match("topic_"),key_to_filter, key2_to_filter) => {"goodkey": 22}, true
//
// Filter out VALUES (not keys) from a list of []string{} for a specific value
//
// filter(split("apples,oranges",","),"ora*") => ["apples"], true
// filter(split("apples,oranges",","),"ora*") => ["apples"], true
//
// Filter out values for single strings
//
// filter("apples","app*") => []string{}, true
//
// filter("apples","app*") => []string{}, true
type Filter struct{}

// Type unknown
Expand Down Expand Up @@ -187,16 +185,15 @@ func FilterEval(ctx expr.EvalContext, vals []value.Value) (value.Value, bool) {
//
// Filter a map of values by key to only keep certain keys
//
// filtermatch(match("topic_"),key_to_filter, key2_to_filter) => {"goodkey": 22}, true
// filtermatch(match("topic_"),key_to_filter, key2_to_filter) => {"goodkey": 22}, true
//
// Filter in VALUES (not keys) from a list of []string{} for a specific value
//
// filtermatch(split("apples,oranges",","),"ora*") => ["oranges"], true
// filtermatch(split("apples,oranges",","),"ora*") => ["oranges"], true
//
// Filter in values for single strings
//
// filtermatch("apples","app*") => []string{"apple"}, true
//
// filtermatch("apples","app*") => []string{"apple"}, true
type FilterMatch struct{}

// Type Unknown
Expand Down Expand Up @@ -272,6 +269,9 @@ func FilterMatchEval(ctx expr.EvalContext, vals []value.Value) (value.Value, boo
lv := make([]string, 0, val.Len())

for _, slv := range val.SliceValue() {
switch slv.Type() {
case value.StringType:
}
sv := slv.ToString()
filteredIn := false
for _, filter := range filters {
Expand Down
38 changes: 21 additions & 17 deletions expr/builtins/json.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,9 @@ var _ = u.EMPTY

// JsonPath jmespath json parser http://jmespath.org/
//
// json_field = `[{"name":"n1","ct":8,"b":true, "tags":["a","b"]},{"name":"n2","ct":10,"b": false, "tags":["a","b"]}]`
//
// json.jmespath(json_field, "[?name == 'n1'].name | [0]") => "n1"
// json_field = `[{"name":"n1","ct":8,"b":true, "tags":["a","b"]},{"name":"n2","ct":10,"b": false, "tags":["a","b"]}]`
//
// json.jmespath(json_field, "[?name == 'n1'].name | [0]") => "n1"
type JsonPath struct{}

func (m *JsonPath) Type() value.ValueType { return value.UnknownType }
Expand All @@ -34,33 +33,38 @@ func (m *JsonPath) Validate(n *expr.FuncNode) (expr.EvaluatorFunc, error) {
default:
return nil, fmt.Errorf("expected a string expression for jmespath got %T", jn)
}

parser := jmespath.NewParser()
_, err := parser.Parse(jsonPathExpr)
jmes, err := jmespath.Compile(jsonPathExpr)
if err != nil {
// if syntaxError, ok := err.(jmespath.SyntaxError); ok {
// u.Warnf("%s\n%s\n", syntaxError, syntaxError.HighlightLocation())
// }
return nil, err
}
return jsonPathEval(jsonPathExpr), nil
return jsonPathEval(jmes), nil
}

func jsonPathEval(expression string) expr.EvaluatorFunc {
func jsonPathEval(jmes *jmespath.JMESPath) expr.EvaluatorFunc {
return func(ctx expr.EvalContext, args []value.Value) (value.Value, bool) {
if args[0] == nil || args[0].Err() || args[0].Nil() {
return nil, false
}

val := args[0].ToString()

// Validate that this is valid json?
a := args[0]
var val []byte
var err error
switch {
case a.Type().IsMap() || a.Type().IsSlice():
// TODO (ajr): need to recursively change value.Value to interface{} and extract the values
// this is a bit of a hack to do that
val, err = json.Marshal(a.Value())
if err != nil {
return nil, false
}
default:
val = []byte(args[0].ToString())
}
var data interface{}
if err := json.Unmarshal([]byte(val), &data); err != nil {
if err := json.Unmarshal(val, &data); err != nil {
return nil, false
}

result, err := jmespath.Search(expression, data)
result, err := jmes.Search(data)
if err != nil {
return nil, false
}
Expand Down
67 changes: 30 additions & 37 deletions expr/builtins/list_map.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,8 @@ var _ = u.EMPTY

// len length of array types
//
// len([1,2,3]) => 3, true
// len(not_a_field) => -- NilInt, false
//
// len([1,2,3]) => 3, true
// len(not_a_field) => -- NilInt, false
type Length struct{}

// Type is IntType
Expand Down Expand Up @@ -63,13 +62,12 @@ func lenEval(ctx expr.EvalContext, args []value.Value) (value.Value, bool) {

// ArrayIndex array.index choose the nth element of an array
//
// // given context input of
// "items" = [1,2,3]
//
// array.index(items, 1) => 1, true
// array.index(items, 5) => nil, false
// array.index(items, -1) => 3, true
// // given context input of
// "items" = [1,2,3]
//
// array.index(items, 1) => 1, true
// array.index(items, 5) => nil, false
// array.index(items, -1) => 3, true
type ArrayIndex struct{}

// Type unknown - returns single value from SliceValue array
Expand Down Expand Up @@ -108,12 +106,12 @@ func arrayIndexEval(ctx expr.EvalContext, args []value.Value) (value.Value, bool

// array.slice slice element m -> n of a slice. First arg must be a slice.
//
// // given context of
// "items" = [1,2,3,4,5]
// // given context of
// "items" = [1,2,3,4,5]
//
// array.slice(items, 1, 3) => [2,3], true
// array.slice(items, 2) => [3,4,5], true
// array.slice(items, -2) => [4,5], true
// array.slice(items, 1, 3) => [2,3], true
// array.slice(items, 2) => [3,4,5], true
// array.slice(items, -2) => [4,5], true
type ArraySlice struct{}

// Type Unknown for Array Slice
Expand Down Expand Up @@ -208,8 +206,7 @@ func arraySliceEval(ctx expr.EvalContext, args []value.Value) (value.Value, bool
// Map Create a map from two values. If the right side value is nil
// then does not evaluate.
//
// map(left, right) => map[string]value{left:right}
//
// map(left, right) => map[string]value{left:right}
type MapFunc struct{}

// Type is MapValueType
Expand All @@ -236,9 +233,8 @@ func mapEval(ctx expr.EvalContext, args []value.Value) (value.Value, bool) {

// MapTime() Create a map[string]time of each key
//
// maptime(field) => map[string]time{field_value:message_timestamp}
// maptime(field, timestamp) => map[string]time{field_value:timestamp}
//
// maptime(field) => map[string]time{field_value:message_timestamp}
// maptime(field, timestamp) => map[string]time{field_value:timestamp}
type MapTime struct{}

// Type MapTime
Expand Down Expand Up @@ -282,13 +278,13 @@ func mapTimeEval(ctx expr.EvalContext, args []value.Value) (value.Value, bool) {
// - May pass as many match strings as you want.
// - Must match on Prefix of key.
//
// given input context of:
// {"score_value":24,"event_click":true, "tag_apple": "apple", "label_orange": "orange"}
// given input context of:
// {"score_value":24,"event_click":true, "tag_apple": "apple", "label_orange": "orange"}
//
// match("score_") => {"value":24}
// match("amount_") => false
// match("event_") => {"click":true}
// match("label_","tag_") => {"apple":"apple","orange":"orange"}
// match("score_") => {"value":24}
// match("amount_") => false
// match("event_") => {"click":true}
// match("label_","tag_") => {"apple":"apple","orange":"orange"}
type Match struct{}

// Type is MapValueType
Expand Down Expand Up @@ -328,11 +324,10 @@ func matchEval(ctx expr.EvalContext, args []value.Value) (value.Value, bool) {

// MapKeys: Take a map and extract array of keys
//
// //given input:
// {"tag.1":"news","tag.2":"sports"}
//
// mapkeys(match("tag.")) => []string{"news","sports"}
// //given input:
// {"tag.1":"news","tag.2":"sports"}
//
// mapkeys(match("tag.")) => []string{"news","sports"}
type MapKeys struct{}

// Type []string aka strings
Expand Down Expand Up @@ -368,11 +363,10 @@ func mapKeysEval(ctx expr.EvalContext, args []value.Value) (value.Value, bool) {

// MapValues: Take a map and extract array of values
//
// // given input:
// {"tag.1":"news","tag.2":"sports"}
//
// mapvalue(match("tag.")) => []string{"1","2"}
// // given input:
// {"tag.1":"news","tag.2":"sports"}
//
// mapvalue(match("tag.")) => []string{"1","2"}
type MapValues struct{}

// Type strings aka []string
Expand Down Expand Up @@ -409,11 +403,10 @@ func mapValuesEval(ctx expr.EvalContext, args []value.Value) (value.Value, bool)

// MapInvert: Take a map and invert key/values
//
// // given input:
// tags = {"1":"news","2":"sports"}
//
// mapinvert(tags) => map[string]string{"news":"1","sports":"2"}
// // given input:
// tags = {"1":"news","2":"sports"}
//
// mapinvert(tags) => map[string]string{"news":"1","sports":"2"}
type MapInvert struct{}

// Type MapValue
Expand Down
Loading