From 945a02c9ab06c60614ded28d7a13e408cd9a0bda Mon Sep 17 00:00:00 2001 From: James Cor Date: Tue, 23 May 2023 14:16:41 -0700 Subject: [PATCH 1/7] add support for scan op --- jsonpath.go | 41 ++++++++++++++++++++++- jsonpath_test.go | 86 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 126 insertions(+), 1 deletion(-) diff --git a/jsonpath.go b/jsonpath.go index 6e69912..3b6f32b 100644 --- a/jsonpath.go +++ b/jsonpath.go @@ -76,7 +76,7 @@ func (c *Compiled) String() string { func (c *Compiled) Lookup(obj interface{}) (interface{}, error) { var err error for _, s := range c.steps { - // "key", "idx" + // "key", "idx", "range", "filter", "scan" switch s.op { case "key": obj, err = get_key(obj, s.key) @@ -138,6 +138,11 @@ func (c *Compiled) Lookup(obj interface{}) (interface{}, error) { if err != nil { return nil, err } + case "scan": + obj, err = get_scan(obj) + if err != nil { + return nil, err + } default: return nil, fmt.Errorf("expression don't support in filter") } @@ -550,6 +555,40 @@ func get_filtered(obj, root interface{}, filter string) ([]interface{}, error) { return res, nil } +func get_scan(obj interface{}) (interface{}, error) { + if reflect.TypeOf(obj) == nil { + return nil, ErrGetFromNullObj + } + switch reflect.TypeOf(obj).Kind() { + case reflect.Map: + var res []interface{} + if jsonMap, ok := obj.(map[string]interface{}); ok { + for _, v := range jsonMap { + res = append(res, v) + } + return res, nil + } + for _, kv := range reflect.ValueOf(obj).MapKeys() { + res = append(res, reflect.ValueOf(obj).MapIndex(kv).Interface()) + } + return res, nil + case reflect.Slice: + // slice we should get from all objects in it. + var res []interface{} + for i := 0; i < reflect.ValueOf(obj).Len(); i++ { + tmp := reflect.ValueOf(obj).Index(i).Interface() + newObj, err := get_scan(tmp) + if err != nil { + return nil, err + } + res = append(res, newObj.([]interface{})...) + } + return res, nil + default: + return nil, fmt.Errorf("object is not scanable: %v", reflect.TypeOf(obj).Kind()) + } +} + // @.isbn => @.isbn, exists, nil // @.price < 10 => @.price, <, 10 // @.price <= $.expensive => @.price, <=, $.expensive diff --git a/jsonpath_test.go b/jsonpath_test.go index 9e343e5..1a2f2e0 100644 --- a/jsonpath_test.go +++ b/jsonpath_test.go @@ -537,6 +537,92 @@ func Test_jsonpath_get_range(t *testing.T) { } } +func Test_jsonpath_get_scan(t *testing.T) { + obj := map[string]interface{}{ + "key": 1, + } + res, err := get_scan(obj) + fmt.Println(err, res) + if err != nil { + t.Errorf("failed to scan: %v", err) + return + } + if res.([]interface{})[0] != 1 { + t.Errorf("scanned value is not 1: %v", res) + return + } + + obj2 := 1 + res, err = get_scan(obj2) + fmt.Println(err, res) + if err == nil { + t.Errorf("object is not scanable error not raised") + return + } + + obj3 := map[string]string{"key1": "hah1", "key2": "hah2", "key3": "hah3"} + res, err = get_scan(obj3) + if err != nil { + t.Errorf("failed to scan: %v", err) + return + } + res_v, ok := res.([]interface{}) + if !ok { + t.Errorf("scanned result is not a slice") + } + if len(res_v) != 3 { + t.Errorf("scanned result is of wrong length") + } + // order of items in maps can't be guaranteed + for _, v := range res_v { + val, _ := v.(string) + if val != "hah1" && val != "hah2" && val != "hah3" { + t.Errorf("scanned result contains unexpected value: %v", val) + } + } + + obj4 := map[string]interface{}{ + "key1" : "abc", + "key2" : 123, + "key3" : map[string]interface{}{ + "a": 1, + "b": 2, + "c": 3, + }, + "key4" : []interface{}{1,2,3}, + } + res, err = get_scan(obj4) + res_v, ok = res.([]interface{}) + if !ok { + t.Errorf("scanned result is not a slice") + } + if len(res_v) != 4 { + t.Errorf("scanned result is of wrong length") + } + // order of items in maps can't be guaranteed + for _, v := range res_v { + switch v.(type) { + case string: + if v_str, ok := v.(string); ok && v_str == "abc" { + continue + } + case int: + if v_int, ok := v.(int); ok && v_int == 123 { + continue + } + case map[string]interface{}: + if v_map, ok := v.(map[string]interface{}); ok && v_map["a"].(int) == 1 && v_map["b"].(int) == 2 && v_map["c"].(int) == 3 { + continue + } + case []interface{}: + if v_slice, ok := v.([]interface{}); ok && v_slice[0].(int) == 1 && v_slice[1].(int) == 2 && v_slice[2].(int) == 3 { + continue + } + } + t.Errorf("scanned result contains unexpected value: %v", v) + } +} + func Test_jsonpath_types_eval(t *testing.T) { fset := token.NewFileSet() res, err := types.Eval(fset, nil, 0, "1 < 2") From 6d16306c8437c7ea8e177d276ac1cd8bf3c58400 Mon Sep 17 00:00:00 2001 From: James Cor Date: Tue, 23 May 2023 15:56:01 -0700 Subject: [PATCH 2/7] improve error message --- jsonpath.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jsonpath.go b/jsonpath.go index 3b6f32b..8a41290 100644 --- a/jsonpath.go +++ b/jsonpath.go @@ -144,7 +144,7 @@ func (c *Compiled) Lookup(obj interface{}) (interface{}, error) { return nil, err } default: - return nil, fmt.Errorf("expression don't support in filter") + return nil, fmt.Errorf("unsupported operation: %s", s.op) } } return obj, nil @@ -359,7 +359,7 @@ func filter_get_from_explicit_path(obj interface{}, path string) (interface{}, e return nil, err } default: - return nil, fmt.Errorf("expression don't support in filter") + return nil, fmt.Errorf("unsupported operation %s in filter", op) } } return xobj, nil From e276706772c79a7516ae14dbd1389b974e720893 Mon Sep 17 00:00:00 2001 From: James Cor Date: Tue, 23 May 2023 16:03:13 -0700 Subject: [PATCH 3/7] fmt to t.Log --- jsonpath.go | 4 ++-- jsonpath_test.go | 46 +++++++++++++++++++++++----------------------- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/jsonpath.go b/jsonpath.go index 8a41290..2dadabc 100644 --- a/jsonpath.go +++ b/jsonpath.go @@ -144,7 +144,7 @@ func (c *Compiled) Lookup(obj interface{}) (interface{}, error) { return nil, err } default: - return nil, fmt.Errorf("unsupported operation: %s", s.op) + return nil, fmt.Errorf("unsupported jsonpath operation: %s", s.op) } } return obj, nil @@ -359,7 +359,7 @@ func filter_get_from_explicit_path(obj interface{}, path string) (interface{}, e return nil, err } default: - return nil, fmt.Errorf("unsupported operation %s in filter", op) + return nil, fmt.Errorf("unsupported jsonpath operation %s in filter", op) } } return xobj, nil diff --git a/jsonpath_test.go b/jsonpath_test.go index 1a2f2e0..90865d6 100644 --- a/jsonpath_test.go +++ b/jsonpath_test.go @@ -337,7 +337,7 @@ func Test_jsonpath_parse_token(t *testing.T) { if op == "range" { if args_v, ok := args.([2]interface{}); ok == true { - fmt.Println(args_v) + t.Logf("%v", args_v) exp_from := exp_args.([2]interface{})[0] exp_to := exp_args.([2]interface{})[1] if args_v[0] != exp_from { @@ -356,7 +356,7 @@ func Test_jsonpath_parse_token(t *testing.T) { if op == "filter" { if args_v, ok := args.(string); ok == true { - fmt.Println(args_v) + t.Logf(args_v) if exp_args.(string) != args_v { t.Errorf("len(args) not expected: (got)%v != (exp)%v", len(args_v), len(exp_args.([]string))) return @@ -374,7 +374,7 @@ func Test_jsonpath_get_key(t *testing.T) { "key": 1, } res, err := get_key(obj, "key") - fmt.Println(err, res) + t.Logf("err: %v, res: %v", err, res) if err != nil { t.Errorf("failed to get key: %v", err) return @@ -385,7 +385,7 @@ func Test_jsonpath_get_key(t *testing.T) { } res, err = get_key(obj, "hah") - fmt.Println(err, res) + t.Logf("err: %v, res: %v", err, res) if err == nil { t.Errorf("key error not raised") return @@ -397,7 +397,7 @@ func Test_jsonpath_get_key(t *testing.T) { obj2 := 1 res, err = get_key(obj2, "key") - fmt.Println(err, res) + t.Logf("err: %v, res: %v", err, res) if err == nil { t.Errorf("object is not map error not raised") @@ -406,7 +406,7 @@ func Test_jsonpath_get_key(t *testing.T) { obj3 := map[string]string{"key": "hah"} res, err = get_key(obj3, "key") if res_v, ok := res.(string); ok != true || res_v != "hah" { - fmt.Println(err, res) + t.Logf("err: %v, res: %v", err, res) t.Errorf("map[string]string support failed") } @@ -419,13 +419,13 @@ func Test_jsonpath_get_key(t *testing.T) { }, } res, err = get_key(obj4, "a") - fmt.Println(err, res) + t.Logf("err: %v, res: %v", err, res) } func Test_jsonpath_get_idx(t *testing.T) { obj := []interface{}{1, 2, 3, 4} res, err := get_idx(obj, 0) - fmt.Println(err, res) + t.Logf("err: %v, res: %v", err, res) if err != nil { t.Errorf("failed to get_idx(obj,0): %v", err) return @@ -435,19 +435,19 @@ func Test_jsonpath_get_idx(t *testing.T) { } res, err = get_idx(obj, 2) - fmt.Println(err, res) + t.Logf("err: %v, res: %v", err, res) if v, ok := res.(int); ok != true || v != 3 { t.Errorf("failed to get int 3") } res, err = get_idx(obj, 4) - fmt.Println(err, res) + t.Logf("err: %v, res: %v", err, res) if err == nil { t.Errorf("index out of range error not raised") return } res, err = get_idx(obj, -1) - fmt.Println(err, res) + t.Logf("err: %v, res: %v", err, res) if err != nil { t.Errorf("failed to get_idx(obj, -1): %v", err) return @@ -457,13 +457,13 @@ func Test_jsonpath_get_idx(t *testing.T) { } res, err = get_idx(obj, -4) - fmt.Println(err, res) + t.Logf("err: %v, res: %v", err, res) if v, ok := res.(int); ok != true || v != 1 { t.Errorf("failed to get int 1") } res, err = get_idx(obj, -5) - fmt.Println(err, res) + t.Logf("err: %v, res: %v", err, res) if err == nil { t.Errorf("index out of range error not raised") return @@ -478,7 +478,7 @@ func Test_jsonpath_get_idx(t *testing.T) { obj2 := []int{1, 2, 3, 4} res, err = get_idx(obj2, 0) - fmt.Println(err, res) + t.Logf("err: %v, res: %v", err, res) if err != nil { t.Errorf("failed to get_idx(obj2,0): %v", err) return @@ -492,7 +492,7 @@ func Test_jsonpath_get_range(t *testing.T) { obj := []int{1, 2, 3, 4, 5} res, err := get_range(obj, 0, 2) - fmt.Println(err, res) + t.Logf("err: %v, res: %v", err, res) if err != nil { t.Errorf("failed to get_range: %v", err) } @@ -502,11 +502,11 @@ func Test_jsonpath_get_range(t *testing.T) { obj1 := []interface{}{1, 2, 3, 4, 5} res, err = get_range(obj1, 3, -1) - fmt.Println(err, res) + t.Logf("err: %v, res: %v", err, res) if err != nil { t.Errorf("failed to get_range: %v", err) } - fmt.Println(res.([]interface{})) + t.Logf("%v", res.([]interface{})) if res.([]interface{})[0] != 4 || res.([]interface{})[1] != 5 { t.Errorf("failed get_range: %v, expect: [4,5]", res) } @@ -531,7 +531,7 @@ func Test_jsonpath_get_range(t *testing.T) { obj2 := 2 res, err = get_range(obj2, 0, 1) - fmt.Println(err, res) + t.Logf("err: %v, res: %v", err, res) if err == nil { t.Errorf("object is Slice error not raised") } @@ -542,7 +542,7 @@ func Test_jsonpath_get_scan(t *testing.T) { "key": 1, } res, err := get_scan(obj) - fmt.Println(err, res) + t.Logf("err: %v, res: %v", err, res) if err != nil { t.Errorf("failed to scan: %v", err) return @@ -554,7 +554,7 @@ func Test_jsonpath_get_scan(t *testing.T) { obj2 := 1 res, err = get_scan(obj2) - fmt.Println(err, res) + t.Logf("err: %v, res: %v", err, res) if err == nil { t.Errorf("object is not scanable error not raised") return @@ -626,7 +626,7 @@ func Test_jsonpath_get_scan(t *testing.T) { func Test_jsonpath_types_eval(t *testing.T) { fset := token.NewFileSet() res, err := types.Eval(fset, nil, 0, "1 < 2") - fmt.Println(err, res, res.Type, res.Value, res.IsValue()) + t.Logf("err: %v, res: %v, res.Type: %v, res.Value: %v, res.IsValue: %v", err, res, res.Type, res.Value, res.IsValue()) } var tcase_parse_filter = []map[string]interface{}{ @@ -833,7 +833,7 @@ var tcase_eval_filter = []map[string]interface{}{ func Test_jsonpath_eval_filter(t *testing.T) { for idx, tcase := range tcase_eval_filter[1:] { - fmt.Println("------------------------------") + t.Logf("------------------------------") obj := tcase["obj"].(map[string]interface{}) root := tcase["root"].(map[string]interface{}) lp := tcase["lp"].(string) @@ -1160,7 +1160,7 @@ var tcases_reg_op = []struct { func TestRegOp(t *testing.T) { for idx, tcase := range tcases_reg_op { - fmt.Println("idx: ", idx, "tcase: ", tcase) + t.Logf("idx: %v, tcase: %v", idx, tcase) res, err := regFilterCompile(tcase.Line) if tcase.Err == true { if err == nil { From a540fe6f52725c016a63799afdab7058ab6c1e53 Mon Sep 17 00:00:00 2001 From: James Cor Date: Tue, 23 May 2023 16:07:12 -0700 Subject: [PATCH 4/7] better map iteration --- jsonpath.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/jsonpath.go b/jsonpath.go index 2dadabc..bbcb181 100644 --- a/jsonpath.go +++ b/jsonpath.go @@ -568,8 +568,9 @@ func get_scan(obj interface{}) (interface{}, error) { } return res, nil } - for _, kv := range reflect.ValueOf(obj).MapKeys() { - res = append(res, reflect.ValueOf(obj).MapIndex(kv).Interface()) + iter := reflect.ValueOf(obj).MapRange() + for iter.Next() { + res = append(res, iter.Value().Interface()) } return res, nil case reflect.Slice: From 798dfee70c00146cd1a3fa5aa36e6e1b20843acc Mon Sep 17 00:00:00 2001 From: James Cor Date: Tue, 23 May 2023 16:59:54 -0700 Subject: [PATCH 5/7] fix logic for empty and null scans --- jsonpath.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/jsonpath.go b/jsonpath.go index bbcb181..215c461 100644 --- a/jsonpath.go +++ b/jsonpath.go @@ -143,6 +143,13 @@ func (c *Compiled) Lookup(obj interface{}) (interface{}, error) { if err != nil { return nil, err } + if obj == nil { + continue + } + // empty scan is NULL + if len(obj.([]interface{})) == 0 { + obj = nil + } default: return nil, fmt.Errorf("unsupported jsonpath operation: %s", s.op) } @@ -557,7 +564,7 @@ func get_filtered(obj, root interface{}, filter string) ([]interface{}, error) { func get_scan(obj interface{}) (interface{}, error) { if reflect.TypeOf(obj) == nil { - return nil, ErrGetFromNullObj + return nil, nil } switch reflect.TypeOf(obj).Kind() { case reflect.Map: From 440ff8ccabbdb0d64d0af9253d1bb02b3dadc2e6 Mon Sep 17 00:00:00 2001 From: James Cor Date: Tue, 23 May 2023 17:02:04 -0700 Subject: [PATCH 6/7] test --- jsonpath_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/jsonpath_test.go b/jsonpath_test.go index 90865d6..951efb5 100644 --- a/jsonpath_test.go +++ b/jsonpath_test.go @@ -590,13 +590,14 @@ func Test_jsonpath_get_scan(t *testing.T) { "c": 3, }, "key4" : []interface{}{1,2,3}, + "key5" : nil, } res, err = get_scan(obj4) res_v, ok = res.([]interface{}) if !ok { t.Errorf("scanned result is not a slice") } - if len(res_v) != 4 { + if len(res_v) != 5 { t.Errorf("scanned result is of wrong length") } // order of items in maps can't be guaranteed @@ -618,6 +619,8 @@ func Test_jsonpath_get_scan(t *testing.T) { if v_slice, ok := v.([]interface{}); ok && v_slice[0].(int) == 1 && v_slice[1].(int) == 2 && v_slice[2].(int) == 3 { continue } + case nil: + continue } t.Errorf("scanned result contains unexpected value: %v", v) } From 5e86621904bdc32d343771addd24d2c22f253d49 Mon Sep 17 00:00:00 2001 From: James Cor Date: Wed, 24 May 2023 11:47:43 -0700 Subject: [PATCH 7/7] updating tests with ordering enforced --- jsonpath.go | 31 +++++++++++++++++++++------ jsonpath_test.go | 55 ++++++++++++++++++++---------------------------- 2 files changed, 48 insertions(+), 38 deletions(-) diff --git a/jsonpath.go b/jsonpath.go index 215c461..4f55bc8 100644 --- a/jsonpath.go +++ b/jsonpath.go @@ -13,6 +13,7 @@ import ( "go/types" "reflect" "regexp" + "sort" "strconv" "strings" ) @@ -568,16 +569,34 @@ func get_scan(obj interface{}) (interface{}, error) { } switch reflect.TypeOf(obj).Kind() { case reflect.Map: + // iterate over keys in sorted by length, then alphabetically var res []interface{} if jsonMap, ok := obj.(map[string]interface{}); ok { - for _, v := range jsonMap { - res = append(res, v) + var sortedKeys []string + for k := range jsonMap { + sortedKeys = append(sortedKeys, k) + } + sort.Slice(sortedKeys, func(i, j int) bool { + if len(sortedKeys[i]) != len(sortedKeys[j]) { + return len(sortedKeys[i]) < len(sortedKeys[j]) + } + return sortedKeys[i] < sortedKeys[j] + }) + for _, k := range sortedKeys { + res = append(res, jsonMap[k]) } return res, nil } - iter := reflect.ValueOf(obj).MapRange() - for iter.Next() { - res = append(res, iter.Value().Interface()) + keys := reflect.ValueOf(obj).MapKeys() + sort.Slice(keys, func(i, j int) bool { + ki, kj := keys[i].String(), keys[j].String() + if len(ki) != len(kj) { + return len(ki) < len(kj) + } + return ki < kj + }) + for _, k := range keys { + res = append(res, reflect.ValueOf(obj).MapIndex(k).Interface()) } return res, nil case reflect.Slice: @@ -593,7 +612,7 @@ func get_scan(obj interface{}) (interface{}, error) { } return res, nil default: - return nil, fmt.Errorf("object is not scanable: %v", reflect.TypeOf(obj).Kind()) + return nil, fmt.Errorf("object is not scannable: %v", reflect.TypeOf(obj).Kind()) } } diff --git a/jsonpath_test.go b/jsonpath_test.go index 951efb5..b98dd53 100644 --- a/jsonpath_test.go +++ b/jsonpath_test.go @@ -542,7 +542,6 @@ func Test_jsonpath_get_scan(t *testing.T) { "key": 1, } res, err := get_scan(obj) - t.Logf("err: %v, res: %v", err, res) if err != nil { t.Errorf("failed to scan: %v", err) return @@ -554,9 +553,8 @@ func Test_jsonpath_get_scan(t *testing.T) { obj2 := 1 res, err = get_scan(obj2) - t.Logf("err: %v, res: %v", err, res) - if err == nil { - t.Errorf("object is not scanable error not raised") + if err == nil || err.Error() != "object is not scannable: int" { + t.Errorf("object is not scannable error not raised") return } @@ -573,12 +571,14 @@ func Test_jsonpath_get_scan(t *testing.T) { if len(res_v) != 3 { t.Errorf("scanned result is of wrong length") } - // order of items in maps can't be guaranteed - for _, v := range res_v { - val, _ := v.(string) - if val != "hah1" && val != "hah2" && val != "hah3" { - t.Errorf("scanned result contains unexpected value: %v", val) - } + if v, ok := res_v[0].(string); !ok || v != "hah1" { + t.Errorf("scanned result contains unexpected value: %v", v) + } + if v, ok := res_v[1].(string); !ok || v != "hah2" { + t.Errorf("scanned result contains unexpected value: %v", v) + } + if v, ok := res_v[2].(string); !ok || v != "hah3" { + t.Errorf("scanned result contains unexpected value: %v", v) } obj4 := map[string]interface{}{ @@ -600,30 +600,21 @@ func Test_jsonpath_get_scan(t *testing.T) { if len(res_v) != 5 { t.Errorf("scanned result is of wrong length") } - // order of items in maps can't be guaranteed - for _, v := range res_v { - switch v.(type) { - case string: - if v_str, ok := v.(string); ok && v_str == "abc" { - continue - } - case int: - if v_int, ok := v.(int); ok && v_int == 123 { - continue - } - case map[string]interface{}: - if v_map, ok := v.(map[string]interface{}); ok && v_map["a"].(int) == 1 && v_map["b"].(int) == 2 && v_map["c"].(int) == 3 { - continue - } - case []interface{}: - if v_slice, ok := v.([]interface{}); ok && v_slice[0].(int) == 1 && v_slice[1].(int) == 2 && v_slice[2].(int) == 3 { - continue - } - case nil: - continue - } + if v, ok := res_v[0].(string); !ok || v != "abc" { + t.Errorf("scanned result contains unexpected value: %v", v) + } + if v, ok := res_v[1].(int); !ok || v != 123 { t.Errorf("scanned result contains unexpected value: %v", v) } + if v, ok := res_v[2].(map[string]interface{}); !ok || v["a"].(int) != 1 || v["b"].(int) != 2 || v["c"].(int) != 3 { + t.Errorf("scanned result contains unexpected value: %v", v) + } + if v, ok := res_v[3].([]interface{}); !ok || v[0].(int) != 1 || v[1].(int) != 2 || v[2].(int) != 3 { + t.Errorf("scanned result contains unexpected value: %v", v) + } + if res_v[4] != nil { + t.Errorf("scanned result contains unexpected value: %v", res_v[4]) + } } func Test_jsonpath_types_eval(t *testing.T) {