From 74f9cd8da2ee483da46d123039fcb304bed749d3 Mon Sep 17 00:00:00 2001 From: Benoit Masson Date: Sat, 16 Jul 2016 22:20:50 +0200 Subject: [PATCH] AllKeys() includes all keys / AllSettings() includes overridden nested values * Function AllKeys() now returns all keys holding a value (for nested values, the nested key is the full path, i.e., a sequence of dot-separated keys). Previously, returned only depth-1 keys, as well as flags and environment variables: this is more generic and may be used widely. Besides, it takes into account shadowed keys (key ignored if shadowed by a path at a higher-priority level). * Function AllSettings() now returns nested maps for all keys holding a value, as specified by AllKeys(). The value stored in the map is the one with highest priority, as returned by the Get() function (taking into account aliases, environment variables, flags, etc.). This fixes Unmarshal(): it fills in correct values for nested configuration elements overridden by flags or env variables. + tests fixed accordingly + test added to TestShadowedNestedValue(), to test Unmarshalling of shadowed keys --- viper.go | 129 ++++++++++++++++++++++++++++++++++++-------------- viper_test.go | 24 ++++++---- 2 files changed, 108 insertions(+), 45 deletions(-) diff --git a/viper.go b/viper.go index 7729e0c..8f27849 100644 --- a/viper.go +++ b/viper.go @@ -1286,55 +1286,114 @@ func (v *Viper) watchRemoteConfig(provider RemoteProvider) (map[string]interface return v.kvstore, err } -// AllKeys returns all keys regardless where they are set. +// AllKeys returns all keys holding a value, regardless of where they are set. +// Nested keys are returned with a v.keyDelim (= ".") separator func AllKeys() []string { return v.AllKeys() } func (v *Viper) AllKeys() []string { - m := map[string]struct{}{} - - for key := range v.defaults { - m[strings.ToLower(key)] = struct{}{} - } - - for key := range v.pflags { - m[strings.ToLower(key)] = struct{}{} - } - - for key := range v.env { - m[strings.ToLower(key)] = struct{}{} - } - - for key := range v.config { - m[strings.ToLower(key)] = struct{}{} - } - - for key := range v.kvstore { - m[strings.ToLower(key)] = struct{}{} - } - - for key := range v.override { - m[strings.ToLower(key)] = struct{}{} - } - - for key := range v.aliases { - m[strings.ToLower(key)] = struct{}{} - } + m := map[string]bool{} + // add all paths, by order of descending priority to ensure correct shadowing + m = v.flattenAndMergeMap(m, castMapStringToMapInterface(v.aliases), "") + m = v.flattenAndMergeMap(m, v.override, "") + m = v.mergeFlatMap(m, v.pflags) + m = v.mergeFlatMap(m, v.env) + m = v.flattenAndMergeMap(m, v.config, "") + m = v.flattenAndMergeMap(m, v.kvstore, "") + m = v.flattenAndMergeMap(m, v.defaults, "") + // convert set of paths to list a := []string{} for x := range m { a = append(a, x) } - return a } -// AllSettings returns all settings as a map[string]interface{}. +// flattenAndMergeMap recursively flattens the given map into a map[string]bool +// of key paths (used as a set, easier to manipulate than a []string): +// - each path is merged into a single key string, delimited with v.keyDelim (= ".") +// - if a path is shadowed by an earlier value in the initial shadow map, +// it is skipped. +// The resulting set of paths is merged to the given shadow set at the same time. +func (v *Viper) flattenAndMergeMap(shadow map[string]bool, m map[string]interface{}, prefix string) map[string]bool { + if shadow != nil && prefix != "" && shadow[prefix] { + // prefix is shadowed => nothing more to flatten + return shadow + } + if shadow == nil { + shadow = make(map[string]bool) + } + + var m2 map[string]interface{} + if prefix != "" { + prefix += v.keyDelim + } + for k, val := range m { + fullKey := prefix + k + switch val.(type) { + case map[string]interface{}: + m2 = val.(map[string]interface{}) + case map[interface{}]interface{}: + m2 = cast.ToStringMap(val) + default: + // immediate value + shadow[strings.ToLower(fullKey)] = true + continue + } + // recursively merge to shadow map + shadow = v.flattenAndMergeMap(shadow, m2, fullKey) + } + return shadow +} + +// mergeFlatMap merges the given maps, excluding values of the second map +// shadowed by values from the first map. +func (v *Viper) mergeFlatMap(shadow map[string]bool, mi interface{}) map[string]bool { + // unify input map + var m map[string]interface{} + switch mi.(type) { + case map[string]string, map[string]FlagValue: + m = cast.ToStringMap(mi) + default: + return shadow + } + + // scan keys +outer: + for k, _ := range m { + path := strings.Split(k, v.keyDelim) + // scan intermediate paths + var parentKey string + for i := 1; i < len(path); i++ { + parentKey = strings.Join(path[0:i], v.keyDelim) + if shadow[parentKey] { + // path is shadowed, continue + continue outer + } + } + // add key + shadow[strings.ToLower(k)] = true + } + return shadow +} + +// AllSettings merges all settings and returns them as a map[string]interface{}. func AllSettings() map[string]interface{} { return v.AllSettings() } func (v *Viper) AllSettings() map[string]interface{} { m := map[string]interface{}{} - for _, x := range v.AllKeys() { - m[x] = v.Get(x) + // start from the list of keys, and construct the map one value at a time + for _, k := range v.AllKeys() { + value := v.Get(k) + if value == nil { + // should not happen, since AllKeys() returns only keys holding a value, + // check just in case anything changes + continue + } + path := strings.Split(k, v.keyDelim) + lastKey := strings.ToLower(path[len(path)-1]) + deepestMap := deepSearch(m, path[0:len(path)-1]) + // set innermost value + deepestMap[lastKey] = value } - return m } diff --git a/viper_test.go b/viper_test.go index 9da2e7e..02d6eb1 100644 --- a/viper_test.go +++ b/viper_test.go @@ -422,9 +422,9 @@ func TestSetEnvReplacer(t *testing.T) { func TestAllKeys(t *testing.T) { initConfigs() - ks := sort.StringSlice{"title", "newkey", "owner", "name", "beard", "ppu", "batters", "hobbies", "clothing", "age", "hacker", "id", "type", "eyes", "p_id", "p_ppu", "p_batters", "p_type", "p_name", "foos"} + ks := sort.StringSlice{"title", "newkey", "owner.organization", "owner.dob", "owner.bio", "name", "beard", "ppu", "batters.batter", "hobbies", "clothing.jacket", "clothing.trousers", "clothing.pants.size", "age", "hacker", "id", "type", "eyes", "p_id", "p_ppu", "p_batters.batter.type", "p_type", "p_name", "foos"} dob, _ := time.Parse(time.RFC3339, "1979-05-27T07:32:00Z") - all := map[string]interface{}{"owner": map[string]interface{}{"organization": "MongoDB", "Bio": "MongoDB Chief Developer Advocate & Hacker at Large", "dob": dob}, "title": "TOML Example", "ppu": 0.55, "eyes": "brown", "clothing": map[string]interface{}{"trousers": "denim", "jacket": "leather", "pants": map[interface{}]interface{}{"size": "large"}}, "id": "0001", "batters": map[string]interface{}{"batter": []interface{}{map[string]interface{}{"type": "Regular"}, map[string]interface{}{"type": "Chocolate"}, map[string]interface{}{"type": "Blueberry"}, map[string]interface{}{"type": "Devil's Food"}}}, "hacker": true, "beard": true, "hobbies": []interface{}{"skateboarding", "snowboarding", "go"}, "age": 35, "type": "donut", "newkey": "remote", "name": "Cake", "p_id": "0001", "p_ppu": "0.55", "p_name": "Cake", "p_batters": map[string]interface{}{"batter": map[string]interface{}{"type": "Regular"}}, "p_type": "donut", "foos": []map[string]interface{}{map[string]interface{}{"foo": []map[string]interface{}{map[string]interface{}{"key": 1}, map[string]interface{}{"key": 2}, map[string]interface{}{"key": 3}, map[string]interface{}{"key": 4}}}}} + all := map[string]interface{}{"owner": map[string]interface{}{"organization": "MongoDB", "bio": "MongoDB Chief Developer Advocate & Hacker at Large", "dob": dob}, "title": "TOML Example", "ppu": 0.55, "eyes": "brown", "clothing": map[string]interface{}{"trousers": "denim", "jacket": "leather", "pants": map[string]interface{}{"size": "large"}}, "id": "0001", "batters": map[string]interface{}{"batter": []interface{}{map[string]interface{}{"type": "Regular"}, map[string]interface{}{"type": "Chocolate"}, map[string]interface{}{"type": "Blueberry"}, map[string]interface{}{"type": "Devil's Food"}}}, "hacker": true, "beard": true, "hobbies": []interface{}{"skateboarding", "snowboarding", "go"}, "age": 35, "type": "donut", "newkey": "remote", "name": "Cake", "p_id": "0001", "p_ppu": "0.55", "p_name": "Cake", "p_batters": map[string]interface{}{"batter": map[string]interface{}{"type": "Regular"}}, "p_type": "donut", "foos": []map[string]interface{}{map[string]interface{}{"foo": []map[string]interface{}{map[string]interface{}{"key": 1}, map[string]interface{}{"key": 2}, map[string]interface{}{"key": 3}, map[string]interface{}{"key": 4}}}}} var allkeys sort.StringSlice allkeys = AllKeys() @@ -886,13 +886,14 @@ func TestMergeConfigNoMerge(t *testing.T) { } func TestUnmarshalingWithAliases(t *testing.T) { - SetDefault("ID", 1) - Set("name", "Steve") - Set("lastname", "Owen") + v := New() + v.SetDefault("ID", 1) + v.Set("name", "Steve") + v.Set("lastname", "Owen") - RegisterAlias("UserID", "ID") - RegisterAlias("Firstname", "name") - RegisterAlias("Surname", "lastname") + v.RegisterAlias("UserID", "ID") + v.RegisterAlias("Firstname", "name") + v.RegisterAlias("Surname", "lastname") type config struct { ID int @@ -901,8 +902,7 @@ func TestUnmarshalingWithAliases(t *testing.T) { } var C config - - err := Unmarshal(&C) + err := v.Unmarshal(&C) if err != nil { t.Fatalf("unable to decode into struct, %v", err) } @@ -925,6 +925,10 @@ func TestShadowedNestedValue(t *testing.T) { assert.Equal(t, "leather", GetString("clothing.jacket")) assert.Nil(t, Get("clothing.jacket.price")) assert.Equal(t, polyester, GetString("clothing.shirt")) + + clothingSettings := AllSettings()["clothing"].(map[string]interface{}) + assert.Equal(t, "leather", clothingSettings["jacket"]) + assert.Equal(t, polyester, clothingSettings["shirt"]) } func TestDotParameter(t *testing.T) {