From 7b841dc66d138aef0c17d42b719371d79a9f3b5d Mon Sep 17 00:00:00 2001 From: Kiril Zvezdarov Date: Mon, 23 Mar 2015 09:07:56 -0400 Subject: [PATCH 1/6] Added indexing of nested config values --- viper.go | 90 ++++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 81 insertions(+), 9 deletions(-) diff --git a/viper.go b/viper.go index 6c249fe..8960d16 100644 --- a/viper.go +++ b/viper.go @@ -38,6 +38,9 @@ import ( crypt "github.com/xordataexchange/crypt/config" ) +const INDEX_DELIM = "." +const ENV_INDEX_DELIM = "__" + var v *Viper func init() { @@ -121,7 +124,10 @@ type Viper struct { automaticEnvApplied bool envKeyReplacer *strings.Replacer + envToIndex *strings.Replacer + indexToEnv *strings.Replacer + index map[string]interface{} config map[string]interface{} override map[string]interface{} defaults map[string]interface{} @@ -135,6 +141,7 @@ type Viper struct { func New() *Viper { v := new(Viper) v.configName = "config" + v.index = make(map[string]interface{}) v.config = make(map[string]interface{}) v.override = make(map[string]interface{}) v.defaults = make(map[string]interface{}) @@ -149,6 +156,8 @@ func New() *Viper { } else { v.AddConfigPath(wd) } + v.indexToEnv = strings.NewReplacer(ENV_INDEX_DELIM, INDEX_DELIM) + v.envToIndex = strings.NewReplacer(INDEX_DELIM, ENV_INDEX_DELIM) return v } @@ -214,6 +223,8 @@ func (v *Viper) mergeWithEnvPrefix(in string) string { // key. This allows env vars which have different keys then the config object // keys func (v *Viper) getEnv(key string) string { + key = v.indexToEnv.Replace(key) + if v.envKeyReplacer != nil { key = v.envKeyReplacer.Replace(key) } @@ -317,12 +328,23 @@ func (v *Viper) providerPathExists(p *remoteProvider) bool { func Get(key string) interface{} { return v.Get(key) } func (v *Viper) Get(key string) interface{} { key = strings.ToLower(key) - val := v.find(key) + var val interface{} + v.buildIndex() + + if val = v.findIndex(key); val == nil { + val = v.find(key) + } if val == nil { return nil } + val = castVal(val) + + return val +} + +func castVal(val interface{}) interface{} { switch val.(type) { case bool: return cast.ToBool(val) @@ -339,6 +361,7 @@ func (v *Viper) Get(key string) interface{} { case []string: return val } + return val } @@ -483,7 +506,7 @@ func (v *Viper) BindEnv(input ...string) (err error) { return fmt.Errorf("BindEnv missing key to bind to") } - key = strings.ToLower(input[0]) + key = v.envToIndex.Replace(strings.ToLower(input[0])) if len(input) == 1 { envkey = v.mergeWithEnvPrefix(key) @@ -496,6 +519,18 @@ func (v *Viper) BindEnv(input ...string) (err error) { return nil } +func (v *Viper) findIndex(key string) interface{} { + // Check index - it is safe ot check it first + // as the paths there should be materialized + // according to their priority + val, exists := v.index[key] + if exists { + jww.TRACE.Println(key, "found in index:", val) + return val + } + return nil +} + // Given a key, find the value // Viper will check in the following order: // flag, env, config file, key/value store, default @@ -507,6 +542,12 @@ func (v *Viper) find(key string) interface{} { // if the requested key is an alias, then return the proper key key = v.realKey(key) + val, exists = v.override[key] + if exists { + jww.TRACE.Println(key, "found in override:", val) + return val + } + // PFlag Override first flag, exists := v.pflags[key] if exists { @@ -516,12 +557,6 @@ func (v *Viper) find(key string) interface{} { } } - val, exists = v.override[key] - if exists { - jww.TRACE.Println(key, "found in override:", val) - return val - } - if v.automaticEnvApplied { // even if it hasn't been registered, if automaticEnv is used, // check any Get request @@ -563,6 +598,29 @@ func (v *Viper) find(key string) interface{} { return nil } +func (v *Viper) buildIndex() { + v.index = make(map[string]interface{}) + v.indexMap(v.AllSettings(), "") +} + +func (v *Viper) indexMap(source map[string]interface{}, prefix string) { + if len(prefix) > 0 { + prefix = prefix + INDEX_DELIM + } + + for key, val := range source { + + indexPath := prefix + key + + v.index[indexPath] = val + + if reflect.TypeOf(val).Kind() == reflect.Map { + v.indexMap(cast.ToStringMap(val), indexPath) + } + } + +} + // Check to see if the key has been set in any of the data locations func IsSet(key string) bool { return v.IsSet(key) } func (v *Viper) IsSet(key string) bool { @@ -782,12 +840,24 @@ func (v *Viper) AllKeys() []string { return a } +func AllIndexes() []string { return v.AllIndexes() } +func (v *Viper) AllIndexes() []string { + v.buildIndex() + + var indexes []string + for key, _ := range v.index { + indexes = append(indexes, key) + } + + return indexes +} + // Return all settings as a map[string]interface{} func AllSettings() map[string]interface{} { return v.AllSettings() } func (v *Viper) AllSettings() map[string]interface{} { m := map[string]interface{}{} for _, x := range v.AllKeys() { - m[x] = v.Get(x) + m[x] = castVal(v.find(x)) } return m @@ -879,6 +949,8 @@ func (v *Viper) findConfigFile() (string, error) { // purposes. func Debug() { v.Debug() } func (v *Viper) Debug() { + fmt.Println("Index:") + pretty.Print(v.index) fmt.Println("Config:") pretty.Println(v.config) fmt.Println("Key/Value Store:") From 10c9322d34803aebc0f15550ad8425c54645029f Mon Sep 17 00:00:00 2001 From: Kiril Zvezdarov Date: Thu, 26 Mar 2015 17:51:40 -0400 Subject: [PATCH 2/6] Use the index as a fall back if the regular key is not found --- viper.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/viper.go b/viper.go index 8960d16..47c5ae7 100644 --- a/viper.go +++ b/viper.go @@ -329,10 +329,10 @@ func Get(key string) interface{} { return v.Get(key) } func (v *Viper) Get(key string) interface{} { key = strings.ToLower(key) var val interface{} - v.buildIndex() - if val = v.findIndex(key); val == nil { - val = v.find(key) + if val = v.find(key); val == nil { + v.buildIndex() + val = v.findIndex(key) } if val == nil { From 34ced34b4fc702bb39cd276cdd6d3676dff3db77 Mon Sep 17 00:00:00 2001 From: Kiril Zvezdarov Date: Fri, 27 Mar 2015 13:40:31 -0400 Subject: [PATCH 3/6] Insensitivise indexes on creation --- viper.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/viper.go b/viper.go index 47c5ae7..3758af2 100644 --- a/viper.go +++ b/viper.go @@ -610,7 +610,7 @@ func (v *Viper) indexMap(source map[string]interface{}, prefix string) { for key, val := range source { - indexPath := prefix + key + indexPath := strings.ToLower(prefix + key) v.index[indexPath] = val From b3a443f0b97e3df40bc6a3963d920959a37a5bcc Mon Sep 17 00:00:00 2001 From: Kiril Zvezdarov Date: Fri, 27 Mar 2015 13:40:58 -0400 Subject: [PATCH 4/6] Unit test coverage for index creation and retrieval --- viper_test.go | 178 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 178 insertions(+) diff --git a/viper_test.go b/viper_test.go index 4d049b6..cfc9d94 100644 --- a/viper_test.go +++ b/viper_test.go @@ -290,6 +290,184 @@ func TestAllKeys(t *testing.T) { assert.Equal(t, all, AllSettings()) } +func TestAllIndexes(t *testing.T) { + initConfigs() + expected := sort.StringSlice{"owner.organization", "name", "ppu", "clothing.trousers", "type", "batters.batter", "clothing.jacket", "hacker", "beard", "newkey", "owner.bio", "batters", "clothing", "owner", "eyes", "hobbies", "owner.dob", "title", "age", "id"} + expected.Sort() + + var actual sort.StringSlice + actual = AllIndexes() + actual.Sort() + + assert.Equal(t, expected, actual) +} + +func TestBuildsIndex(t *testing.T) { + initConfigs() + dob, _ := time.Parse(time.RFC3339, "1979-05-27T07:32:00Z") + + Set("super", map[string]interface{}{ + "deep": map[string]interface{}{ + "nested": "value", + }, + }) + + expected := map[string]interface{}{ + "super": map[string]interface{}{ + "deep": map[string]interface{}{ + "nested": "value", + }, + }, + "super.deep": map[string]interface{}{ + "nested": "value", + }, + "super.deep.nested": "value", + "owner.organization": "MongoDB", + "batters.batter": []interface{}{ + map[string]interface{}{ + "type": "Regular", + }, + map[string]interface{}{ + "type": "Chocolate", + }, + map[string]interface{}{ + "type": "Blueberry", + }, + map[string]interface{}{ + "type": "Devil's Food", + }, + }, + "hobbies": []interface{}{ + "skateboarding", "snowboarding", "go", + }, + "title": "TOML Example", + "newkey": "remote", + "batters": map[string]interface{}{ + "batter": []interface{}{ + map[string]interface{}{ + "type": "Regular", + }, + map[string]interface{}{ + "type": "Chocolate", + }, map[string]interface{}{ + "type": "Blueberry", + }, map[string]interface{}{ + "type": "Devil's Food", + }, + }, + }, + "eyes": "brown", + "age": 35, + "owner": map[string]interface{}{ + "organization": "MongoDB", + "Bio": "MongoDB Chief Developer Advocate & Hacker at Large", + "dob": dob, + }, + "owner.bio": "MongoDB Chief Developer Advocate & Hacker at Large", + "type": "donut", + "id": "0001", + "name": "Cake", + "hacker": true, + "ppu": 0.55, + "clothing": map[interface{}]interface{}{ + "jacket": "leather", + "trousers": "denim", + }, + "clothing.jacket": "leather", + "clothing.trousers": "denim", + "owner.dob": dob, + "beard": true, + } + + v.buildIndex() + + assert.Equal(t, expected, v.index) +} + +func TestRetrievesIndex(t *testing.T) { + initConfigs() + + dob, _ := time.Parse(time.RFC3339, "1979-05-27T07:32:00Z") + + Set("super", map[string]interface{}{ + "deep": map[string]interface{}{ + "nested": "value", + }, + }) + expected := map[string]interface{}{ + "super": map[string]interface{}{ + "deep": map[string]interface{}{ + "nested": "value", + }, + }, + "super.deep": map[string]interface{}{ + "nested": "value", + }, + "super.deep.nested": "value", + "owner.organization": "MongoDB", + "batters.batter": []interface{}{ + map[string]interface{}{ + "type": "Regular", + }, + map[string]interface{}{ + "type": "Chocolate", + }, + map[string]interface{}{ + "type": "Blueberry", + }, + map[string]interface{}{ + "type": "Devil's Food", + }, + }, + "hobbies": []interface{}{ + "skateboarding", "snowboarding", "go", + }, + "title": "TOML Example", + "newkey": "remote", + "batters": map[string]interface{}{ + "batter": []interface{}{ + map[string]interface{}{ + "type": "Regular", + }, + map[string]interface{}{ + "type": "Chocolate", + }, map[string]interface{}{ + "type": "Blueberry", + }, map[string]interface{}{ + "type": "Devil's Food", + }, + }, + }, + "eyes": "brown", + "age": 35, + "owner": map[string]interface{}{ + "organization": "MongoDB", + "Bio": "MongoDB Chief Developer Advocate & Hacker at Large", + "dob": dob, + }, + "owner.bio": "MongoDB Chief Developer Advocate & Hacker at Large", + "type": "donut", + "id": "0001", + "name": "Cake", + "hacker": true, + "ppu": 0.55, + "clothing": map[interface{}]interface{}{ + "jacket": "leather", + "trousers": "denim", + }, + "clothing.jacket": "leather", + "clothing.trousers": "denim", + "owner.dob": dob, + "beard": true, + } + + v.buildIndex() + + for key, value := range expected { + assert.Equal(t, value, v.Get(key)) + } +} + func TestCaseInSensitive(t *testing.T) { assert.Equal(t, true, Get("hacker")) Set("Title", "Checking Case") From 59bc322d1a342a5113e0a5333e1464982194ce47 Mon Sep 17 00:00:00 2001 From: Kiril Zvezdarov Date: Fri, 27 Mar 2015 17:03:12 -0400 Subject: [PATCH 5/6] Moved indexing function to utils and added docs --- README.md | 6 ++++++ util.go | 21 +++++++++++++++++++++ viper.go | 26 +++++++------------------- 3 files changed, 34 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 5918fb7..cbce88a 100644 --- a/README.md +++ b/README.md @@ -235,6 +235,12 @@ Example: fmt.Println("verbose enabled") } +You can get deeply nested values by providing the expected key path. + +Example: + + viper.Get("log.verbose") // case-insensitive path .....etc + ### Marshaling You also have the option of Marshaling all or a specific value to a struct, map, etc. diff --git a/util.go b/util.go index 2f974d8..5aaa117 100644 --- a/util.go +++ b/util.go @@ -17,6 +17,7 @@ import ( "io" "os" "path/filepath" + "reflect" "runtime" "strings" "unicode" @@ -184,3 +185,23 @@ func parseSizeInBytes(sizeStr string) uint { return safeMul(uint(size), multiplier) } + +// Recursively walks through the source map, populating the index with keys representing +// the nesting of the value and the value itself. +func indexMap(source map[string]interface{}, prefix string, index map[string]interface{}) { + if len(prefix) > 0 { + prefix = prefix + INDEX_DELIM + } + + for key, val := range source { + + indexPath := strings.ToLower(prefix + key) + + v.index[indexPath] = val + + if reflect.TypeOf(val).Kind() == reflect.Map { + indexMap(cast.ToStringMap(val), indexPath, index) + } + } + +} diff --git a/viper.go b/viper.go index 3758af2..c3fe683 100644 --- a/viper.go +++ b/viper.go @@ -598,27 +598,13 @@ func (v *Viper) find(key string) interface{} { return nil } +// Recursively walks through the structure returned by +// AllSettings, indexing deeply nested values. +// It uses AllSettings in order to get a properly +// prioritized config structure. func (v *Viper) buildIndex() { v.index = make(map[string]interface{}) - v.indexMap(v.AllSettings(), "") -} - -func (v *Viper) indexMap(source map[string]interface{}, prefix string) { - if len(prefix) > 0 { - prefix = prefix + INDEX_DELIM - } - - for key, val := range source { - - indexPath := strings.ToLower(prefix + key) - - v.index[indexPath] = val - - if reflect.TypeOf(val).Kind() == reflect.Map { - v.indexMap(cast.ToStringMap(val), indexPath) - } - } - + indexMap(v.AllSettings(), "", v.index) } // Check to see if the key has been set in any of the data locations @@ -840,6 +826,8 @@ func (v *Viper) AllKeys() []string { return a } +// Return all keys found in the index, including the deeply +// nested keys. func AllIndexes() []string { return v.AllIndexes() } func (v *Viper) AllIndexes() []string { v.buildIndex() From b477d087b3243463d4afb25c2862170d7a33eabc Mon Sep 17 00:00:00 2001 From: Kiril Zvezdarov Date: Mon, 6 Apr 2015 17:31:13 -0400 Subject: [PATCH 6/6] Fixed missing test preconditions --- viper_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/viper_test.go b/viper_test.go index cfc9d94..b0e50fa 100644 --- a/viper_test.go +++ b/viper_test.go @@ -512,6 +512,7 @@ func TestMarshal(t *testing.T) { } func TestBindPFlags(t *testing.T) { + Reset() flagSet := pflag.NewFlagSet("test", pflag.ContinueOnError) var testValues = map[string]*string{ @@ -568,6 +569,7 @@ func TestBindPFlag(t *testing.T) { } func TestBoundCaseSensitivity(t *testing.T) { + initConfigs() assert.Equal(t, "brown", Get("eyes"))