From 30ce444d046d1ba16b2d620223ab1189454cb8f5 Mon Sep 17 00:00:00 2001 From: ryanwalls Date: Sat, 30 Jan 2016 11:44:53 -0700 Subject: [PATCH 01/38] Update README with info about supporting Java properties files --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 8fdcd37..1c28ec9 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ to work within an application, and can handle all types of configuration needs and formats. It supports: * setting defaults -* reading from JSON, TOML, YAML and HCL config files +* reading from JSON, TOML, YAML, HCL, and Java properties config files * live watching and re-reading of config files (optional) * reading from environment variables * reading from remote config systems (etcd or Consul), and watching changes @@ -31,7 +31,7 @@ Viper is here to help with that. Viper does the following for you: -1. Find, load, and unmarshal a configuration file in JSON, TOML, YAML or HCL. +1. Find, load, and unmarshal a configuration file in JSON, TOML, YAML, HCL, or Java properties formats. 2. Provide a mechanism to set default values for your different configuration options. 3. Provide a mechanism to set override values for options specified through @@ -72,7 +72,7 @@ viper.SetDefault("Taxonomies", map[string]string{"tag": "tags", "category": "cat ### Reading Config Files Viper requires minimal configuration so it knows where to look for config files. -Viper supports JSON, TOML, YAML and HCL files. Viper can search multiple paths, but +Viper supports JSON, TOML, YAML, HCL, and Java Properties files. Viper can search multiple paths, but currently a single Viper instance only supports a single configuration file. Viper does not default to any configuration search paths leaving defaults decision to an application. @@ -343,7 +343,7 @@ how to use Consul. ```go viper.AddRemoteProvider("etcd", "http://127.0.0.1:4001","/config/hugo.json") -viper.SetConfigType("json") // because there is no file extension in a stream of bytes +viper.SetConfigType("json") // because there is no file extension in a stream of bytes, supported extensions are "json", "toml", "yaml", "yml", "properties", "props", "prop" err := viper.ReadRemoteConfig() ``` @@ -351,7 +351,7 @@ err := viper.ReadRemoteConfig() ```go viper.AddSecureRemoteProvider("etcd","http://127.0.0.1:4001","/config/hugo.json","/etc/secrets/mykeyring.gpg") -viper.SetConfigType("json") // because there is no file extension in a stream of bytes +viper.SetConfigType("json") // because there is no file extension in a stream of bytes, supported extensions are "json", "toml", "yaml", "yml", "properties", "props", "prop" err := viper.ReadRemoteConfig() ``` @@ -362,7 +362,7 @@ err := viper.ReadRemoteConfig() var runtime_viper = viper.New() runtime_viper.AddRemoteProvider("etcd", "http://127.0.0.1:4001", "/config/hugo.yml") -runtime_viper.SetConfigType("yaml") // because there is no file extension in a stream of bytes +runtime_viper.SetConfigType("yaml") // because there is no file extension in a stream of bytes, supported extensions are "json", "toml", "yaml", "yml", "properties", "props", "prop" // read from remote config the first time. err := runtime_viper.ReadRemoteConfig() From e072d51737b175ac83688b04999b509edc7908e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20Stanis=C5=82awski?= Date: Wed, 20 Jan 2016 22:15:43 +0100 Subject: [PATCH 02/38] use aliases in all keys list to enable proper Unmarshaling --- viper.go | 4 ++++ viper_test.go | 25 +++++++++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/viper.go b/viper.go index 00049ec..a056e61 100644 --- a/viper.go +++ b/viper.go @@ -1149,6 +1149,10 @@ func (v *Viper) AllKeys() []string { m[strings.ToLower(key)] = struct{}{} } + for key, _ := range v.aliases { + m[strings.ToLower(key)] = struct{}{} + } + a := []string{} for x, _ := range m { a = append(a, x) diff --git a/viper_test.go b/viper_test.go index fcb9c4e..ebde5ba 100644 --- a/viper_test.go +++ b/viper_test.go @@ -838,3 +838,28 @@ func TestMergeConfigNoMerge(t *testing.T) { t.Fatalf("fu != \"bar\", = %s", fu) } } + +func TestUnmarshalingWithAliases(t *testing.T) { + SetDefault("Id", 1) + Set("name", "Steve") + Set("lastname", "Owen") + + RegisterAlias("UserID","Id") + RegisterAlias("Firstname","name") + RegisterAlias("Surname","lastname") + + type config struct { + Id int + FirstName string + Surname string + } + + var C config + + err := Unmarshal(&C) + if err != nil { + t.Fatalf("unable to decode into struct, %v", err) + } + + assert.Equal(t, &C, &config{Id: 1, FirstName: "Steve", Surname: "Owen"}) +} \ No newline at end of file From dd66c894defa7532e594d9ada941bb3610558305 Mon Sep 17 00:00:00 2001 From: Steve Francia Date: Mon, 8 Feb 2016 17:01:43 -0500 Subject: [PATCH 03/38] Populate readme with viper based apps --- README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/README.md b/README.md index 1c28ec9..cad242d 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,16 @@ Go configuration with fangs! +Many Go projects are built using Viper including: + +* [Hugo](http://gohugo.io) +* [EMC RexRay](http://rexray.readthedocs.org/en/stable/) +* [Imgur's Incus](https://github.com/Imgur/incus) +* [Nanobox](https://github.com/nanobox-io/nanobox)/[Nanopack](https://github.com/nanopack) +* [Docker Notary](https://github.com/docker/Notary) +* [BloomApi](https://www.bloomapi.com/) +* [DOIt](https://github.com/bryanl/doit) + [![Build Status](https://travis-ci.org/spf13/viper.svg)](https://travis-ci.org/spf13/viper) [![Join the chat at https://gitter.im/spf13/viper](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/spf13/viper?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) From c975dc1b4eacf4ec7fdbf0873638de5d090ba323 Mon Sep 17 00:00:00 2001 From: dsp Date: Tue, 29 Dec 2015 23:11:39 -0700 Subject: [PATCH 04/38] implementing a weak decode wrapper called UnmarshalExact that errors on non existant fields in the destination struct --- viper.go | 31 +++++++++++++++++++++++++++++++ viper_test.go | 32 ++++++++++++++++++++++++++------ 2 files changed, 57 insertions(+), 6 deletions(-) diff --git a/viper.go b/viper.go index a056e61..7a49a0a 100644 --- a/viper.go +++ b/viper.go @@ -623,6 +623,37 @@ func (v *Viper) Unmarshal(rawVal interface{}) error { return nil } +// A wrapper around mapstructure.Decode that mimics the WeakDecode functionality +// while erroring on non existing vals in the destination struct +func weakDecodeExact(input, output interface{}) error { + config := &mapstructure.DecoderConfig{ + ErrorUnused: true, + Metadata: nil, + Result: output, + WeaklyTypedInput: true, + } + + decoder, err := mapstructure.NewDecoder(config) + if err != nil { + return err + } + return decoder.Decode(input) +} + +// Unmarshals the config into a Struct, erroring if a field is non-existant +// in the destination struct +func (v *Viper) UnmarshalExact(rawVal interface{}) error { + err := weakDecodeExact(v.AllSettings(), rawVal) + + if err != nil { + return err + } + + v.insensitiviseMaps() + + return nil +} + // Bind a full flag set to the configuration, using each flag's long // name as the config key. func BindPFlags(flags *pflag.FlagSet) (err error) { return v.BindPFlags(flags) } diff --git a/viper_test.go b/viper_test.go index ebde5ba..858caff 100644 --- a/viper_test.go +++ b/viper_test.go @@ -37,6 +37,14 @@ eyes : brown beard: true `) +var yamlExampleWithExtras = []byte(`Existing: true +Bogus: true +`) + +type testUnmarshalExtra struct { + Existing bool +} + var tomlExample = []byte(` title = "TOML Example" @@ -255,6 +263,18 @@ func TestUnmarshalling(t *testing.T) { assert.Equal(t, 35, Get("age")) } +func TestUnmarshalExact(t *testing.T) { + vip := New() + target := &testUnmarshalExtra{} + vip.SetConfigType("yaml") + r := bytes.NewReader(yamlExampleWithExtras) + vip.ReadConfig(r) + err := vip.UnmarshalExact(target) + if err == nil { + t.Fatal("UnmarshalExact should error when populating a struct from a conf that contains unused fields") + } +} + func TestOverrides(t *testing.T) { Set("age", 40) assert.Equal(t, 40, Get("age")) @@ -844,14 +864,14 @@ func TestUnmarshalingWithAliases(t *testing.T) { Set("name", "Steve") Set("lastname", "Owen") - RegisterAlias("UserID","Id") - RegisterAlias("Firstname","name") - RegisterAlias("Surname","lastname") + RegisterAlias("UserID", "Id") + RegisterAlias("Firstname", "name") + RegisterAlias("Surname", "lastname") type config struct { - Id int + Id int FirstName string - Surname string + Surname string } var C config @@ -862,4 +882,4 @@ func TestUnmarshalingWithAliases(t *testing.T) { } assert.Equal(t, &C, &config{Id: 1, FirstName: "Steve", Surname: "Owen"}) -} \ No newline at end of file +} From 45b73b72adfb3137795cd0d51df58de933fcae9b Mon Sep 17 00:00:00 2001 From: Tony Narlock Date: Mon, 7 Mar 2016 02:07:42 -0600 Subject: [PATCH 05/38] Fix typo --- viper.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/viper.go b/viper.go index 7a49a0a..70f8d03 100644 --- a/viper.go +++ b/viper.go @@ -540,7 +540,7 @@ func (v *Viper) GetString(key string) string { return cast.ToString(v.Get(key)) } -// Returns the value associated with the key asa boolean +// Returns the value associated with the key as a boolean func GetBool(key string) bool { return v.GetBool(key) } func (v *Viper) GetBool(key string) bool { return cast.ToBool(v.Get(key)) From 8e57fea7a87e3905de2f3279842c6c14564fe4e4 Mon Sep 17 00:00:00 2001 From: Anthony Fok Date: Wed, 20 Apr 2016 21:51:22 +0800 Subject: [PATCH 06/38] Update import path of fsnotify Rename "gopkg.in/fsnotify.v1" to "github.com/fsnotify/fsnotify" per upstream recommendation. See https://github.com/fsnotify/fsnotify/issues/108 for rationale. --- viper.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/viper.go b/viper.go index 70f8d03..df18448 100644 --- a/viper.go +++ b/viper.go @@ -31,12 +31,12 @@ import ( "strings" "time" + "github.com/fsnotify/fsnotify" "github.com/kr/pretty" "github.com/mitchellh/mapstructure" "github.com/spf13/cast" jww "github.com/spf13/jwalterweatherman" "github.com/spf13/pflag" - "gopkg.in/fsnotify.v1" ) var v *Viper From f904a1790545169368e8d37badd19d1895d94f29 Mon Sep 17 00:00:00 2001 From: Anthony Fok Date: Thu, 21 Apr 2016 06:40:14 +0800 Subject: [PATCH 07/38] Update Travis config to Go 1.4.3, 1.5.4 and 1.6.1 --- .travis.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9e6d621..96d8489 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,8 @@ language: go go: - - 1.3 - - release + - 1.4.3 + - 1.5.4 + - 1.6.1 - tip script: From a0cdbddebd2b4e614f360b2220f32055d1afe7dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Sun, 8 May 2016 13:12:02 +0200 Subject: [PATCH 08/38] Remove kr/pretty dependency See https://github.com/spf13/hugo/issues/2124 See https://github.com/kr/text/issues/6 --- viper.go | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/viper.go b/viper.go index df18448..a2633b0 100644 --- a/viper.go +++ b/viper.go @@ -32,7 +32,6 @@ import ( "time" "github.com/fsnotify/fsnotify" - "github.com/kr/pretty" "github.com/mitchellh/mapstructure" "github.com/spf13/cast" jww "github.com/spf13/jwalterweatherman" @@ -1284,17 +1283,11 @@ func (v *Viper) findConfigFile() (string, error) { func Debug() { v.Debug() } func (v *Viper) Debug() { fmt.Println("Aliases:") - pretty.Println(v.aliases) - fmt.Println("Override:") - pretty.Println(v.override) - fmt.Println("PFlags") - pretty.Println(v.pflags) - fmt.Println("Env:") - pretty.Println(v.env) - fmt.Println("Key/Value Store:") - pretty.Println(v.kvstore) - fmt.Println("Config:") - pretty.Println(v.config) - fmt.Println("Defaults:") - pretty.Println(v.defaults) + fmt.Printf("Aliases:\n%#v\n", v.aliases) + fmt.Printf("Override:\n%#v\n", v.override) + fmt.Printf("PFlags:\n%#v\n", v.pflags) + fmt.Printf("Env:\n%#v\n", v.env) + fmt.Printf("Key/Value Store:\n%#v\n", v.kvstore) + fmt.Printf("Config:\n%#v\n", v.config) + fmt.Printf("Defaults:\n%#v\n", v.defaults) } From 960e69f7c4cc9c9b67923647b59354868510cbc4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Sun, 8 May 2016 14:34:24 +0200 Subject: [PATCH 09/38] Allow tip to fail on Travis --- .travis.yml | 4 ++++ nohup.out | 1 + 2 files changed, 5 insertions(+) create mode 100644 nohup.out diff --git a/.travis.yml b/.travis.yml index 96d8489..5ebb6cc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,6 +5,10 @@ go: - 1.6.1 - tip +matrix: + allow_failures: + - go: tip + script: - go test -v ./... sudo: false diff --git a/nohup.out b/nohup.out new file mode 100644 index 0000000..8973bf2 --- /dev/null +++ b/nohup.out @@ -0,0 +1 @@ +QProcess::start: Process is already running From d8a428b8a30606e1d0b355d91edf282609ade1a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Sun, 8 May 2016 20:07:13 +0200 Subject: [PATCH 10/38] Bump Travis to 1.6.2 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 5ebb6cc..00b2b67 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,7 @@ language: go go: - 1.4.3 - 1.5.4 - - 1.6.1 + - 1.6.2 - tip matrix: From c1ccc378a054ea8d4e38d8c67f6938d4760b53dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Mon, 6 Jun 2016 00:03:07 +0200 Subject: [PATCH 11/38] Test on both Linux and OSX --- .travis.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index 00b2b67..4bb6878 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,10 @@ go: - 1.5.4 - 1.6.2 - tip + +os: + - linux + - osx matrix: allow_failures: From b53595fb56a492ecef90ee0457595a999eb6ec15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Mon, 18 Jul 2016 23:48:14 +0200 Subject: [PATCH 12/38] Bump Travis to Go 1.6.3 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 4bb6878..f27ab60 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,7 @@ language: go go: - 1.4.3 - 1.5.4 - - 1.6.2 + - 1.6.3 - tip os: From 7402856f06d639275f36058cd60fb1d8a10d2a52 Mon Sep 17 00:00:00 2001 From: Schley Andrew Kutz Date: Fri, 5 Aug 2016 02:15:24 -0500 Subject: [PATCH 13/38] Handle TOML Library Licensing This patch updates the package used for parsing TOML content from "github.com/BurntSushi/toml" to "github.com/pelletier/go-toml" as the latter uses a more accepted OSS license (MIT), enabling the inclusion of Viper or projects that depend on Viper in projects that have licensing requirements incongruent with the license of the previous TOML package. This patch replaces the PR https://github.com/spf13/viper/pull/208 after discussing the matter with @spf13 and deciding to update the TOML parser instead of making TOML build-optional. --- util.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/util.go b/util.go index 0cc4553..3c4a320 100644 --- a/util.go +++ b/util.go @@ -21,9 +21,9 @@ import ( "strings" "unicode" - "github.com/BurntSushi/toml" "github.com/hashicorp/hcl" "github.com/magiconair/properties" + toml "github.com/pelletier/go-toml" "github.com/spf13/cast" jww "github.com/spf13/jwalterweatherman" "gopkg.in/yaml.v2" @@ -155,9 +155,14 @@ func unmarshallConfigReader(in io.Reader, c map[string]interface{}, configType s } case "toml": - if _, err := toml.Decode(buf.String(), &c); err != nil { + tree, err := toml.LoadReader(buf) + if err != nil { return ConfigParseError{err} } + tmap := tree.ToMap() + for k, v := range tmap { + c[k] = v + } case "properties", "props", "prop": var p *properties.Properties From 6d2589cd85af2ffe1b01db0c6aebb5b68d9e87e4 Mon Sep 17 00:00:00 2001 From: Bryan Liles Date: Fri, 5 Aug 2016 03:15:51 -0400 Subject: [PATCH 14/38] Update Readme with doit's new location doit has been moved and renamed to doctl --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index cad242d..4ebd8dd 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ Many Go projects are built using Viper including: * [Nanobox](https://github.com/nanobox-io/nanobox)/[Nanopack](https://github.com/nanopack) * [Docker Notary](https://github.com/docker/Notary) * [BloomApi](https://www.bloomapi.com/) -* [DOIt](https://github.com/bryanl/doit) +* [doctl(https://github.com/digitalocean/doctl) [![Build Status](https://travis-ci.org/spf13/viper.svg)](https://travis-ci.org/spf13/viper) [![Join the chat at https://gitter.im/spf13/viper](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/spf13/viper?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) From 64dc6f681062822d3823f74c615e3f6c1f79a30c Mon Sep 17 00:00:00 2001 From: Chuanjian Wang Date: Fri, 5 Aug 2016 15:16:55 +0800 Subject: [PATCH 15/38] Add GetInt64 --- viper.go | 6 ++++++ viper_test.go | 18 ++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/viper.go b/viper.go index a2633b0..e5a360a 100644 --- a/viper.go +++ b/viper.go @@ -551,6 +551,12 @@ func (v *Viper) GetInt(key string) int { return cast.ToInt(v.Get(key)) } +// Returns the value associated with the key as an integer +func GetInt64(key string) int64 { return v.GetInt64(key) } +func (v *Viper) GetInt64(key string) int64 { + return cast.ToInt64(v.Get(key)) +} + // Returns the value associated with the key as a float64 func GetFloat64(key string) float64 { return v.GetFloat64(key) } func (v *Viper) GetFloat64(key string) float64 { diff --git a/viper_test.go b/viper_test.go index 858caff..aa9d7fe 100644 --- a/viper_test.go +++ b/viper_test.go @@ -763,6 +763,7 @@ func TestSub(t *testing.T) { var yamlMergeExampleTgt = []byte(` hello: pop: 37890 + lagrenum: 765432101234567 world: - us - uk @@ -773,6 +774,7 @@ hello: var yamlMergeExampleSrc = []byte(` hello: pop: 45000 + lagrenum: 7654321001234567 universe: - mw - ad @@ -790,6 +792,14 @@ func TestMergeConfig(t *testing.T) { t.Fatalf("pop != 37890, = %d", pop) } + if pop := v.GetInt("hello.lagrenum"); pop != 765432101234567 { + t.Fatalf("lagrenum != 765432101234567, = %d", pop) + } + + if pop := v.GetInt64("hello.lagrenum"); pop != int64(765432101234567) { + t.Fatalf("int64 lagrenum != 765432101234567, = %d", pop) + } + if world := v.GetStringSlice("hello.world"); len(world) != 4 { t.Fatalf("len(world) != 4, = %d", len(world)) } @@ -806,6 +816,14 @@ func TestMergeConfig(t *testing.T) { t.Fatalf("pop != 45000, = %d", pop) } + if pop := v.GetInt("hello.lagrenum"); pop != 7654321001234567 { + t.Fatalf("lagrenum != 7654321001234567, = %d", pop) + } + + if pop := v.GetInt64("hello.lagrenum"); pop != int64(7654321001234567) { + t.Fatalf("int64 lagrenum != 7654321001234567, = %d", pop) + } + if world := v.GetStringSlice("hello.world"); len(world) != 4 { t.Fatalf("len(world) != 4, = %d", len(world)) } From 5619c0edbec91139df79dd0d0d8d4967f4646a08 Mon Sep 17 00:00:00 2001 From: Roland Schilter Date: Fri, 5 Aug 2016 09:18:19 +0200 Subject: [PATCH 16/38] Reset cache on config name change I stumbled over this when trying to merge multiple configs. ``` viper.SetConfigName("default") err := viper.MergeInConfig() ``` which caches file path resolvemenet in `v.configFile` and then ``` viper.SetConfigName("prod") err := viper.MergeInConfig() ``` which reuses `v.configFile` without updating it accordingly to the new name. See https://github.com/spf13/viper/blob/c1ccc378a054ea8d4e38d8c67f6938d4760b53dd/viper.go#L1240 --- viper.go | 1 + viper_test.go | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/viper.go b/viper.go index e5a360a..d800395 100644 --- a/viper.go +++ b/viper.go @@ -1214,6 +1214,7 @@ func SetConfigName(in string) { v.SetConfigName(in) } func (v *Viper) SetConfigName(in string) { if in != "" { v.configName = in + v.configFile = "" } } diff --git a/viper_test.go b/viper_test.go index aa9d7fe..0c0c7e5 100644 --- a/viper_test.go +++ b/viper_test.go @@ -901,3 +901,9 @@ func TestUnmarshalingWithAliases(t *testing.T) { assert.Equal(t, &C, &config{Id: 1, FirstName: "Steve", Surname: "Owen"}) } + +func TestSetConfigNameClearsFileCache(t *testing.T) { + SetConfigFile("/tmp/config.yaml") + SetConfigName("default") + assert.Empty(t, v.getConfigFile()) +} From 4cf0bd27897685ffc5c5409b6a2f8ff5b04cf525 Mon Sep 17 00:00:00 2001 From: Matthieu Grieger Date: Fri, 5 Aug 2016 00:24:49 -0700 Subject: [PATCH 17/38] Add support for Afero filesystems --- util.go | 2 +- viper.go | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/util.go b/util.go index 3c4a320..6b34e14 100644 --- a/util.go +++ b/util.go @@ -77,7 +77,7 @@ func absPathify(inPath string) string { // Check if File / Directory Exists func exists(path string) (bool, error) { - _, err := os.Stat(path) + _, err := fs.Stat(path) if err == nil { return true, nil } diff --git a/viper.go b/viper.go index d800395..ba3c190 100644 --- a/viper.go +++ b/viper.go @@ -23,7 +23,6 @@ import ( "bytes" "fmt" "io" - "io/ioutil" "log" "os" "path/filepath" @@ -33,15 +32,18 @@ import ( "github.com/fsnotify/fsnotify" "github.com/mitchellh/mapstructure" + "github.com/spf13/afero" "github.com/spf13/cast" jww "github.com/spf13/jwalterweatherman" "github.com/spf13/pflag" ) var v *Viper +var fs afero.Fs func init() { v = New() + fs = afero.NewOsFs() } type remoteConfigFactory interface { @@ -936,7 +938,7 @@ func (v *Viper) ReadInConfig() error { return UnsupportedConfigError(v.getConfigType()) } - file, err := ioutil.ReadFile(v.getConfigFile()) + file, err := afero.ReadFile(fs, v.getConfigFile()) if err != nil { return err } @@ -954,7 +956,7 @@ func (v *Viper) MergeInConfig() error { return UnsupportedConfigError(v.getConfigType()) } - file, err := ioutil.ReadFile(v.getConfigFile()) + file, err := afero.ReadFile(fs, v.getConfigFile()) if err != nil { return err } From d0c2644870d873080734367010e57b96a0234259 Mon Sep 17 00:00:00 2001 From: Albert Date: Fri, 5 Aug 2016 12:25:24 +0500 Subject: [PATCH 18/38] Add dot in BindPFlag comment --- viper.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/viper.go b/viper.go index ba3c190..e39a564 100644 --- a/viper.go +++ b/viper.go @@ -668,8 +668,8 @@ func (v *Viper) BindPFlags(flags *pflag.FlagSet) (err error) { return v.BindFlagValues(pflagValueSet{flags}) } -// Bind a specific key to a pflag (as used by cobra) -// Example(where serverCmd is a Cobra instance): +// Bind a specific key to a pflag (as used by cobra). +// Example (where serverCmd is a Cobra instance): // // serverCmd.Flags().Int("port", 1138, "Port to run Application server on") // Viper.BindPFlag("port", serverCmd.Flags().Lookup("port")) From a59dcccc828173bb1f58d1650619724b59d18af5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Fri, 5 Aug 2016 09:45:58 +0200 Subject: [PATCH 19/38] Move the Afero fs to the Viper type And make a exported setter for it. --- util.go | 2 +- viper.go | 16 ++++++++++++---- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/util.go b/util.go index 6b34e14..fe6cb45 100644 --- a/util.go +++ b/util.go @@ -77,7 +77,7 @@ func absPathify(inPath string) string { // Check if File / Directory Exists func exists(path string) (bool, error) { - _, err := fs.Stat(path) + _, err := v.fs.Stat(path) if err == nil { return true, nil } diff --git a/viper.go b/viper.go index e39a564..cdf2113 100644 --- a/viper.go +++ b/viper.go @@ -39,11 +39,9 @@ import ( ) var v *Viper -var fs afero.Fs func init() { v = New() - fs = afero.NewOsFs() } type remoteConfigFactory interface { @@ -134,6 +132,9 @@ type Viper struct { // A set of paths to look for the config file in configPaths []string + // The filesystem to read config from. + fs afero.Fs + // A set of remote providers to search for the configuration remoteProviders []*defaultRemoteProvider @@ -163,6 +164,7 @@ func New() *Viper { v := new(Viper) v.keyDelim = "." v.configName = "config" + v.fs = afero.NewOsFs() v.config = make(map[string]interface{}) v.override = make(map[string]interface{}) v.defaults = make(map[string]interface{}) @@ -938,7 +940,7 @@ func (v *Viper) ReadInConfig() error { return UnsupportedConfigError(v.getConfigType()) } - file, err := afero.ReadFile(fs, v.getConfigFile()) + file, err := afero.ReadFile(v.fs, v.getConfigFile()) if err != nil { return err } @@ -956,7 +958,7 @@ func (v *Viper) MergeInConfig() error { return UnsupportedConfigError(v.getConfigType()) } - file, err := afero.ReadFile(fs, v.getConfigFile()) + file, err := afero.ReadFile(v.fs, v.getConfigFile()) if err != nil { return err } @@ -1210,6 +1212,12 @@ func (v *Viper) AllSettings() map[string]interface{} { return m } +// Se the filesystem to use to read configuration. +func SetFs(fs afero.Fs) { v.SetFs(fs) } +func (v *Viper) SetFs(fs afero.Fs) { + v.fs = fs +} + // Name for the config file. // Does not include extension. func SetConfigName(in string) { v.SetConfigName(in) } From abafbf243b9583ac1360ae4ef680d8acaa361089 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Fri, 5 Aug 2016 11:20:27 +0200 Subject: [PATCH 20/38] Revert "Handle TOML Library Licensing" It breaks Hugo, will have to test/investigate. Erro when building Hugo docs: ``` ERROR: 2016/08/05 11:19:23 site.go:1208: unable to process menus in site config ERROR: 2016/08/05 11:19:23 site.go:1209: Unable to Cast map[string]interface {}{"pre":"", "weight":-20, "url":"/commands/", "name":"Hugo Cmd Reference", "identifier":"commands"} of type map[string]interface {} to []interface{} ``` This reverts commit 7402856f06d639275f36058cd60fb1d8a10d2a52. --- util.go | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/util.go b/util.go index fe6cb45..46f923c 100644 --- a/util.go +++ b/util.go @@ -21,9 +21,9 @@ import ( "strings" "unicode" + "github.com/BurntSushi/toml" "github.com/hashicorp/hcl" "github.com/magiconair/properties" - toml "github.com/pelletier/go-toml" "github.com/spf13/cast" jww "github.com/spf13/jwalterweatherman" "gopkg.in/yaml.v2" @@ -155,14 +155,9 @@ func unmarshallConfigReader(in io.Reader, c map[string]interface{}, configType s } case "toml": - tree, err := toml.LoadReader(buf) - if err != nil { + if _, err := toml.Decode(buf.String(), &c); err != nil { return ConfigParseError{err} } - tmap := tree.ToMap() - for k, v := range tmap { - c[k] = v - } case "properties", "props", "prop": var p *properties.Properties From 346299ea79e446ebdddb834371ceba2e5926b732 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Sat, 6 Aug 2016 18:06:49 +0200 Subject: [PATCH 21/38] Add getter for global Viper --- viper.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/viper.go b/viper.go index cdf2113..f17790e 100644 --- a/viper.go +++ b/viper.go @@ -450,6 +450,11 @@ func (v *Viper) SetTypeByDefaultValue(enable bool) { v.typeByDefValue = enable } +// GetViper gets the global Viper instance. +func GetViper() *Viper { + return v +} + // Viper is essentially repository for configurations // Get can retrieve any value given the key to use // Get has the behavior of returning the value associated with the first From 654fc7bb54d0c138ef80405ff577391f79c0c32d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Tue, 16 Aug 2016 10:09:34 +0200 Subject: [PATCH 22/38] Bump Travis to Go 1.7 --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index f27ab60..46d5247 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,8 +1,8 @@ language: go go: - - 1.4.3 - 1.5.4 - 1.6.3 + - 1.7 - tip os: @@ -12,6 +12,7 @@ os: matrix: allow_failures: - go: tip + fast_finish: true script: - go test -v ./... From fe9c8b59e176537a2a08498fe6a5e131a3b6a304 Mon Sep 17 00:00:00 2001 From: Schley Andrew Kutz Date: Sat, 20 Aug 2016 08:03:44 -0500 Subject: [PATCH 23/38] Add Hugo CI Step to Validate Viper Since Hugo is such a heavy user of Viper, this patch adds an after_success section to the Travis-CI build that validates the Viper commit by attempting to build Hugo and executing `hugo -s docs`. This patch handles issue #222. --- .travis.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 46d5247..e793edb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,10 +1,12 @@ +go_import_path: github.com/spf13/viper + language: go go: - 1.5.4 - 1.6.3 - 1.7 - tip - + os: - linux - osx @@ -15,5 +17,11 @@ matrix: fast_finish: true script: + - go install ./... - go test -v ./... + +after_success: + - go get -u -d github.com/spf13/hugo + - cd $GOPATH/src/github.com/spf13/hugo && make && ./hugo -s docs && cd - + sudo: false From 7fb2782df3d83e0036cc89f461ed0422628776f4 Mon Sep 17 00:00:00 2001 From: Vladimir Vivien Date: Fri, 12 Aug 2016 14:14:35 -0400 Subject: [PATCH 24/38] Handle TOML Library Licensing This patch updates the package used for parsing TOML content from "github.com/BurntSushi/toml" to "github.com/pelletier/go-toml" as the latter uses a more accepted OSS license (MIT), enabling the inclusion of Viper or projects that depend on Viper in projects that have licensing requirements incongruent with the license of the previous TOML package. Closes #228 Closes #225 Fixes #179 --- util.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/util.go b/util.go index 46f923c..fe6cb45 100644 --- a/util.go +++ b/util.go @@ -21,9 +21,9 @@ import ( "strings" "unicode" - "github.com/BurntSushi/toml" "github.com/hashicorp/hcl" "github.com/magiconair/properties" + toml "github.com/pelletier/go-toml" "github.com/spf13/cast" jww "github.com/spf13/jwalterweatherman" "gopkg.in/yaml.v2" @@ -155,9 +155,14 @@ func unmarshallConfigReader(in io.Reader, c map[string]interface{}, configType s } case "toml": - if _, err := toml.Decode(buf.String(), &c); err != nil { + tree, err := toml.LoadReader(buf) + if err != nil { return ConfigParseError{err} } + tmap := tree.ToMap() + for k, v := range tmap { + c[k] = v + } case "properties", "props", "prop": var p *properties.Properties From 16990631d4aa7e38f73dbbbf37fa13e67c648531 Mon Sep 17 00:00:00 2001 From: Albert Date: Tue, 30 Aug 2016 19:32:46 +0500 Subject: [PATCH 25/38] Fix typo in README (#227) Fixed typo & superfluous white space in README --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 4ebd8dd..cf17560 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ Many Go projects are built using Viper including: * [Nanobox](https://github.com/nanobox-io/nanobox)/[Nanopack](https://github.com/nanopack) * [Docker Notary](https://github.com/docker/Notary) * [BloomApi](https://www.bloomapi.com/) -* [doctl(https://github.com/digitalocean/doctl) +* [doctl](https://github.com/digitalocean/doctl) [![Build Status](https://travis-ci.org/spf13/viper.svg)](https://travis-ci.org/spf13/viper) [![Join the chat at https://gitter.im/spf13/viper](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/spf13/viper?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) @@ -110,7 +110,7 @@ Gone are the days of needing to restart a server to have a config take effect, viper powered applications can read an update to a config file while running and not miss a beat. -Simply tell the viper instance to watchConfig. +Simply tell the viper instance to watchConfig. Optionally you can provide a function for Viper to run each time a change occurs. **Make sure you add all of the configPaths prior to calling `WatchConfig()`** @@ -298,7 +298,7 @@ type myFlagSet struct { func (f myFlagSet) VisitAll(fn func(FlagValue)) { for _, flag := range flags { - fn(flag) + fn(flag) } } ``` From 2f6a41490bc86fa9a8fb2fc03526a5795e904a17 Mon Sep 17 00:00:00 2001 From: Max Wolter Date: Mon, 19 Sep 2016 19:37:56 +0200 Subject: [PATCH 26/38] Prevent shadowning of keys when a nested key's value is nil. --- viper.go | 6 ++++-- viper_test.go | 9 +++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/viper.go b/viper.go index f17790e..bad121e 100644 --- a/viper.go +++ b/viper.go @@ -802,8 +802,10 @@ func (v *Viper) find(key string) interface{} { if source != nil { if reflect.TypeOf(source).Kind() == reflect.Map { val := v.searchMap(cast.ToStringMap(source), path[1:]) - jww.TRACE.Println(key, "found in nested config:", val) - return val + if val != nil { + jww.TRACE.Println(key, "found in nested config:", val) + return val + } } } } diff --git a/viper_test.go b/viper_test.go index 0c0c7e5..fdbe4d9 100644 --- a/viper_test.go +++ b/viper_test.go @@ -907,3 +907,12 @@ func TestSetConfigNameClearsFileCache(t *testing.T) { SetConfigName("default") assert.Empty(t, v.getConfigFile()) } + +func TestShadowedNestedValue(t *testing.T) { + polyester := "polyester" + initYAML() + SetDefault("clothing.shirt", polyester) + + assert.Equal(t, GetString("clothing.jacket"), "leather") + assert.Equal(t, GetString("clothing.shirt"), polyester) +} From a78f70b5b977efe08e313a9e2341c3f5457abdaf Mon Sep 17 00:00:00 2001 From: Albert Date: Tue, 20 Sep 2016 10:17:41 +0200 Subject: [PATCH 27/38] Small refactorings (#230) * Fix typo in description of UnmarshalExact * Omit 2nd values from range loops * Delete findCWD method from util (was unused) * Edit documentation according to golint * Fix documentation in util * Use RemoteProvider interface instead of defaultRemoteProvider * Fix err variable in BindFlagValues --- flags_test.go | 2 +- util.go | 33 ++------ viper.go | 205 +++++++++++++++++++++++--------------------------- viper_test.go | 2 +- 4 files changed, 100 insertions(+), 142 deletions(-) diff --git a/flags_test.go b/flags_test.go index 5489278..5bffca3 100644 --- a/flags_test.go +++ b/flags_test.go @@ -22,7 +22,7 @@ func TestBindFlagValueSet(t *testing.T) { "endpoint": "/public", } - for name, _ := range testValues { + for name := range testValues { testValues[name] = flagSet.String(name, "", "test") } diff --git a/util.go b/util.go index fe6cb45..5f93d65 100644 --- a/util.go +++ b/util.go @@ -29,12 +29,12 @@ import ( "gopkg.in/yaml.v2" ) -// Denotes failing to parse configuration file. +// ConfigParseError denotes failing to parse configuration file. type ConfigParseError struct { err error } -// Returns the formatted configuration error. +// Error returns the formatted configuration error. func (pe ConfigParseError) Error() string { return fmt.Sprintf("While parsing config: %s", pe.err.Error()) } @@ -68,10 +68,10 @@ func absPathify(inPath string) string { p, err := filepath.Abs(inPath) if err == nil { return filepath.Clean(p) - } else { - jww.ERROR.Println("Couldn't discover absolute path") - jww.ERROR.Println(err) } + + jww.ERROR.Println("Couldn't discover absolute path") + jww.ERROR.Println(err) return "" } @@ -107,29 +107,6 @@ func userHomeDir() string { return os.Getenv("HOME") } -func findCWD() (string, error) { - serverFile, err := filepath.Abs(os.Args[0]) - - if err != nil { - return "", fmt.Errorf("Can't get absolute path for executable: %v", err) - } - - path := filepath.Dir(serverFile) - realFile, err := filepath.EvalSymlinks(serverFile) - - if err != nil { - if _, err = os.Stat(serverFile + ".exe"); err == nil { - realFile = filepath.Clean(serverFile + ".exe") - } - } - - if err == nil && realFile != serverFile { - path = filepath.Dir(realFile) - } - - return path, nil -} - func unmarshallConfigReader(in io.Reader, c map[string]interface{}, configType string) error { buf := new(bytes.Buffer) buf.ReadFrom(in) diff --git a/viper.go b/viper.go index bad121e..bc1bf5e 100644 --- a/viper.go +++ b/viper.go @@ -52,40 +52,40 @@ type remoteConfigFactory interface { // RemoteConfig is optional, see the remote package var RemoteConfig remoteConfigFactory -// Denotes encountering an unsupported +// UnsupportedConfigError denotes encountering an unsupported // configuration filetype. type UnsupportedConfigError string -// Returns the formatted configuration error. +// Error returns the formatted configuration error. func (str UnsupportedConfigError) Error() string { return fmt.Sprintf("Unsupported Config Type %q", string(str)) } -// Denotes encountering an unsupported remote +// UnsupportedRemoteProviderError denotes encountering an unsupported remote // provider. Currently only etcd and Consul are // supported. type UnsupportedRemoteProviderError string -// Returns the formatted remote provider error. +// Error returns the formatted remote provider error. func (str UnsupportedRemoteProviderError) Error() string { return fmt.Sprintf("Unsupported Remote Provider Type %q", string(str)) } -// Denotes encountering an error while trying to +// RemoteConfigError denotes encountering an error while trying to // pull the configuration from the remote provider. type RemoteConfigError string -// Returns the formatted remote provider error +// Error returns the formatted remote provider error func (rce RemoteConfigError) Error() string { return fmt.Sprintf("Remote Configurations Error: %s", string(rce)) } -// Denotes failing to find configuration file. +// ConfigFileNotFoundError denotes failing to find configuration file. type ConfigFileNotFoundError struct { name, locations string } -// Returns the formatted configuration error. +// Error returns the formatted configuration error. func (fnfe ConfigFileNotFoundError) Error() string { return fmt.Sprintf("Config File %q Not Found in %q", fnfe.name, fnfe.locations) } @@ -159,7 +159,7 @@ type Viper struct { onConfigChange func(fsnotify.Event) } -// Returns an initialized Viper instance. +// New returns an initialized Viper instance. func New() *Viper { v := new(Viper) v.keyDelim = "." @@ -220,11 +220,11 @@ type RemoteProvider interface { SecretKeyring() string } -// Universally supported extensions. -var SupportedExts []string = []string{"json", "toml", "yaml", "yml", "properties", "props", "prop", "hcl"} +// SupportedExts are universally supported extensions. +var SupportedExts = []string{"json", "toml", "yaml", "yml", "properties", "props", "prop", "hcl"} -// Universally supported remote providers. -var SupportedRemoteProviders []string = []string{"etcd", "consul"} +// SupportedRemoteProviders are universally supported remote providers. +var SupportedRemoteProviders = []string{"etcd", "consul"} func OnConfigChange(run func(in fsnotify.Event)) { v.OnConfigChange(run) } func (v *Viper) OnConfigChange(run func(in fsnotify.Event)) { @@ -270,7 +270,7 @@ func (v *Viper) WatchConfig() { }() } -// Explicitly define the path, name and extension of the config file +// SetConfigFile explicitly defines the path, name and extension of the config file // Viper will use this and not check any of the config paths func SetConfigFile(in string) { v.SetConfigFile(in) } func (v *Viper) SetConfigFile(in string) { @@ -279,7 +279,7 @@ func (v *Viper) SetConfigFile(in string) { } } -// Define a prefix that ENVIRONMENT variables will use. +// SetEnvPrefix defines a prefix that ENVIRONMENT variables will use. // E.g. if your prefix is "spf", the env registry // will look for env. variables that start with "SPF_" func SetEnvPrefix(in string) { v.SetEnvPrefix(in) } @@ -301,7 +301,7 @@ func (v *Viper) mergeWithEnvPrefix(in string) string { // rewriting keys many things, Ex: Get('someKey') -> some_key // (cammel case to snake case for JSON keys perhaps) -// getEnv s a wrapper around os.Getenv which replaces characters in the original +// getEnv is a wrapper around os.Getenv which replaces characters in the original // key. This allows env vars which have different keys then the config object // keys func (v *Viper) getEnv(key string) string { @@ -311,11 +311,11 @@ func (v *Viper) getEnv(key string) string { return os.Getenv(key) } -// Return the file used to populate the config registry +// ConfigFileUsed returns the file used to populate the config registry func ConfigFileUsed() string { return v.ConfigFileUsed() } func (v *Viper) ConfigFileUsed() string { return v.configFile } -// Add a path for Viper to search for the config file in. +// AddConfigPath adds a path for Viper to search for the config file in. // Can be called multiple times to define multiple search paths. func AddConfigPath(in string) { v.AddConfigPath(in) } func (v *Viper) AddConfigPath(in string) { @@ -455,8 +455,7 @@ func GetViper() *Viper { return v } -// Viper is essentially repository for configurations -// Get can retrieve any value given the key to use +// Get can retrieve any value given the key to use. // Get has the behavior of returning the value associated with the first // place from where it is set. Viper will check in the following order: // override, flag, env, config file, key/value store, default @@ -529,7 +528,7 @@ func (v *Viper) Get(key string) interface{} { return val } -// Returns new Viper instance representing a sub tree of this instance +// Sub returns new Viper instance representing a sub tree of this instance. func Sub(key string) *Viper { return v.Sub(key) } func (v *Viper) Sub(key string) *Viper { subv := New() @@ -537,78 +536,77 @@ func (v *Viper) Sub(key string) *Viper { if reflect.TypeOf(data).Kind() == reflect.Map { subv.config = cast.ToStringMap(data) return subv - } else { - return nil } + return nil } -// Returns the value associated with the key as a string +// GetString returns the value associated with the key as a string. func GetString(key string) string { return v.GetString(key) } func (v *Viper) GetString(key string) string { return cast.ToString(v.Get(key)) } -// Returns the value associated with the key as a boolean +// GetBool returns the value associated with the key as a boolean. func GetBool(key string) bool { return v.GetBool(key) } func (v *Viper) GetBool(key string) bool { return cast.ToBool(v.Get(key)) } -// Returns the value associated with the key as an integer +// GetInt returns the value associated with the key as an integer. func GetInt(key string) int { return v.GetInt(key) } func (v *Viper) GetInt(key string) int { return cast.ToInt(v.Get(key)) } -// Returns the value associated with the key as an integer +// GetInt64 returns the value associated with the key as an integer. func GetInt64(key string) int64 { return v.GetInt64(key) } func (v *Viper) GetInt64(key string) int64 { return cast.ToInt64(v.Get(key)) } -// Returns the value associated with the key as a float64 +// GetFloat64 returns the value associated with the key as a float64. func GetFloat64(key string) float64 { return v.GetFloat64(key) } func (v *Viper) GetFloat64(key string) float64 { return cast.ToFloat64(v.Get(key)) } -// Returns the value associated with the key as time +// GetTime returns the value associated with the key as time. func GetTime(key string) time.Time { return v.GetTime(key) } func (v *Viper) GetTime(key string) time.Time { return cast.ToTime(v.Get(key)) } -// Returns the value associated with the key as a duration +// GetDuration returns the value associated with the key as a duration. func GetDuration(key string) time.Duration { return v.GetDuration(key) } func (v *Viper) GetDuration(key string) time.Duration { return cast.ToDuration(v.Get(key)) } -// Returns the value associated with the key as a slice of strings +// GetStringSlice returns the value associated with the key as a slice of strings. func GetStringSlice(key string) []string { return v.GetStringSlice(key) } func (v *Viper) GetStringSlice(key string) []string { return cast.ToStringSlice(v.Get(key)) } -// Returns the value associated with the key as a map of interfaces +// GetStringMap returns the value associated with the key as a map of interfaces. func GetStringMap(key string) map[string]interface{} { return v.GetStringMap(key) } func (v *Viper) GetStringMap(key string) map[string]interface{} { return cast.ToStringMap(v.Get(key)) } -// Returns the value associated with the key as a map of strings +// GetStringMapString returns the value associated with the key as a map of strings. func GetStringMapString(key string) map[string]string { return v.GetStringMapString(key) } func (v *Viper) GetStringMapString(key string) map[string]string { return cast.ToStringMapString(v.Get(key)) } -// Returns the value associated with the key as a map to a slice of strings. +// GetStringMapStringSlice returns the value associated with the key as a map to a slice of strings. func GetStringMapStringSlice(key string) map[string][]string { return v.GetStringMapStringSlice(key) } func (v *Viper) GetStringMapStringSlice(key string) map[string][]string { return cast.ToStringMapStringSlice(v.Get(key)) } -// Returns the size of the value associated with the given key +// GetSizeInBytes returns the size of the value associated with the given key // in bytes. func GetSizeInBytes(key string) uint { return v.GetSizeInBytes(key) } func (v *Viper) GetSizeInBytes(key string) uint { @@ -616,13 +614,13 @@ func (v *Viper) GetSizeInBytes(key string) uint { return parseSizeInBytes(sizeStr) } -// Takes a single key and unmarshals it into a Struct +// UnmarshalKey takes a single key and unmarshals it into a Struct. func UnmarshalKey(key string, rawVal interface{}) error { return v.UnmarshalKey(key, rawVal) } func (v *Viper) UnmarshalKey(key string, rawVal interface{}) error { return mapstructure.Decode(v.Get(key), rawVal) } -// Unmarshals the config into a Struct. Make sure that the tags +// Unmarshal unmarshals the config into a Struct. Make sure that the tags // on the fields of the structure are properly set. func Unmarshal(rawVal interface{}) error { return v.Unmarshal(rawVal) } func (v *Viper) Unmarshal(rawVal interface{}) error { @@ -638,7 +636,7 @@ func (v *Viper) Unmarshal(rawVal interface{}) error { } // A wrapper around mapstructure.Decode that mimics the WeakDecode functionality -// while erroring on non existing vals in the destination struct +// while erroring on non existing vals in the destination struct. func weakDecodeExact(input, output interface{}) error { config := &mapstructure.DecoderConfig{ ErrorUnused: true, @@ -654,8 +652,8 @@ func weakDecodeExact(input, output interface{}) error { return decoder.Decode(input) } -// Unmarshals the config into a Struct, erroring if a field is non-existant -// in the destination struct +// UnmarshalExact unmarshals the config into a Struct, erroring if a field is nonexistent +// in the destination struct. func (v *Viper) UnmarshalExact(rawVal interface{}) error { err := weakDecodeExact(v.AllSettings(), rawVal) @@ -668,27 +666,27 @@ func (v *Viper) UnmarshalExact(rawVal interface{}) error { return nil } -// Bind a full flag set to the configuration, using each flag's long +// BindPFlags binds a full flag set to the configuration, using each flag's long // name as the config key. -func BindPFlags(flags *pflag.FlagSet) (err error) { return v.BindPFlags(flags) } -func (v *Viper) BindPFlags(flags *pflag.FlagSet) (err error) { +func BindPFlags(flags *pflag.FlagSet) error { return v.BindPFlags(flags) } +func (v *Viper) BindPFlags(flags *pflag.FlagSet) error { return v.BindFlagValues(pflagValueSet{flags}) } -// Bind a specific key to a pflag (as used by cobra). +// BindPFlag binds a specific key to a pflag (as used by cobra). // Example (where serverCmd is a Cobra instance): // // serverCmd.Flags().Int("port", 1138, "Port to run Application server on") // Viper.BindPFlag("port", serverCmd.Flags().Lookup("port")) // -func BindPFlag(key string, flag *pflag.Flag) (err error) { return v.BindPFlag(key, flag) } -func (v *Viper) BindPFlag(key string, flag *pflag.Flag) (err error) { +func BindPFlag(key string, flag *pflag.Flag) error { return v.BindPFlag(key, flag) } +func (v *Viper) BindPFlag(key string, flag *pflag.Flag) error { return v.BindFlagValue(key, pflagValue{flag}) } -// Bind a full FlagValue set to the configuration, using each flag's long +// BindFlagValues binds a full FlagValue set to the configuration, using each flag's long // name as the config key. -func BindFlagValues(flags FlagValueSet) (err error) { return v.BindFlagValues(flags) } +func BindFlagValues(flags FlagValueSet) error { return v.BindFlagValues(flags) } func (v *Viper) BindFlagValues(flags FlagValueSet) (err error) { flags.VisitAll(func(flag FlagValue) { if err = v.BindFlagValue(flag.Name(), flag); err != nil { @@ -698,14 +696,14 @@ func (v *Viper) BindFlagValues(flags FlagValueSet) (err error) { return nil } -// Bind a specific key to a FlagValue. +// BindFlagValue binds a specific key to a FlagValue. // Example(where serverCmd is a Cobra instance): // // serverCmd.Flags().Int("port", 1138, "Port to run Application server on") // Viper.BindFlagValue("port", serverCmd.Flags().Lookup("port")) // -func BindFlagValue(key string, flag FlagValue) (err error) { return v.BindFlagValue(key, flag) } -func (v *Viper) BindFlagValue(key string, flag FlagValue) (err error) { +func BindFlagValue(key string, flag FlagValue) error { return v.BindFlagValue(key, flag) } +func (v *Viper) BindFlagValue(key string, flag FlagValue) error { if flag == nil { return fmt.Errorf("flag for %q is nil", key) } @@ -713,12 +711,12 @@ func (v *Viper) BindFlagValue(key string, flag FlagValue) (err error) { return nil } -// Binds a Viper key to a ENV variable -// ENV variables are case sensitive +// BindEnv binds a Viper key to a ENV variable. +// ENV variables are case sensitive. // If only a key is provided, it will use the env key matching the key, uppercased. // EnvPrefix will be used when set when env name is not provided. -func BindEnv(input ...string) (err error) { return v.BindEnv(input...) } -func (v *Viper) BindEnv(input ...string) (err error) { +func BindEnv(input ...string) error { return v.BindEnv(input...) } +func (v *Viper) BindEnv(input ...string) error { var key, envkey string if len(input) == 0 { return fmt.Errorf("BindEnv missing key to bind to") @@ -737,10 +735,10 @@ func (v *Viper) BindEnv(input ...string) (err error) { return nil } -// Given a key, find the value +// Given a key, find the value. // Viper will check in the following order: -// flag, env, config file, key/value store, default -// Viper will check to see if an alias exists first +// flag, env, config file, key/value store, default. +// Viper will check to see if an alias exists first. func (v *Viper) find(key string) interface{} { var val interface{} var exists bool @@ -783,9 +781,8 @@ func (v *Viper) find(key string) interface{} { if val = v.getEnv(envkey); val != "" { jww.TRACE.Println(envkey, "found in environment with val:", val) return val - } else { - jww.TRACE.Println(envkey, "env value unset:") } + jww.TRACE.Println(envkey, "env value unset:") } val, exists = v.config[key] @@ -825,7 +822,7 @@ func (v *Viper) find(key string) interface{} { return nil } -// Check to see if the key has been set in any of the data locations +// IsSet checks to see if the key has been set in any of the data locations. func IsSet(key string) bool { return v.IsSet(key) } func (v *Viper) IsSet(key string) bool { path := strings.Split(key, v.keyDelim) @@ -845,7 +842,7 @@ func (v *Viper) IsSet(key string) bool { return val != nil } -// Have Viper check ENV variables for all +// AutomaticEnv has Viper check ENV variables for all. // keys set in config, default & flags func AutomaticEnv() { v.AutomaticEnv() } func (v *Viper) AutomaticEnv() { @@ -904,12 +901,11 @@ func (v *Viper) realKey(key string) string { if exists { jww.DEBUG.Println("Alias", key, "to", newkey) return v.realKey(newkey) - } else { - return key } + return key } -// Check to see if the given key (or an alias) is in the config file +// InConfig checks to see if the given key (or an alias) is in the config file. func InConfig(key string) bool { return v.InConfig(key) } func (v *Viper) InConfig(key string) bool { // if the requested key is an alias, then return the proper key @@ -919,7 +915,7 @@ func (v *Viper) InConfig(key string) bool { return exists } -// Set the default value for this key. +// SetDefault sets the default value for this key. // Default only used when no value is provided by the user via flag, config or ENV. func SetDefault(key string, value interface{}) { v.SetDefault(key, value) } func (v *Viper) SetDefault(key string, value interface{}) { @@ -928,9 +924,9 @@ func (v *Viper) SetDefault(key string, value interface{}) { v.defaults[key] = value } -// Sets the value for the key in the override regiser. +// Set sets the value for the key in the override regiser. // Will be used instead of values obtained via -// flags, config file, ENV, default, or key/value store +// flags, config file, ENV, default, or key/value store. func Set(key string, value interface{}) { v.Set(key, value) } func (v *Viper) Set(key string, value interface{}) { // If alias passed in, then set the proper override @@ -938,7 +934,7 @@ func (v *Viper) Set(key string, value interface{}) { v.override[key] = value } -// Viper will discover and load the configuration file from disk +// ReadInConfig will discover and load the configuration file from disk // and key/value stores, searching in one of the defined paths. func ReadInConfig() error { return v.ReadInConfig() } func (v *Viper) ReadInConfig() error { @@ -973,7 +969,7 @@ func (v *Viper) MergeInConfig() error { return v.MergeConfig(bytes.NewReader(file)) } -// Viper will read a configuration file, setting existing keys to nil if the +// ReadConfig will read a configuration file, setting existing keys to nil if the // key does not exist in the file. func ReadConfig(in io.Reader) error { return v.ReadConfig(in) } func (v *Viper) ReadConfig(in io.Reader) error { @@ -1075,34 +1071,20 @@ func mergeMaps( } } -// func ReadBufConfig(buf *bytes.Buffer) error { return v.ReadBufConfig(buf) } -// func (v *Viper) ReadBufConfig(buf *bytes.Buffer) error { -// v.config = make(map[string]interface{}) -// return v.unmarshalReader(buf, v.config) -// } - -// Attempts to get configuration from a remote source +// ReadRemoteConfig attempts to get configuration from a remote source // and read it in the remote configuration registry. func ReadRemoteConfig() error { return v.ReadRemoteConfig() } func (v *Viper) ReadRemoteConfig() error { - err := v.getKeyValueConfig() - if err != nil { - return err - } - return nil + return v.getKeyValueConfig() } func WatchRemoteConfig() error { return v.WatchRemoteConfig() } func (v *Viper) WatchRemoteConfig() error { - err := v.watchKeyValueConfig() - if err != nil { - return err - } - return nil + return v.watchKeyValueConfig() } -// Unmarshall a Reader into a map -// Should probably be an unexported function +// Unmarshall a Reader into a map. +// Should probably be an unexported function. func unmarshalReader(in io.Reader, c map[string]interface{}) error { return v.unmarshalReader(in, c) } @@ -1118,7 +1100,7 @@ func (v *Viper) insensitiviseMaps() { insensitiviseMap(v.kvstore) } -// retrieve the first found remote configuration +// Retrieve the first found remote configuration. func (v *Viper) getKeyValueConfig() error { if RemoteConfig == nil { return RemoteConfigError("Enable the remote features by doing a blank import of the viper/remote package: '_ github.com/spf13/viper/remote'") @@ -1135,8 +1117,7 @@ func (v *Viper) getKeyValueConfig() error { return RemoteConfigError("No Files Found") } -func (v *Viper) getRemoteConfig(provider *defaultRemoteProvider) (map[string]interface{}, error) { - +func (v *Viper) getRemoteConfig(provider RemoteProvider) (map[string]interface{}, error) { reader, err := RemoteConfig.Get(provider) if err != nil { return nil, err @@ -1145,7 +1126,7 @@ func (v *Viper) getRemoteConfig(provider *defaultRemoteProvider) (map[string]int return v.kvstore, err } -// retrieve the first found remote configuration +// Retrieve the first found remote configuration. func (v *Viper) watchKeyValueConfig() error { for _, rp := range v.remoteProviders { val, err := v.watchRemoteConfig(rp) @@ -1158,7 +1139,7 @@ func (v *Viper) watchKeyValueConfig() error { return RemoteConfigError("No Files Found") } -func (v *Viper) watchRemoteConfig(provider *defaultRemoteProvider) (map[string]interface{}, error) { +func (v *Viper) watchRemoteConfig(provider RemoteProvider) (map[string]interface{}, error) { reader, err := RemoteConfig.Watch(provider) if err != nil { return nil, err @@ -1167,48 +1148,48 @@ func (v *Viper) watchRemoteConfig(provider *defaultRemoteProvider) (map[string]i return v.kvstore, err } -// Return all keys regardless where they are set +// AllKeys returns all keys regardless where they are set. func AllKeys() []string { return v.AllKeys() } func (v *Viper) AllKeys() []string { m := map[string]struct{}{} - for key, _ := range v.defaults { + for key := range v.defaults { m[strings.ToLower(key)] = struct{}{} } - for key, _ := range v.pflags { + for key := range v.pflags { m[strings.ToLower(key)] = struct{}{} } - for key, _ := range v.env { + for key := range v.env { m[strings.ToLower(key)] = struct{}{} } - for key, _ := range v.config { + for key := range v.config { m[strings.ToLower(key)] = struct{}{} } - for key, _ := range v.kvstore { + for key := range v.kvstore { m[strings.ToLower(key)] = struct{}{} } - for key, _ := range v.override { + for key := range v.override { m[strings.ToLower(key)] = struct{}{} } - for key, _ := range v.aliases { + for key := range v.aliases { m[strings.ToLower(key)] = struct{}{} } a := []string{} - for x, _ := range m { + for x := range m { a = append(a, x) } return a } -// Return all settings as a map[string]interface{} +// AllSettings returns all settings as a map[string]interface{}. func AllSettings() map[string]interface{} { return v.AllSettings() } func (v *Viper) AllSettings() map[string]interface{} { m := map[string]interface{}{} @@ -1219,13 +1200,13 @@ func (v *Viper) AllSettings() map[string]interface{} { return m } -// Se the filesystem to use to read configuration. +// SetFs sets the filesystem to use to read configuration. func SetFs(fs afero.Fs) { v.SetFs(fs) } func (v *Viper) SetFs(fs afero.Fs) { v.fs = fs } -// Name for the config file. +// SetConfigName sets name for the config file. // Does not include extension. func SetConfigName(in string) { v.SetConfigName(in) } func (v *Viper) SetConfigName(in string) { @@ -1235,7 +1216,7 @@ func (v *Viper) SetConfigName(in string) { } } -// Sets the type of the configuration returned by the +// SetConfigType sets the type of the configuration returned by the // remote source, e.g. "json". func SetConfigType(in string) { v.SetConfigType(in) } func (v *Viper) SetConfigType(in string) { @@ -1254,9 +1235,9 @@ func (v *Viper) getConfigType() string { if len(ext) > 1 { return ext[1:] - } else { - return "" } + + return "" } func (v *Viper) getConfigFile() string { @@ -1287,8 +1268,8 @@ func (v *Viper) searchInPath(in string) (filename string) { return "" } -// search all configPaths for any config file. -// Returns the first path that exists (and is a config file) +// Search all configPaths for any config file. +// Returns the first path that exists (and is a config file). func (v *Viper) findConfigFile() (string, error) { jww.INFO.Println("Searching for config in ", v.configPaths) @@ -1302,7 +1283,7 @@ func (v *Viper) findConfigFile() (string, error) { return "", ConfigFileNotFoundError{v.configName, fmt.Sprintf("%s", v.configPaths)} } -// Prints all configuration registries for debugging +// Debug prints all configuration registries for debugging // purposes. func Debug() { v.Debug() } func (v *Viper) Debug() { diff --git a/viper_test.go b/viper_test.go index fdbe4d9..cf897d6 100644 --- a/viper_test.go +++ b/viper_test.go @@ -491,7 +491,7 @@ func TestBindPFlags(t *testing.T) { "endpoint": "/public", } - for name, _ := range testValues { + for name := range testValues { testValues[name] = flagSet.String(name, "", "test") } From ed0a9674c6ef3646e218d8bb70708d250e94354b Mon Sep 17 00:00:00 2001 From: Cameron Moore Date: Thu, 22 Sep 2016 13:19:24 -0500 Subject: [PATCH 28/38] Fix retrieval of pflag stringSlice (#240) Fixes #112 Changes some logging directives to use Printf --- viper.go | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/viper.go b/viper.go index bc1bf5e..8a32482 100644 --- a/viper.go +++ b/viper.go @@ -749,12 +749,15 @@ func (v *Viper) find(key string) interface{} { // PFlag Override first flag, exists := v.pflags[key] if exists && flag.HasChanged() { - jww.TRACE.Println(key, "found in override (via pflag):", flag.ValueString()) + jww.TRACE.Printf("%q found in pflag override (%s): %s", key, flag.ValueType(), flag.ValueString()) switch flag.ValueType() { case "int", "int8", "int16", "int32", "int64": return cast.ToInt(flag.ValueString()) case "bool": return cast.ToBool(flag.ValueString()) + case "stringSlice": + s := strings.TrimPrefix(flag.ValueString(), "[") + return strings.TrimSuffix(s, "]") default: return flag.ValueString() } @@ -762,7 +765,7 @@ func (v *Viper) find(key string) interface{} { val, exists = v.override[key] if exists { - jww.TRACE.Println(key, "found in override:", val) + jww.TRACE.Printf("%q found in override: %s", key, val) return val } @@ -770,24 +773,24 @@ func (v *Viper) find(key string) interface{} { // even if it hasn't been registered, if automaticEnv is used, // check any Get request if val = v.getEnv(v.mergeWithEnvPrefix(key)); val != "" { - jww.TRACE.Println(key, "found in environment with val:", val) + jww.TRACE.Printf("%q found in environment: %s", key, val) return val } } envkey, exists := v.env[key] if exists { - jww.TRACE.Println(key, "registered as env var", envkey) + jww.TRACE.Printf("%q registered as env var %q", key, envkey) if val = v.getEnv(envkey); val != "" { - jww.TRACE.Println(envkey, "found in environment with val:", val) + jww.TRACE.Printf("%q found in environment: %s", envkey, val) return val } - jww.TRACE.Println(envkey, "env value unset:") + jww.TRACE.Printf("%q env value unset", envkey) } val, exists = v.config[key] if exists { - jww.TRACE.Println(key, "found in config:", val) + jww.TRACE.Printf("%q found in config (%T): %s", key, val, val) return val } @@ -800,7 +803,7 @@ func (v *Viper) find(key string) interface{} { if reflect.TypeOf(source).Kind() == reflect.Map { val := v.searchMap(cast.ToStringMap(source), path[1:]) if val != nil { - jww.TRACE.Println(key, "found in nested config:", val) + jww.TRACE.Printf("%q found in nested config: %s", key, val) return val } } @@ -809,13 +812,13 @@ func (v *Viper) find(key string) interface{} { val, exists = v.kvstore[key] if exists { - jww.TRACE.Println(key, "found in key/value store:", val) + jww.TRACE.Printf("%q found in key/value store: %s", key, val) return val } val, exists = v.defaults[key] if exists { - jww.TRACE.Println(key, "found in defaults:", val) + jww.TRACE.Printf("%q found in defaults: ", key, val) return val } From e26d6aedc15334f7c2bca88e37037f717659d9b8 Mon Sep 17 00:00:00 2001 From: Yauhen Lazurkin Date: Sat, 24 Sep 2016 02:20:44 +0300 Subject: [PATCH 29/38] Support time.Duration in viper.Unmarshal (#205) * Fixes #105 --- viper.go | 20 +++++++++++++------- viper_test.go | 10 ++++++---- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/viper.go b/viper.go index 8a32482..fafabde 100644 --- a/viper.go +++ b/viper.go @@ -624,7 +624,7 @@ func (v *Viper) UnmarshalKey(key string, rawVal interface{}) error { // on the fields of the structure are properly set. func Unmarshal(rawVal interface{}) error { return v.Unmarshal(rawVal) } func (v *Viper) Unmarshal(rawVal interface{}) error { - err := mapstructure.WeakDecode(v.AllSettings(), rawVal) + err := decode(v.AllSettings(), defaultDecoderConfig(rawVal)) if err != nil { return err @@ -635,16 +635,19 @@ func (v *Viper) Unmarshal(rawVal interface{}) error { return nil } -// A wrapper around mapstructure.Decode that mimics the WeakDecode functionality -// while erroring on non existing vals in the destination struct. -func weakDecodeExact(input, output interface{}) error { - config := &mapstructure.DecoderConfig{ - ErrorUnused: true, +// defaultDecoderConfig returns default mapsstructure.DecoderConfig with suppot +// of time.Duration values +func defaultDecoderConfig(output interface{}) *mapstructure.DecoderConfig { + return &mapstructure.DecoderConfig{ Metadata: nil, Result: output, WeaklyTypedInput: true, + DecodeHook: mapstructure.StringToTimeDurationHookFunc(), } +} +// A wrapper around mapstructure.Decode that mimics the WeakDecode functionality +func decode(input interface{}, config *mapstructure.DecoderConfig) error { decoder, err := mapstructure.NewDecoder(config) if err != nil { return err @@ -655,7 +658,10 @@ func weakDecodeExact(input, output interface{}) error { // UnmarshalExact unmarshals the config into a Struct, erroring if a field is nonexistent // in the destination struct. func (v *Viper) UnmarshalExact(rawVal interface{}) error { - err := weakDecodeExact(v.AllSettings(), rawVal) + config := defaultDecoderConfig(rawVal) + config.ErrorUnused = true + + err := decode(v.AllSettings(), config) if err != nil { return err diff --git a/viper_test.go b/viper_test.go index cf897d6..127e650 100644 --- a/viper_test.go +++ b/viper_test.go @@ -453,10 +453,12 @@ func TestRecursiveAliases(t *testing.T) { func TestUnmarshal(t *testing.T) { SetDefault("port", 1313) Set("name", "Steve") + Set("duration", "1s1ms") type config struct { - Port int - Name string + Port int + Name string + Duration time.Duration } var C config @@ -466,14 +468,14 @@ func TestUnmarshal(t *testing.T) { t.Fatalf("unable to decode into struct, %v", err) } - assert.Equal(t, &C, &config{Name: "Steve", Port: 1313}) + assert.Equal(t, &C, &config{Name: "Steve", Port: 1313, Duration: time.Second + time.Millisecond}) Set("port", 1234) err = Unmarshal(&C) if err != nil { t.Fatalf("unable to decode into struct, %v", err) } - assert.Equal(t, &C, &config{Name: "Steve", Port: 1234}) + assert.Equal(t, &C, &config{Name: "Steve", Port: 1234, Duration: time.Second + time.Millisecond}) } func TestBindPFlags(t *testing.T) { From 438cc0d5c8791cdfc48adf605914f57c9bb0c29e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Mon, 26 Sep 2016 10:02:50 +0200 Subject: [PATCH 30/38] Add benchmark for GetBool ``` BenchmarkGetBool-4 100000 12952 ns/op 225 B/op 10 allocs/op BenchmarkGetBoolFromMap-4 100000000 10.8 ns/op 0 B/op 0 allocs/op ``` --- viper_test.go | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/viper_test.go b/viper_test.go index 127e650..a9f21ea 100644 --- a/viper_test.go +++ b/viper_test.go @@ -918,3 +918,28 @@ func TestShadowedNestedValue(t *testing.T) { assert.Equal(t, GetString("clothing.jacket"), "leather") assert.Equal(t, GetString("clothing.shirt"), polyester) } + +func BenchmarkGetBool(b *testing.B) { + key := "BenchmarkGetBool" + v = New() + v.Set(key, true) + + for i := 0; i < b.N; i++ { + if !v.GetBool(key) { + b.Fatal("GetBool returned false") + } + } +} + +// This is the "perfect result" for the above. +func BenchmarkGetBoolFromMap(b *testing.B) { + m := make(map[string]bool) + key := "BenchmarkGetBool" + m[key] = true + + for i := 0; i < b.N; i++ { + if !m[key] { + b.Fatal("Map value was false") + } + } +} From 382f87b929b84ce13e9c8a375a4b217f224e6c65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Mon, 26 Sep 2016 17:04:02 +0200 Subject: [PATCH 31/38] Remove expensive TRACE logging Also avoid doing a strings.Split in the Get common case. Any TRACE statements in these hot paths must be totally turned off when not testing. ``` benchmark old ns/op new ns/op delta BenchmarkGetBool-4 4090 409 -90.00% BenchmarkGetBoolFromMap-4 6.33 6.28 -0.79% benchmark old allocs new allocs delta BenchmarkGetBool-4 6 3 -50.00% BenchmarkGetBoolFromMap-4 0 0 +0.00% benchmark old bytes new bytes delta BenchmarkGetBool-4 129 33 -74.42% BenchmarkGetBoolFromMap-4 0 0 +0.00% ``` Fixes #242 --- .gitignore | 1 + viper.go | 13 +------------ viper_test.go | 12 ++++++++++++ 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/.gitignore b/.gitignore index 8365624..352a34a 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,4 @@ _testmain.go *.exe *.test +*.bench \ No newline at end of file diff --git a/viper.go b/viper.go index fafabde..0fbd051 100644 --- a/viper.go +++ b/viper.go @@ -463,12 +463,11 @@ func GetViper() *Viper { // Get returns an interface. For a specific value use one of the Get____ methods. func Get(key string) interface{} { return v.Get(key) } func (v *Viper) Get(key string) interface{} { - path := strings.Split(key, v.keyDelim) - lcaseKey := strings.ToLower(key) val := v.find(lcaseKey) if val == nil { + path := strings.Split(key, v.keyDelim) source := v.find(strings.ToLower(path[0])) if source != nil { if reflect.TypeOf(source).Kind() == reflect.Map { @@ -755,7 +754,6 @@ func (v *Viper) find(key string) interface{} { // PFlag Override first flag, exists := v.pflags[key] if exists && flag.HasChanged() { - jww.TRACE.Printf("%q found in pflag override (%s): %s", key, flag.ValueType(), flag.ValueString()) switch flag.ValueType() { case "int", "int8", "int16", "int32", "int64": return cast.ToInt(flag.ValueString()) @@ -771,7 +769,6 @@ func (v *Viper) find(key string) interface{} { val, exists = v.override[key] if exists { - jww.TRACE.Printf("%q found in override: %s", key, val) return val } @@ -779,24 +776,19 @@ func (v *Viper) find(key string) interface{} { // even if it hasn't been registered, if automaticEnv is used, // check any Get request if val = v.getEnv(v.mergeWithEnvPrefix(key)); val != "" { - jww.TRACE.Printf("%q found in environment: %s", key, val) return val } } envkey, exists := v.env[key] if exists { - jww.TRACE.Printf("%q registered as env var %q", key, envkey) if val = v.getEnv(envkey); val != "" { - jww.TRACE.Printf("%q found in environment: %s", envkey, val) return val } - jww.TRACE.Printf("%q env value unset", envkey) } val, exists = v.config[key] if exists { - jww.TRACE.Printf("%q found in config (%T): %s", key, val, val) return val } @@ -809,7 +801,6 @@ func (v *Viper) find(key string) interface{} { if reflect.TypeOf(source).Kind() == reflect.Map { val := v.searchMap(cast.ToStringMap(source), path[1:]) if val != nil { - jww.TRACE.Printf("%q found in nested config: %s", key, val) return val } } @@ -818,13 +809,11 @@ func (v *Viper) find(key string) interface{} { val, exists = v.kvstore[key] if exists { - jww.TRACE.Printf("%q found in key/value store: %s", key, val) return val } val, exists = v.defaults[key] if exists { - jww.TRACE.Printf("%q found in defaults: ", key, val) return val } diff --git a/viper_test.go b/viper_test.go index a9f21ea..8558f92 100644 --- a/viper_test.go +++ b/viper_test.go @@ -919,6 +919,18 @@ func TestShadowedNestedValue(t *testing.T) { assert.Equal(t, GetString("clothing.shirt"), polyester) } +func TestGetBool(t *testing.T) { + key := "BooleanKey" + v = New() + v.Set(key, true) + if !v.GetBool(key) { + t.Fatal("GetBool returned false") + } + if v.GetBool("NotFound") { + t.Fatal("GetBool returned true") + } +} + func BenchmarkGetBool(b *testing.B) { key := "BenchmarkGetBool" v = New() From 670c42a85b2a2215949acd943cb8f11add317e3f Mon Sep 17 00:00:00 2001 From: Cameron Moore Date: Wed, 5 Oct 2016 18:22:39 -0500 Subject: [PATCH 32/38] Check for nil in viper.Sub Fixes #191 --- viper.go | 4 ++++ viper_test.go | 3 +++ 2 files changed, 7 insertions(+) diff --git a/viper.go b/viper.go index 0fbd051..6964041 100644 --- a/viper.go +++ b/viper.go @@ -532,6 +532,10 @@ func Sub(key string) *Viper { return v.Sub(key) } func (v *Viper) Sub(key string) *Viper { subv := New() data := v.Get(key) + if data == nil { + return nil + } + if reflect.TypeOf(data).Kind() == reflect.Map { subv.config = cast.ToStringMap(data) return subv diff --git a/viper_test.go b/viper_test.go index 8558f92..72f695e 100644 --- a/viper_test.go +++ b/viper_test.go @@ -760,6 +760,9 @@ func TestSub(t *testing.T) { subv = v.Sub("clothing.pants.size") assert.Equal(t, subv, (*Viper)(nil)) + + subv = v.Sub("missing.key") + assert.Equal(t, subv, (*Viper)(nil)) } var yamlMergeExampleTgt = []byte(` From ec4eb2fa8549869ae7a2accd4fcc83d1c0555c15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Masson?= Date: Sat, 8 Oct 2016 10:00:18 +0200 Subject: [PATCH 33/38] Nested maps (#195) Fixes #71, #93, #158, #168, #209, #141, #160, #162, #190 * Fixed: indentation in comment * Fixed: Get() returns nil when nested element not found * Fixed: insensitiviseMaps() made recursive so that nested keys are lowercased * Fixed: order of expected<=>actual in assert.Equal() statements * Fixed: find() looks into "overrides" first * Fixed: TestBindPFlags() to use a new Viper instance * Fixed: removed extra aliases from display in Debug() * Added: test for checking precedence of dot-containing keys. * Fixed: Set() and SetDefault() insert nested values * Added: tests for overriding nested values * Changed: AllKeys() includes all keys / AllSettings() includes overridden nested values * Added: test for shadowed nested key * Fixed: properties parsing generates nested maps * Fixed: Get() and IsSet() work correctly on nested values * Changed: modifier README.md to reflect changes --- README.md | 19 +- overrides_test.go | 173 ++++++++++++++++++ util.go | 42 ++++- viper.go | 438 +++++++++++++++++++++++++++++++++------------- viper_test.go | 68 ++++--- 5 files changed, 584 insertions(+), 156 deletions(-) create mode 100644 overrides_test.go diff --git a/README.md b/README.md index cf17560..f4e72f8 100644 --- a/README.md +++ b/README.md @@ -458,16 +458,17 @@ Viper can access a nested field by passing a `.` delimited path of keys: GetString("datastore.metric.host") // (returns "127.0.0.1") ``` -This obeys the precedence rules established above; the search for the root key -(in this example, `datastore`) will cascade through the remaining configuration -registries until found. The search for the sub-keys (`metric` and `host`), -however, will not. +This obeys the precedence rules established above; the search for the path +will cascade through the remaining configuration registries until found. -For example, if the `metric` key was not defined in the configuration loaded -from file, but was defined in the defaults, Viper would return the zero value. +For example, given this configuration file, both `datastore.metric.host` and +`datastore.metric.port` are already defined (and may be overridden). If in addition +`datastore.metric.protocol` was defined in the defaults, Viper would also find it. -On the other hand, if the primary key was not defined, Viper would go through -the remaining registries looking for it. +However, if `datastore.metric` was overridden (by a flag, an environment variable, +the `Set()` method, …) with an immediate value, then all sub-keys of +`datastore.metric` become undefined, they are “shadowed” by the higher-priority +configuration level. Lastly, if there exists a key that matches the delimited key path, its value will be returned instead. E.g. @@ -491,7 +492,7 @@ will be returned instead. E.g. } } -GetString("datastore.metric.host") //returns "0.0.0.0" +GetString("datastore.metric.host") // returns "0.0.0.0" ``` ### Extract sub-tree diff --git a/overrides_test.go b/overrides_test.go new file mode 100644 index 0000000..dd2aa9b --- /dev/null +++ b/overrides_test.go @@ -0,0 +1,173 @@ +package viper + +import ( + "fmt" + "strings" + "testing" + + "github.com/spf13/cast" + "github.com/stretchr/testify/assert" +) + +type layer int + +const ( + defaultLayer layer = iota + 1 + overrideLayer +) + +func TestNestedOverrides(t *testing.T) { + assert := assert.New(t) + var v *Viper + + // Case 0: value overridden by a value + overrideDefault(assert, "tom", 10, "tom", 20) // "tom" is first given 10 as default value, then overridden by 20 + override(assert, "tom", 10, "tom", 20) // "tom" is first given value 10, then overridden by 20 + overrideDefault(assert, "tom.age", 10, "tom.age", 20) + override(assert, "tom.age", 10, "tom.age", 20) + overrideDefault(assert, "sawyer.tom.age", 10, "sawyer.tom.age", 20) + override(assert, "sawyer.tom.age", 10, "sawyer.tom.age", 20) + + // Case 1: key:value overridden by a value + v = overrideDefault(assert, "tom.age", 10, "tom", "boy") // "tom.age" is first given 10 as default value, then "tom" is overridden by "boy" + assert.Nil(v.Get("tom.age")) // "tom.age" should not exist anymore + v = override(assert, "tom.age", 10, "tom", "boy") + assert.Nil(v.Get("tom.age")) + + // Case 2: value overridden by a key:value + overrideDefault(assert, "tom", "boy", "tom.age", 10) // "tom" is first given "boy" as default value, then "tom" is overridden by map{"age":10} + override(assert, "tom.age", 10, "tom", "boy") + + // Case 3: key:value overridden by a key:value + v = overrideDefault(assert, "tom.size", 4, "tom.age", 10) + assert.Equal(4, v.Get("tom.size")) // value should still be reachable + v = override(assert, "tom.size", 4, "tom.age", 10) + assert.Equal(4, v.Get("tom.size")) + deepCheckValue(assert, v, overrideLayer, []string{"tom", "size"}, 4) + + // Case 4: key:value overridden by a map + v = overrideDefault(assert, "tom.size", 4, "tom", map[string]interface{}{"age": 10}) // "tom.size" is first given "4" as default value, then "tom" is overridden by map{"age":10} + assert.Equal(4, v.Get("tom.size")) // "tom.size" should still be reachable + assert.Equal(10, v.Get("tom.age")) // new value should be there + deepCheckValue(assert, v, overrideLayer, []string{"tom", "age"}, 10) // new value should be there + v = override(assert, "tom.size", 4, "tom", map[string]interface{}{"age": 10}) + assert.Nil(v.Get("tom.size")) + assert.Equal(10, v.Get("tom.age")) + deepCheckValue(assert, v, overrideLayer, []string{"tom", "age"}, 10) + + // Case 5: array overridden by a value + overrideDefault(assert, "tom", []int{10, 20}, "tom", 30) + override(assert, "tom", []int{10, 20}, "tom", 30) + overrideDefault(assert, "tom.age", []int{10, 20}, "tom.age", 30) + override(assert, "tom.age", []int{10, 20}, "tom.age", 30) + + // Case 6: array overridden by an array + overrideDefault(assert, "tom", []int{10, 20}, "tom", []int{30, 40}) + override(assert, "tom", []int{10, 20}, "tom", []int{30, 40}) + overrideDefault(assert, "tom.age", []int{10, 20}, "tom.age", []int{30, 40}) + v = override(assert, "tom.age", []int{10, 20}, "tom.age", []int{30, 40}) + // explicit array merge: + s, ok := v.Get("tom.age").([]int) + if assert.True(ok, "tom[\"age\"] is not a slice") { + v.Set("tom.age", append(s, []int{50, 60}...)) + assert.Equal([]int{30, 40, 50, 60}, v.Get("tom.age")) + deepCheckValue(assert, v, overrideLayer, []string{"tom", "age"}, []int{30, 40, 50, 60}) + } +} + +func overrideDefault(assert *assert.Assertions, firstPath string, firstValue interface{}, secondPath string, secondValue interface{}) *Viper { + return overrideFromLayer(defaultLayer, assert, firstPath, firstValue, secondPath, secondValue) +} +func override(assert *assert.Assertions, firstPath string, firstValue interface{}, secondPath string, secondValue interface{}) *Viper { + return overrideFromLayer(overrideLayer, assert, firstPath, firstValue, secondPath, secondValue) +} + +// overrideFromLayer performs the sequential override and low-level checks. +// +// First assignment is made on layer l for path firstPath with value firstValue, +// the second one on the override layer (i.e., with the Set() function) +// for path secondPath with value secondValue. +// +// firstPath and secondPath can include an arbitrary number of dots to indicate +// a nested element. +// +// After each assignment, the value is checked, retrieved both by its full path +// and by its key sequence (successive maps). +func overrideFromLayer(l layer, assert *assert.Assertions, firstPath string, firstValue interface{}, secondPath string, secondValue interface{}) *Viper { + v := New() + firstKeys := strings.Split(firstPath, v.keyDelim) + if assert == nil || + len(firstKeys) == 0 || len(firstKeys[0]) == 0 { + return v + } + + // Set and check first value + switch l { + case defaultLayer: + v.SetDefault(firstPath, firstValue) + case overrideLayer: + v.Set(firstPath, firstValue) + default: + return v + } + assert.Equal(firstValue, v.Get(firstPath)) + deepCheckValue(assert, v, l, firstKeys, firstValue) + + // Override and check new value + secondKeys := strings.Split(secondPath, v.keyDelim) + if len(secondKeys) == 0 || len(secondKeys[0]) == 0 { + return v + } + v.Set(secondPath, secondValue) + assert.Equal(secondValue, v.Get(secondPath)) + deepCheckValue(assert, v, overrideLayer, secondKeys, secondValue) + + return v +} + +// deepCheckValue checks that all given keys correspond to a valid path in the +// configuration map of the given layer, and that the final value equals the one given +func deepCheckValue(assert *assert.Assertions, v *Viper, l layer, keys []string, value interface{}) { + if assert == nil || v == nil || + len(keys) == 0 || len(keys[0]) == 0 { + return + } + + // init + var val interface{} + var ms string + switch l { + case defaultLayer: + val = v.defaults + ms = "v.defaults" + case overrideLayer: + val = v.override + ms = "v.override" + } + + // loop through map + var m map[string]interface{} + err := false + for _, k := range keys { + if val == nil { + assert.Fail(fmt.Sprintf("%s is not a map[string]interface{}", ms)) + return + } + + // deep scan of the map to get the final value + switch val.(type) { + case map[interface{}]interface{}: + m = cast.ToStringMap(val) + case map[string]interface{}: + m = val.(map[string]interface{}) + default: + assert.Fail(fmt.Sprintf("%s is not a map[string]interface{}", ms)) + return + } + ms = ms + "[\"" + k + "\"]" + val = m[k] + } + if !err { + assert.Equal(value, val) + } +} diff --git a/util.go b/util.go index 5f93d65..b0903fb 100644 --- a/util.go +++ b/util.go @@ -45,6 +45,10 @@ func insensitiviseMap(m map[string]interface{}) { if key != lower { delete(m, key) m[lower] = val + if m2, ok := val.(map[string]interface{}); ok { + // nested map: recursively insensitivise + insensitiviseMap(m2) + } } } } @@ -149,7 +153,12 @@ func unmarshallConfigReader(in io.Reader, c map[string]interface{}, configType s } for _, key := range p.Keys() { value, _ := p.Get(key) - c[key] = value + // recursively build nested maps + path := strings.Split(key, ".") + lastKey := strings.ToLower(path[len(path)-1]) + deepestMap := deepSearch(c, path[0:len(path)-1]) + // set innermost value + deepestMap[lastKey] = value } } @@ -199,3 +208,34 @@ func parseSizeInBytes(sizeStr string) uint { return safeMul(uint(size), multiplier) } + +// deepSearch scans deep maps, following the key indexes listed in the +// sequence "path". +// The last value is expected to be another map, and is returned. +// +// In case intermediate keys do not exist, or map to a non-map value, +// a new map is created and inserted, and the search continues from there: +// the initial map "m" may be modified! +func deepSearch(m map[string]interface{}, path []string) map[string]interface{} { + for _, k := range path { + m2, ok := m[k] + if !ok { + // intermediate key does not exist + // => create it and continue from there + m3 := make(map[string]interface{}) + m[k] = m3 + m = m3 + continue + } + m3, ok := m2.(map[string]interface{}) + if !ok { + // intermediate key is a value + // => replace with a new map + m3 = make(map[string]interface{}) + m[k] = m3 + } + // continue search from here + m = m3 + } + return m +} diff --git a/viper.go b/viper.go index 6964041..8f27849 100644 --- a/viper.go +++ b/viper.go @@ -107,11 +107,11 @@ func (fnfe ConfigFileNotFoundError) Error() string { // Defaults : { // "secret": "", // "user": "default", -// "endpoint": "https://localhost" +// "endpoint": "https://localhost" // } // Config : { // "user": "root" -// "secret": "defaultsecret" +// "secret": "defaultsecret" // } // Env : { // "secret": "somesecretkey" @@ -399,8 +399,9 @@ func (v *Viper) providerPathExists(p *defaultRemoteProvider) bool { return false } +// searchMap recursively searches for a value for path in source map. +// Returns nil if not found. func (v *Viper) searchMap(source map[string]interface{}, path []string) interface{} { - if len(path) == 0 { return source } @@ -424,11 +425,133 @@ func (v *Viper) searchMap(source map[string]interface{}, path []string) interfac // if the type of `next` is the same as the type being asserted return v.searchMap(next.(map[string]interface{}), path[1:]) default: - return next + if len(path) == 1 { + return next + } + // got a value but nested key expected, return "nil" for not found + return nil } - } else { - return nil } + return nil +} + +// searchMapWithPathPrefixes recursively searches for a value for path in source map. +// +// While searchMap() considers each path element as a single map key, this +// function searches for, and prioritizes, merged path elements. +// e.g., if in the source, "foo" is defined with a sub-key "bar", and "foo.bar" +// is also defined, this latter value is returned for path ["foo", "bar"]. +// +// This should be useful only at config level (other maps may not contain dots +// in their keys). +func (v *Viper) searchMapWithPathPrefixes(source map[string]interface{}, path []string) interface{} { + if len(path) == 0 { + return source + } + + // search for path prefixes, starting from the longest one + for i := len(path); i > 0; i-- { + prefixKey := strings.ToLower(strings.Join(path[0:i], v.keyDelim)) + + var ok bool + var next interface{} + for k, v := range source { + if strings.ToLower(k) == prefixKey { + ok = true + next = v + break + } + } + + if ok { + var val interface{} + switch next.(type) { + case map[interface{}]interface{}: + val = v.searchMapWithPathPrefixes(cast.ToStringMap(next), path[i:]) + case map[string]interface{}: + // Type assertion is safe here since it is only reached + // if the type of `next` is the same as the type being asserted + val = v.searchMapWithPathPrefixes(next.(map[string]interface{}), path[i:]) + default: + if len(path) == i { + val = next + } + // got a value but nested key expected, do nothing and look for next prefix + } + if val != nil { + return val + } + } + } + + // not found + return nil +} + +// isPathShadowedInDeepMap makes sure the given path is not shadowed somewhere +// on its path in the map. +// e.g., if "foo.bar" has a value in the given map, it “shadows” +// "foo.bar.baz" in a lower-priority map +func (v *Viper) isPathShadowedInDeepMap(path []string, m map[string]interface{}) string { + var parentVal interface{} + for i := 1; i < len(path); i++ { + parentVal = v.searchMap(m, path[0:i]) + if parentVal == nil { + // not found, no need to add more path elements + return "" + } + switch parentVal.(type) { + case map[interface{}]interface{}: + continue + case map[string]interface{}: + continue + default: + // parentVal is a regular value which shadows "path" + return strings.Join(path[0:i], v.keyDelim) + } + } + return "" +} + +// isPathShadowedInFlatMap makes sure the given path is not shadowed somewhere +// in a sub-path of the map. +// e.g., if "foo.bar" has a value in the given map, it “shadows” +// "foo.bar.baz" in a lower-priority map +func (v *Viper) isPathShadowedInFlatMap(path []string, mi interface{}) string { + // unify input map + var m map[string]interface{} + switch mi.(type) { + case map[string]string, map[string]FlagValue: + m = cast.ToStringMap(mi) + default: + return "" + } + + // scan paths + var parentKey string + for i := 1; i < len(path); i++ { + parentKey = strings.Join(path[0:i], v.keyDelim) + if _, ok := m[parentKey]; ok { + return parentKey + } + } + return "" +} + +// isPathShadowedInAutoEnv makes sure the given path is not shadowed somewhere +// in the environment, when automatic env is on. +// e.g., if "foo.bar" has a value in the environment, it “shadows” +// "foo.bar.baz" in a lower-priority map +func (v *Viper) isPathShadowedInAutoEnv(path []string) string { + var parentKey string + var val string + for i := 1; i < len(path); i++ { + parentKey = strings.Join(path[0:i], v.keyDelim) + if val = v.getEnv(v.mergeWithEnvPrefix(parentKey)); val != "" { + return parentKey + } + } + return "" } // SetTypeByDefaultValue enables or disables the inference of a key value's @@ -465,46 +588,16 @@ func Get(key string) interface{} { return v.Get(key) } func (v *Viper) Get(key string) interface{} { lcaseKey := strings.ToLower(key) val := v.find(lcaseKey) - - if val == nil { - path := strings.Split(key, v.keyDelim) - source := v.find(strings.ToLower(path[0])) - if source != nil { - if reflect.TypeOf(source).Kind() == reflect.Map { - val = v.searchMap(cast.ToStringMap(source), path[1:]) - } - } - } - - // if no other value is returned and a flag does exist for the value, - // get the flag's value even if the flag's value has not changed - if val == nil { - if flag, exists := v.pflags[lcaseKey]; exists { - jww.TRACE.Println(key, "get pflag default", val) - switch flag.ValueType() { - case "int", "int8", "int16", "int32", "int64": - val = cast.ToInt(flag.ValueString()) - case "bool": - val = cast.ToBool(flag.ValueString()) - default: - val = flag.ValueString() - } - } - } - if val == nil { return nil } - var valType interface{} - if !v.typeByDefValue { - valType = val - } else { - defVal, defExists := v.defaults[lcaseKey] - if defExists { + valType := val + if v.typeByDefValue { + path := strings.Split(lcaseKey, v.keyDelim) + defVal := v.searchMap(v.defaults, path) + if defVal != nil { valType = defVal - } else { - valType = val } } @@ -752,10 +845,27 @@ func (v *Viper) find(key string) interface{} { var val interface{} var exists bool + // compute the path through the nested maps to the nested value + path := strings.Split(key, v.keyDelim) + if shadow := v.isPathShadowedInDeepMap(path, castMapStringToMapInterface(v.aliases)); shadow != "" { + return nil + } + // if the requested key is an alias, then return the proper key key = v.realKey(key) + // re-compute the path + path = strings.Split(key, v.keyDelim) - // PFlag Override first + // Set() override first + val = v.searchMap(v.override, path) + if val != nil { + return val + } + if shadow := v.isPathShadowedInDeepMap(path, v.override); shadow != "" { + return nil + } + + // PFlag override next flag, exists := v.pflags[key] if exists && flag.HasChanged() { switch flag.ValueType() { @@ -770,56 +880,74 @@ func (v *Viper) find(key string) interface{} { return flag.ValueString() } } - - val, exists = v.override[key] - if exists { - return val + if shadow := v.isPathShadowedInFlatMap(path, v.pflags); shadow != "" { + return nil } + // Env override next if v.automaticEnvApplied { // even if it hasn't been registered, if automaticEnv is used, // check any Get request if val = v.getEnv(v.mergeWithEnvPrefix(key)); val != "" { return val } + if shadow := v.isPathShadowedInAutoEnv(path); shadow != "" { + return nil + } } - envkey, exists := v.env[key] if exists { if val = v.getEnv(envkey); val != "" { return val } } - - val, exists = v.config[key] - if exists { - return val + if shadow := v.isPathShadowedInFlatMap(path, v.env); shadow != "" { + return nil } - // Test for nested config parameter - if strings.Contains(key, v.keyDelim) { - path := strings.Split(key, v.keyDelim) + // Config file next + val = v.searchMapWithPathPrefixes(v.config, path) + if val != nil { + return val + } + if shadow := v.isPathShadowedInDeepMap(path, v.config); shadow != "" { + return nil + } - source := v.find(path[0]) - if source != nil { - if reflect.TypeOf(source).Kind() == reflect.Map { - val := v.searchMap(cast.ToStringMap(source), path[1:]) - if val != nil { - return val - } - } + // K/V store next + val = v.searchMap(v.kvstore, path) + if val != nil { + return val + } + if shadow := v.isPathShadowedInDeepMap(path, v.kvstore); shadow != "" { + return nil + } + + // Default next + val = v.searchMap(v.defaults, path) + if val != nil { + return val + } + if shadow := v.isPathShadowedInDeepMap(path, v.defaults); shadow != "" { + return nil + } + + // last chance: if no other value is returned and a flag does exist for the value, + // get the flag's value even if the flag's value has not changed + if flag, exists := v.pflags[key]; exists { + switch flag.ValueType() { + case "int", "int8", "int16", "int32", "int64": + return cast.ToInt(flag.ValueString()) + case "bool": + return cast.ToBool(flag.ValueString()) + case "stringSlice": + s := strings.TrimPrefix(flag.ValueString(), "[") + return strings.TrimSuffix(s, "]") + default: + return flag.ValueString() } } - - val, exists = v.kvstore[key] - if exists { - return val - } - - val, exists = v.defaults[key] - if exists { - return val - } + // last item, no need to check shadowing return nil } @@ -827,20 +955,8 @@ func (v *Viper) find(key string) interface{} { // IsSet checks to see if the key has been set in any of the data locations. func IsSet(key string) bool { return v.IsSet(key) } func (v *Viper) IsSet(key string) bool { - path := strings.Split(key, v.keyDelim) - lcaseKey := strings.ToLower(key) val := v.find(lcaseKey) - - if val == nil { - source := v.find(strings.ToLower(path[0])) - if source != nil { - if reflect.TypeOf(source).Kind() == reflect.Map { - val = v.searchMap(cast.ToStringMap(source), path[1:]) - } - } - } - return val != nil } @@ -923,7 +1039,13 @@ func SetDefault(key string, value interface{}) { v.SetDefault(key, value) } func (v *Viper) SetDefault(key string, value interface{}) { // If alias passed in, then set the proper default key = v.realKey(strings.ToLower(key)) - v.defaults[key] = value + + path := strings.Split(key, v.keyDelim) + lastKey := strings.ToLower(path[len(path)-1]) + deepestMap := deepSearch(v.defaults, path[0:len(path)-1]) + + // set innermost value + deepestMap[lastKey] = value } // Set sets the value for the key in the override regiser. @@ -933,7 +1055,13 @@ func Set(key string, value interface{}) { v.Set(key, value) } func (v *Viper) Set(key string, value interface{}) { // If alias passed in, then set the proper override key = v.realKey(strings.ToLower(key)) - v.override[key] = value + + path := strings.Split(key, v.keyDelim) + lastKey := strings.ToLower(path[len(path)-1]) + deepestMap := deepSearch(v.override, path[0:len(path)-1]) + + // set innermost value + deepestMap[lastKey] = value } // ReadInConfig will discover and load the configuration file from disk @@ -1013,6 +1141,14 @@ func castToMapStringInterface( return tgt } +func castMapStringToMapInterface(src map[string]string) map[string]interface{} { + tgt := map[string]interface{}{} + for k, v := range src { + tgt[k] = v + } + return tgt +} + // mergeMaps merges two maps. The `itgt` parameter is for handling go-yaml's // insistence on parsing nested structures as `map[interface{}]interface{}` // instead of using a `string` as the key for nest structures beyond one level @@ -1150,55 +1286,114 @@ func (v *Viper) watchRemoteConfig(provider RemoteProvider) (map[string]interface return v.kvstore, err } -// AllKeys returns all keys regardless where they are set. +// AllKeys returns all keys holding a value, regardless of where they are set. +// Nested keys are returned with a v.keyDelim (= ".") separator func AllKeys() []string { return v.AllKeys() } func (v *Viper) AllKeys() []string { - m := map[string]struct{}{} - - for key := range v.defaults { - m[strings.ToLower(key)] = struct{}{} - } - - for key := range v.pflags { - m[strings.ToLower(key)] = struct{}{} - } - - for key := range v.env { - m[strings.ToLower(key)] = struct{}{} - } - - for key := range v.config { - m[strings.ToLower(key)] = struct{}{} - } - - for key := range v.kvstore { - m[strings.ToLower(key)] = struct{}{} - } - - for key := range v.override { - m[strings.ToLower(key)] = struct{}{} - } - - for key := range v.aliases { - m[strings.ToLower(key)] = struct{}{} - } + m := map[string]bool{} + // add all paths, by order of descending priority to ensure correct shadowing + m = v.flattenAndMergeMap(m, castMapStringToMapInterface(v.aliases), "") + m = v.flattenAndMergeMap(m, v.override, "") + m = v.mergeFlatMap(m, v.pflags) + m = v.mergeFlatMap(m, v.env) + m = v.flattenAndMergeMap(m, v.config, "") + m = v.flattenAndMergeMap(m, v.kvstore, "") + m = v.flattenAndMergeMap(m, v.defaults, "") + // convert set of paths to list a := []string{} for x := range m { a = append(a, x) } - return a } -// AllSettings returns all settings as a map[string]interface{}. +// flattenAndMergeMap recursively flattens the given map into a map[string]bool +// of key paths (used as a set, easier to manipulate than a []string): +// - each path is merged into a single key string, delimited with v.keyDelim (= ".") +// - if a path is shadowed by an earlier value in the initial shadow map, +// it is skipped. +// The resulting set of paths is merged to the given shadow set at the same time. +func (v *Viper) flattenAndMergeMap(shadow map[string]bool, m map[string]interface{}, prefix string) map[string]bool { + if shadow != nil && prefix != "" && shadow[prefix] { + // prefix is shadowed => nothing more to flatten + return shadow + } + if shadow == nil { + shadow = make(map[string]bool) + } + + var m2 map[string]interface{} + if prefix != "" { + prefix += v.keyDelim + } + for k, val := range m { + fullKey := prefix + k + switch val.(type) { + case map[string]interface{}: + m2 = val.(map[string]interface{}) + case map[interface{}]interface{}: + m2 = cast.ToStringMap(val) + default: + // immediate value + shadow[strings.ToLower(fullKey)] = true + continue + } + // recursively merge to shadow map + shadow = v.flattenAndMergeMap(shadow, m2, fullKey) + } + return shadow +} + +// mergeFlatMap merges the given maps, excluding values of the second map +// shadowed by values from the first map. +func (v *Viper) mergeFlatMap(shadow map[string]bool, mi interface{}) map[string]bool { + // unify input map + var m map[string]interface{} + switch mi.(type) { + case map[string]string, map[string]FlagValue: + m = cast.ToStringMap(mi) + default: + return shadow + } + + // scan keys +outer: + for k, _ := range m { + path := strings.Split(k, v.keyDelim) + // scan intermediate paths + var parentKey string + for i := 1; i < len(path); i++ { + parentKey = strings.Join(path[0:i], v.keyDelim) + if shadow[parentKey] { + // path is shadowed, continue + continue outer + } + } + // add key + shadow[strings.ToLower(k)] = true + } + return shadow +} + +// AllSettings merges all settings and returns them as a map[string]interface{}. func AllSettings() map[string]interface{} { return v.AllSettings() } func (v *Viper) AllSettings() map[string]interface{} { m := map[string]interface{}{} - for _, x := range v.AllKeys() { - m[x] = v.Get(x) + // start from the list of keys, and construct the map one value at a time + for _, k := range v.AllKeys() { + value := v.Get(k) + if value == nil { + // should not happen, since AllKeys() returns only keys holding a value, + // check just in case anything changes + continue + } + path := strings.Split(k, v.keyDelim) + lastKey := strings.ToLower(path[len(path)-1]) + deepestMap := deepSearch(m, path[0:len(path)-1]) + // set innermost value + deepestMap[lastKey] = value } - return m } @@ -1289,7 +1484,6 @@ func (v *Viper) findConfigFile() (string, error) { // purposes. func Debug() { v.Debug() } func (v *Viper) Debug() { - fmt.Println("Aliases:") fmt.Printf("Aliases:\n%#v\n", v.aliases) fmt.Printf("Override:\n%#v\n", v.override) fmt.Printf("PFlags:\n%#v\n", v.pflags) diff --git a/viper_test.go b/viper_test.go index 72f695e..02d6eb1 100644 --- a/viper_test.go +++ b/viper_test.go @@ -8,6 +8,7 @@ package viper import ( "bytes" "fmt" + "io" "io/ioutil" "os" "path" @@ -104,8 +105,9 @@ var remoteExample = []byte(`{ func initConfigs() { Reset() + var r io.Reader SetConfigType("yaml") - r := bytes.NewReader(yamlExample) + r = bytes.NewReader(yamlExample) unmarshalReader(r, v.config) SetConfigType("json") @@ -259,7 +261,7 @@ func TestUnmarshalling(t *testing.T) { assert.False(t, InConfig("state")) assert.Equal(t, "steve", Get("name")) assert.Equal(t, []interface{}{"skateboarding", "snowboarding", "go"}, Get("hobbies")) - assert.Equal(t, map[interface{}]interface{}{"jacket": "leather", "trousers": "denim", "pants": map[interface{}]interface{}{"size": "large"}}, Get("clothing")) + assert.Equal(t, map[string]interface{}{"jacket": "leather", "trousers": "denim", "pants": map[interface{}]interface{}{"size": "large"}}, Get("clothing")) assert.Equal(t, 35, Get("age")) } @@ -420,9 +422,9 @@ func TestSetEnvReplacer(t *testing.T) { func TestAllKeys(t *testing.T) { initConfigs() - ks := sort.StringSlice{"title", "newkey", "owner", "name", "beard", "ppu", "batters", "hobbies", "clothing", "age", "hacker", "id", "type", "eyes", "p_id", "p_ppu", "p_batters.batter.type", "p_type", "p_name", "foos"} + ks := sort.StringSlice{"title", "newkey", "owner.organization", "owner.dob", "owner.bio", "name", "beard", "ppu", "batters.batter", "hobbies", "clothing.jacket", "clothing.trousers", "clothing.pants.size", "age", "hacker", "id", "type", "eyes", "p_id", "p_ppu", "p_batters.batter.type", "p_type", "p_name", "foos"} dob, _ := time.Parse(time.RFC3339, "1979-05-27T07:32:00Z") - all := map[string]interface{}{"owner": map[string]interface{}{"organization": "MongoDB", "Bio": "MongoDB Chief Developer Advocate & Hacker at Large", "dob": dob}, "title": "TOML Example", "ppu": 0.55, "eyes": "brown", "clothing": map[interface{}]interface{}{"trousers": "denim", "jacket": "leather", "pants": map[interface{}]interface{}{"size": "large"}}, "id": "0001", "batters": map[string]interface{}{"batter": []interface{}{map[string]interface{}{"type": "Regular"}, map[string]interface{}{"type": "Chocolate"}, map[string]interface{}{"type": "Blueberry"}, map[string]interface{}{"type": "Devil's Food"}}}, "hacker": true, "beard": true, "hobbies": []interface{}{"skateboarding", "snowboarding", "go"}, "age": 35, "type": "donut", "newkey": "remote", "name": "Cake", "p_id": "0001", "p_ppu": "0.55", "p_name": "Cake", "p_batters.batter.type": "Regular", "p_type": "donut", "foos": []map[string]interface{}{map[string]interface{}{"foo": []map[string]interface{}{map[string]interface{}{"key": 1}, map[string]interface{}{"key": 2}, map[string]interface{}{"key": 3}, map[string]interface{}{"key": 4}}}}} + all := map[string]interface{}{"owner": map[string]interface{}{"organization": "MongoDB", "bio": "MongoDB Chief Developer Advocate & Hacker at Large", "dob": dob}, "title": "TOML Example", "ppu": 0.55, "eyes": "brown", "clothing": map[string]interface{}{"trousers": "denim", "jacket": "leather", "pants": map[string]interface{}{"size": "large"}}, "id": "0001", "batters": map[string]interface{}{"batter": []interface{}{map[string]interface{}{"type": "Regular"}, map[string]interface{}{"type": "Chocolate"}, map[string]interface{}{"type": "Blueberry"}, map[string]interface{}{"type": "Devil's Food"}}}, "hacker": true, "beard": true, "hobbies": []interface{}{"skateboarding", "snowboarding", "go"}, "age": 35, "type": "donut", "newkey": "remote", "name": "Cake", "p_id": "0001", "p_ppu": "0.55", "p_name": "Cake", "p_batters": map[string]interface{}{"batter": map[string]interface{}{"type": "Regular"}}, "p_type": "donut", "foos": []map[string]interface{}{map[string]interface{}{"foo": []map[string]interface{}{map[string]interface{}{"key": 1}, map[string]interface{}{"key": 2}, map[string]interface{}{"key": 3}, map[string]interface{}{"key": 4}}}}} var allkeys sort.StringSlice allkeys = AllKeys() @@ -468,17 +470,18 @@ func TestUnmarshal(t *testing.T) { t.Fatalf("unable to decode into struct, %v", err) } - assert.Equal(t, &C, &config{Name: "Steve", Port: 1313, Duration: time.Second + time.Millisecond}) + assert.Equal(t, &config{Name: "Steve", Port: 1313, Duration: time.Second + time.Millisecond}, &C) Set("port", 1234) err = Unmarshal(&C) if err != nil { t.Fatalf("unable to decode into struct, %v", err) } - assert.Equal(t, &C, &config{Name: "Steve", Port: 1234, Duration: time.Second + time.Millisecond}) + assert.Equal(t, &config{Name: "Steve", Port: 1234, Duration: time.Second + time.Millisecond}, &C) } func TestBindPFlags(t *testing.T) { + v := New() // create independent Viper object flagSet := pflag.NewFlagSet("test", pflag.ContinueOnError) var testValues = map[string]*string{ @@ -497,7 +500,7 @@ func TestBindPFlags(t *testing.T) { testValues[name] = flagSet.String(name, "", "test") } - err := BindPFlags(flagSet) + err := v.BindPFlags(flagSet) if err != nil { t.Fatalf("error binding flag set, %v", err) } @@ -508,7 +511,7 @@ func TestBindPFlags(t *testing.T) { }) for name, expected := range mutatedTestValues { - assert.Equal(t, Get(name), expected) + assert.Equal(t, expected, v.Get(name)) } } @@ -641,7 +644,7 @@ func TestFindsNestedKeys(t *testing.T) { "name": "Cake", "hacker": true, "ppu": 0.55, - "clothing": map[interface{}]interface{}{ + "clothing": map[string]interface{}{ "jacket": "leather", "trousers": "denim", "pants": map[interface{}]interface{}{ @@ -690,7 +693,7 @@ func TestReadBufConfig(t *testing.T) { assert.False(t, v.InConfig("state")) assert.Equal(t, "steve", v.Get("name")) assert.Equal(t, []interface{}{"skateboarding", "snowboarding", "go"}, v.Get("hobbies")) - assert.Equal(t, map[interface{}]interface{}{"jacket": "leather", "trousers": "denim", "pants": map[interface{}]interface{}{"size": "large"}}, v.Get("clothing")) + assert.Equal(t, map[string]interface{}{"jacket": "leather", "trousers": "denim", "pants": map[interface{}]interface{}{"size": "large"}}, v.Get("clothing")) assert.Equal(t, 35, v.Get("age")) } @@ -759,10 +762,10 @@ func TestSub(t *testing.T) { assert.Equal(t, v.Get("clothing.pants.size"), subv.Get("size")) subv = v.Sub("clothing.pants.size") - assert.Equal(t, subv, (*Viper)(nil)) + assert.Equal(t, (*Viper)(nil), subv) subv = v.Sub("missing.key") - assert.Equal(t, subv, (*Viper)(nil)) + assert.Equal(t, (*Viper)(nil), subv) } var yamlMergeExampleTgt = []byte(` @@ -883,28 +886,28 @@ func TestMergeConfigNoMerge(t *testing.T) { } func TestUnmarshalingWithAliases(t *testing.T) { - SetDefault("Id", 1) - Set("name", "Steve") - Set("lastname", "Owen") + v := New() + v.SetDefault("ID", 1) + v.Set("name", "Steve") + v.Set("lastname", "Owen") - RegisterAlias("UserID", "Id") - RegisterAlias("Firstname", "name") - RegisterAlias("Surname", "lastname") + v.RegisterAlias("UserID", "ID") + v.RegisterAlias("Firstname", "name") + v.RegisterAlias("Surname", "lastname") type config struct { - Id int + ID int FirstName string Surname string } var C config - - err := Unmarshal(&C) + err := v.Unmarshal(&C) if err != nil { t.Fatalf("unable to decode into struct, %v", err) } - assert.Equal(t, &C, &config{Id: 1, FirstName: "Steve", Surname: "Owen"}) + assert.Equal(t, &config{ID: 1, FirstName: "Steve", Surname: "Owen"}, &C) } func TestSetConfigNameClearsFileCache(t *testing.T) { @@ -917,9 +920,26 @@ func TestShadowedNestedValue(t *testing.T) { polyester := "polyester" initYAML() SetDefault("clothing.shirt", polyester) + SetDefault("clothing.jacket.price", 100) - assert.Equal(t, GetString("clothing.jacket"), "leather") - assert.Equal(t, GetString("clothing.shirt"), polyester) + assert.Equal(t, "leather", GetString("clothing.jacket")) + assert.Nil(t, Get("clothing.jacket.price")) + assert.Equal(t, polyester, GetString("clothing.shirt")) + + clothingSettings := AllSettings()["clothing"].(map[string]interface{}) + assert.Equal(t, "leather", clothingSettings["jacket"]) + assert.Equal(t, polyester, clothingSettings["shirt"]) +} + +func TestDotParameter(t *testing.T) { + initJSON() + // shoud take precedence over batters defined in jsonExample + r := bytes.NewReader([]byte(`{ "batters.batter": [ { "type": "Small" } ] }`)) + unmarshalReader(r, v.config) + + actual := Get("batters.batter") + expected := []interface{}{map[string]interface{}{"type": "Small"}} + assert.Equal(t, expected, actual) } func TestGetBool(t *testing.T) { From b61deff9070db474aa7668f55bf6eec00008409e Mon Sep 17 00:00:00 2001 From: Albert Date: Sun, 9 Oct 2016 20:33:40 +0200 Subject: [PATCH 34/38] Add GoDoc badge --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f4e72f8..261bd16 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ Many Go projects are built using Viper including: * [BloomApi](https://www.bloomapi.com/) * [doctl](https://github.com/digitalocean/doctl) - [![Build Status](https://travis-ci.org/spf13/viper.svg)](https://travis-ci.org/spf13/viper) [![Join the chat at https://gitter.im/spf13/viper](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/spf13/viper?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) + [![Build Status](https://travis-ci.org/spf13/viper.svg)](https://travis-ci.org/spf13/viper) [![Join the chat at https://gitter.im/spf13/viper](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/spf13/viper?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) [![GoDoc](https://godoc.org/github.com/spf13/viper?status.svg)](https://godoc.org/github.com/spf13/viper) ## What is Viper? From 21ea37b67349cb6594eea6eaf765b2a7b96f03e2 Mon Sep 17 00:00:00 2001 From: Albert Date: Sun, 9 Oct 2016 21:34:00 +0200 Subject: [PATCH 35/38] Fix space between Go projects and badges --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 261bd16..ae6d50f 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ Many Go projects are built using Viper including: * [BloomApi](https://www.bloomapi.com/) * [doctl](https://github.com/digitalocean/doctl) - [![Build Status](https://travis-ci.org/spf13/viper.svg)](https://travis-ci.org/spf13/viper) [![Join the chat at https://gitter.im/spf13/viper](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/spf13/viper?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) [![GoDoc](https://godoc.org/github.com/spf13/viper?status.svg)](https://godoc.org/github.com/spf13/viper) +[![Build Status](https://travis-ci.org/spf13/viper.svg)](https://travis-ci.org/spf13/viper) [![Join the chat at https://gitter.im/spf13/viper](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/spf13/viper?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) [![GoDoc](https://godoc.org/github.com/spf13/viper?status.svg)](https://godoc.org/github.com/spf13/viper) ## What is Viper? From 54b81535af6c3bceed4672011ef221de7104bbd6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Mon, 10 Oct 2016 11:47:23 +0200 Subject: [PATCH 36/38] Add BenchmarkGet --- viper_test.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/viper_test.go b/viper_test.go index 02d6eb1..c04b5f8 100644 --- a/viper_test.go +++ b/viper_test.go @@ -966,6 +966,18 @@ func BenchmarkGetBool(b *testing.B) { } } +func BenchmarkGet(b *testing.B) { + key := "BenchmarkGet" + v = New() + v.Set(key, true) + + for i := 0; i < b.N; i++ { + if !v.Get(key).(bool) { + b.Fatal("Get returned false") + } + } +} + // This is the "perfect result" for the above. func BenchmarkGetBoolFromMap(b *testing.B) { m := make(map[string]bool) From 51f23d1f1c56a7773ae8f2cfd038f7996ecc9ac2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Mon, 10 Oct 2016 13:40:38 +0200 Subject: [PATCH 37/38] Restore performance for the simple case ``` BenchmarkGetBool-4 1021 481 -52.89% BenchmarkGet-4 879 403 -54.15% BenchmarkGetBoolFromMap-4 6.56 6.40 -2.44% benchmark old allocs new allocs delta BenchmarkGetBool-4 6 4 -33.33% BenchmarkGet-4 6 4 -33.33% BenchmarkGetBoolFromMap-4 0 0 +0.00% benchmark old bytes new bytes delta BenchmarkGetBool-4 113 49 -56.64% BenchmarkGet-4 112 48 -57.14% BenchmarkGetBoolFromMap-4 0 0 +0.00% ``` Fixes #249 Fixes https://github.com/spf13/hugo/issues/2536 --- viper.go | 71 +++++++++++++++++++++++++---------- viper_test.go | 102 +++++++++++++++++++++++++++++++++++++++----------- 2 files changed, 133 insertions(+), 40 deletions(-) diff --git a/viper.go b/viper.go index 8f27849..a03540c 100644 --- a/viper.go +++ b/viper.go @@ -399,17 +399,42 @@ func (v *Viper) providerPathExists(p *defaultRemoteProvider) bool { return false } +// searchMapForKey may end up traversing the map if the key references a nested +// item (foo.bar), but will use a fast path for the common case. +// Note: This assumes that the key given is already lowercase. +func (v *Viper) searchMapForKey(source map[string]interface{}, lcaseKey string) interface{} { + if !strings.Contains(lcaseKey, v.keyDelim) { + v, ok := source[lcaseKey] + if ok { + return v + } + return nil + } + + path := strings.Split(lcaseKey, v.keyDelim) + return v.searchMap(source, path) +} + // searchMap recursively searches for a value for path in source map. // Returns nil if not found. +// Note: This assumes that the path entries are lower cased. func (v *Viper) searchMap(source map[string]interface{}, path []string) interface{} { if len(path) == 0 { return source } + // Fast path + if len(path) == 1 { + if v, ok := source[path[0]]; ok { + return v + } + return nil + } + var ok bool var next interface{} for k, v := range source { - if strings.ToLower(k) == strings.ToLower(path[0]) { + if k == path[0] { ok = true next = v break @@ -594,8 +619,8 @@ func (v *Viper) Get(key string) interface{} { valType := val if v.typeByDefValue { - path := strings.Split(lcaseKey, v.keyDelim) - defVal := v.searchMap(v.defaults, path) + // TODO(bep) this branch isn't covered by a single test. + defVal := v.searchMapForKey(v.defaults, lcaseKey) if defVal != nil { valType = defVal } @@ -841,32 +866,39 @@ func (v *Viper) BindEnv(input ...string) error { // Viper will check in the following order: // flag, env, config file, key/value store, default. // Viper will check to see if an alias exists first. -func (v *Viper) find(key string) interface{} { - var val interface{} - var exists bool +// Note: this assumes a lower-cased key given. +func (v *Viper) find(lcaseKey string) interface{} { + + var ( + val interface{} + exists bool + path = strings.Split(lcaseKey, v.keyDelim) + nested = len(path) > 1 + ) // compute the path through the nested maps to the nested value - path := strings.Split(key, v.keyDelim) - if shadow := v.isPathShadowedInDeepMap(path, castMapStringToMapInterface(v.aliases)); shadow != "" { + if nested && v.isPathShadowedInDeepMap(path, castMapStringToMapInterface(v.aliases)) != "" { return nil } // if the requested key is an alias, then return the proper key - key = v.realKey(key) - // re-compute the path - path = strings.Split(key, v.keyDelim) + lcaseKey = v.realKey(lcaseKey) // Set() override first - val = v.searchMap(v.override, path) + val = v.searchMapForKey(v.override, lcaseKey) if val != nil { return val } - if shadow := v.isPathShadowedInDeepMap(path, v.override); shadow != "" { + + path = strings.Split(lcaseKey, v.keyDelim) + nested = len(path) > 1 + + if nested && v.isPathShadowedInDeepMap(path, v.override) != "" { return nil } // PFlag override next - flag, exists := v.pflags[key] + flag, exists := v.pflags[lcaseKey] if exists && flag.HasChanged() { switch flag.ValueType() { case "int", "int8", "int16", "int32", "int64": @@ -880,7 +912,8 @@ func (v *Viper) find(key string) interface{} { return flag.ValueString() } } - if shadow := v.isPathShadowedInFlatMap(path, v.pflags); shadow != "" { + + if nested && v.isPathShadowedInFlatMap(path, v.pflags) != "" { return nil } @@ -888,14 +921,14 @@ func (v *Viper) find(key string) interface{} { if v.automaticEnvApplied { // even if it hasn't been registered, if automaticEnv is used, // check any Get request - if val = v.getEnv(v.mergeWithEnvPrefix(key)); val != "" { + if val = v.getEnv(v.mergeWithEnvPrefix(lcaseKey)); val != "" { return val } - if shadow := v.isPathShadowedInAutoEnv(path); shadow != "" { + if nested && v.isPathShadowedInAutoEnv(path) != "" { return nil } } - envkey, exists := v.env[key] + envkey, exists := v.env[lcaseKey] if exists { if val = v.getEnv(envkey); val != "" { return val @@ -934,7 +967,7 @@ func (v *Viper) find(key string) interface{} { // last chance: if no other value is returned and a flag does exist for the value, // get the flag's value even if the flag's value has not changed - if flag, exists := v.pflags[key]; exists { + if flag, exists := v.pflags[lcaseKey]; exists { switch flag.ValueType() { case "int", "int8", "int16", "int32", "int64": return cast.ToInt(flag.ValueString()) diff --git a/viper_test.go b/viper_test.go index c04b5f8..963c3ce 100644 --- a/viper_test.go +++ b/viper_test.go @@ -18,6 +18,8 @@ import ( "testing" "time" + "github.com/spf13/cast" + "github.com/spf13/pflag" "github.com/stretchr/testify/assert" ) @@ -131,12 +133,18 @@ func initConfigs() { unmarshalReader(remote, v.kvstore) } -func initYAML() { +func initConfig(typ, config string) { Reset() - SetConfigType("yaml") - r := bytes.NewReader(yamlExample) + SetConfigType(typ) + r := strings.NewReader(config) - unmarshalReader(r, v.config) + if err := unmarshalReader(r, v.config); err != nil { + panic(err) + } +} + +func initYAML() { + initConfig("yaml", string(yamlExample)) } func initJSON() { @@ -435,13 +443,8 @@ func TestAllKeys(t *testing.T) { assert.Equal(t, all, AllSettings()) } -func TestCaseInSensitive(t *testing.T) { - assert.Equal(t, true, Get("hacker")) - Set("Title", "Checking Case") - assert.Equal(t, "Checking Case", Get("tItle")) -} - func TestAliasesOfAliases(t *testing.T) { + Set("Title", "Checking Case") RegisterAlias("Foo", "Bar") RegisterAlias("Bar", "Title") assert.Equal(t, "Checking Case", Get("FOO")) @@ -538,7 +541,6 @@ func TestBindPFlag(t *testing.T) { } func TestBoundCaseSensitivity(t *testing.T) { - assert.Equal(t, "brown", Get("eyes")) BindEnv("eYEs", "TURTLE_EYES") @@ -917,8 +919,19 @@ func TestSetConfigNameClearsFileCache(t *testing.T) { } func TestShadowedNestedValue(t *testing.T) { + + config := `name: steve +clothing: + jacket: leather + trousers: denim + pants: + size: large +` + initConfig("yaml", config) + + assert.Equal(t, "steve", GetString("name")) + polyester := "polyester" - initYAML() SetDefault("clothing.shirt", polyester) SetDefault("clothing.jacket.price", 100) @@ -942,18 +955,65 @@ func TestDotParameter(t *testing.T) { assert.Equal(t, expected, actual) } -func TestGetBool(t *testing.T) { - key := "BooleanKey" - v = New() - v.Set(key, true) - if !v.GetBool(key) { - t.Fatal("GetBool returned false") - } - if v.GetBool("NotFound") { - t.Fatal("GetBool returned true") +func TestCaseInSensitive(t *testing.T) { + for _, config := range []struct { + typ string + content string + }{ + {"yaml", ` +aBcD: 1 +eF: + gH: 2 + iJk: 3 + Lm: + nO: 4 + P: + Q: 5 + R: 6 +`}, + {"json", `{ + "aBcD": 1, + "eF": { + "iJk": 3, + "Lm": { + "P": { + "Q": 5, + "R": 6 + }, + "nO": 4 + }, + "gH": 2 + } +}`}, + {"toml", `aBcD = 1 +[eF] +gH = 2 +iJk = 3 +[eF.Lm] +nO = 4 +[eF.Lm.P] +Q = 5 +R = 6 +`}, + } { + doTestCaseInSensitive(t, config.typ, config.content) } } +func doTestCaseInSensitive(t *testing.T, typ, config string) { + initConfig(typ, config) + Set("RfD", true) + assert.Equal(t, true, Get("rfd")) + assert.Equal(t, true, Get("rFD")) + assert.Equal(t, 1, cast.ToInt(Get("abcd"))) + assert.Equal(t, 1, cast.ToInt(Get("Abcd"))) + assert.Equal(t, 2, cast.ToInt(Get("ef.gh"))) + assert.Equal(t, 3, cast.ToInt(Get("ef.ijk"))) + assert.Equal(t, 4, cast.ToInt(Get("ef.lm.no"))) + assert.Equal(t, 5, cast.ToInt(Get("ef.lm.p.q"))) + +} + func BenchmarkGetBool(b *testing.B) { key := "BenchmarkGetBool" v = New() From c14ce6d43cf5e22e2bd44938f3f45a2761c18350 Mon Sep 17 00:00:00 2001 From: Nils Landt Date: Wed, 12 Oct 2016 11:40:13 +0200 Subject: [PATCH 38/38] Correct FlagValue interface examples in Readme (#254) `IsChanged` is actually named `HasChanged`, and the examples were missing function return declarations, so you couldn't just copy/paste it. --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index ae6d50f..25181df 100644 --- a/README.md +++ b/README.md @@ -277,10 +277,10 @@ Viper provides two Go interfaces to bind other flag systems if you don't use `Pf ```go type myFlag struct {} -func (f myFlag) IsChanged() { return false } -func (f myFlag) Name() { return "my-flag-name" } -func (f myFlag) ValueString() { return "my-flag-value" } -func (f myFlag) ValueType() { return "string" } +func (f myFlag) HasChanged() bool { return false } +func (f myFlag) Name() string { return "my-flag-name" } +func (f myFlag) ValueString() string { return "my-flag-value" } +func (f myFlag) ValueType() string { return "string" } ``` Once your flag implements this interface, you can simply tell Viper to bind it: