diff --git a/.gitignore b/.gitignore index 352a34a..01b5c44 100644 --- a/.gitignore +++ b/.gitignore @@ -21,4 +21,9 @@ _testmain.go *.exe *.test -*.bench \ No newline at end of file +*.bench + +.vscode + +# exclude dependencies in the `/vendor` folder +vendor diff --git a/.travis.yml b/.travis.yml index fa39805..bb83057 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,9 +1,13 @@ go_import_path: github.com/spf13/viper language: go + +env: + global: + - GO111MODULE="on" + go: - - 1.9.x - - 1.10.x + - 1.11.x - tip os: diff --git a/README.md b/README.md index 64bf474..0208eac 100644 --- a/README.md +++ b/README.md @@ -179,19 +179,20 @@ viper.GetBool("verbose") // true ### Working with Environment Variables Viper has full support for environment variables. This enables 12 factor -applications out of the box. There are four methods that exist to aid working +applications out of the box. There are five methods that exist to aid working with ENV: * `AutomaticEnv()` * `BindEnv(string...) : error` * `SetEnvPrefix(string)` * `SetEnvKeyReplacer(string...) *strings.Replacer` + * `AllowEmptyEnvVar(bool)` _When working with ENV variables, it’s important to recognize that Viper treats ENV variables as case sensitive._ Viper provides a mechanism to try to ensure that ENV variables are unique. By -using `SetEnvPrefix`, you can tell Viper to use add a prefix while reading from +using `SetEnvPrefix`, you can tell Viper to use a prefix while reading from the environment variables. Both `BindEnv` and `AutomaticEnv` will use this prefix. @@ -217,6 +218,10 @@ keys to an extent. This is useful if you want to use `-` or something in your `Get()` calls, but want your environmental variables to use `_` delimiters. An example of using it can be found in `viper_test.go`. +By default empty environment variables are considered unset and will fall back to +the next configuration source. To treat empty environment variables as set, use +the `AllowEmptyEnv` method. + #### Env example ```go @@ -373,12 +378,33 @@ how to use Consul. ### Remote Key/Value Store Example - Unencrypted +#### etcd ```go viper.AddRemoteProvider("etcd", "http://127.0.0.1:4001","/config/hugo.json") viper.SetConfigType("json") // because there is no file extension in a stream of bytes, supported extensions are "json", "toml", "yaml", "yml", "properties", "props", "prop" err := viper.ReadRemoteConfig() ``` +#### Consul +You need to set a key to Consul key/value storage with JSON value containing your desired config. +For example, create a Consul key/value store key `MY_CONSUL_KEY` with value: + +```json +{ + "port": 8080, + "hostname": "myhostname.com" +} +``` + +```go +viper.AddRemoteProvider("consul", "localhost:8500", "MY_CONSUL_KEY") +viper.SetConfigType("json") // Need to explicitly set this to json +err := viper.ReadRemoteConfig() + +fmt.Println(viper.Get("port")) // 8080 +fmt.Println(viper.Get("hostname")) // myhostname.com +``` + ### Remote Key/Value Store Example - Encrypted ```go @@ -437,6 +463,7 @@ The following functions and methods exist: * `GetTime(key string) : time.Time` * `GetDuration(key string) : time.Duration` * `IsSet(key string) : bool` + * `AllSettings() : map[string]interface{}` One important thing to recognize is that each Get function will return a zero value if it’s not found. To check if a given key exists, the `IsSet()` method @@ -590,6 +617,27 @@ if err != nil { } ``` +### Marshalling to string + +You may need to marhsal all the settings held in viper into a string rather than write them to a file. +You can use your favorite format's marshaller with the config returned by `AllSettings()`. + +```go +import ( + yaml "gopkg.in/yaml.v2" + // ... +) + +func yamlStringSettings() string { + c := viper.AllSettings() + bs, err := yaml.Marshal(c) + if err != nil { + t.Fatalf("unable to marshal config to YAML: %v", err) + } + return string(bs) +} +``` + ## Viper or Vipers? Viper comes ready to use out of the box. There is no configuration or diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..86e801c --- /dev/null +++ b/go.mod @@ -0,0 +1,24 @@ +module github.com/spf13/viper + +require ( + github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6 // indirect + github.com/coreos/etcd v3.3.10+incompatible // indirect + github.com/coreos/go-etcd v2.0.0+incompatible // indirect + github.com/coreos/go-semver v0.2.0 // indirect + github.com/fsnotify/fsnotify v1.4.7 + github.com/hashicorp/hcl v1.0.0 + github.com/magiconair/properties v1.8.0 + github.com/mitchellh/mapstructure v1.1.2 + github.com/pelletier/go-toml v1.2.0 + github.com/spf13/afero v1.1.2 + github.com/spf13/cast v1.3.0 + github.com/spf13/jwalterweatherman v1.0.0 + github.com/spf13/pflag v1.0.3 + github.com/stretchr/testify v1.2.2 + github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8 // indirect + github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77 + golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9 // indirect + golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a // indirect + golang.org/x/text v0.3.0 // indirect + gopkg.in/yaml.v2 v2.2.2 +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..5c9fb7d --- /dev/null +++ b/go.sum @@ -0,0 +1,35 @@ +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a h1:1n5lsVfiQW3yfsRGu98756EH1YthsFqr/5mxHduZW2A= +golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/viper.go b/viper.go index 13c2518..a3d37f8 100644 --- a/viper.go +++ b/viper.go @@ -30,6 +30,7 @@ import ( "path/filepath" "reflect" "strings" + "sync" "time" yaml "gopkg.in/yaml.v2" @@ -113,6 +114,23 @@ func (fnfe ConfigFileNotFoundError) Error() string { return fmt.Sprintf("Config File %q Not Found in %q", fnfe.name, fnfe.locations) } +// A DecoderConfigOption can be passed to viper.Unmarshal to configure +// mapstructure.DecoderConfig options +type DecoderConfigOption func(*mapstructure.DecoderConfig) + +// DecodeHook returns a DecoderConfigOption which overrides the default +// DecoderConfig.DecodeHook value, the default is: +// +// mapstructure.ComposeDecodeHookFunc( +// mapstructure.StringToTimeDurationHookFunc(), +// mapstructure.StringToSliceHookFunc(","), +// ) +func DecodeHook(hook mapstructure.DecodeHookFunc) DecoderConfigOption { + return func(c *mapstructure.DecoderConfig) { + c.DecodeHook = hook + } +} + // Viper is a prioritized configuration registry. It // maintains a set of configuration sources, fetches // values to populate those, and provides them according @@ -162,13 +180,15 @@ type Viper struct { remoteProviders []*defaultRemoteProvider // Name of file to look for inside the path - configName string - configFile string - configType string - envPrefix string + configName string + configFile string + configType string + configPermissions os.FileMode + envPrefix string automaticEnvApplied bool envKeyReplacer *strings.Replacer + allowEmptyEnv bool config map[string]interface{} override map[string]interface{} @@ -191,6 +211,7 @@ func New() *Viper { v := new(Viper) v.keyDelim = "." v.configName = "config" + v.configPermissions = os.FileMode(0644) v.fs = afero.NewOsFs() v.config = make(map[string]interface{}) v.override = make(map[string]interface{}) @@ -259,48 +280,73 @@ func (v *Viper) OnConfigChange(run func(in fsnotify.Event)) { } func WatchConfig() { v.WatchConfig() } + func (v *Viper) WatchConfig() { + initWG := sync.WaitGroup{} + initWG.Add(1) go func() { watcher, err := fsnotify.NewWatcher() if err != nil { log.Fatal(err) } defer watcher.Close() - // we have to watch the entire directory to pick up renames/atomic saves in a cross-platform way filename, err := v.getConfigFile() if err != nil { - log.Println("error:", err) + log.Printf("error: %v\n", err) return } configFile := filepath.Clean(filename) configDir, _ := filepath.Split(configFile) + realConfigFile, _ := filepath.EvalSymlinks(filename) - done := make(chan bool) + eventsWG := sync.WaitGroup{} + eventsWG.Add(1) go func() { for { select { - case event := <-watcher.Events: - // we only care about the config file - if filepath.Clean(event.Name) == configFile { - if event.Op&fsnotify.Write == fsnotify.Write || event.Op&fsnotify.Create == fsnotify.Create { - err := v.ReadInConfig() - if err != nil { - log.Println("error:", err) - } + case event, ok := <-watcher.Events: + if !ok { // 'Events' channel is closed + eventsWG.Done() + return + } + currentConfigFile, _ := filepath.EvalSymlinks(filename) + // we only care about the config file with the following cases: + // 1 - if the config file was modified or created + // 2 - if the real path to the config file changed (eg: k8s ConfigMap replacement) + const writeOrCreateMask = fsnotify.Write | fsnotify.Create + if (filepath.Clean(event.Name) == configFile && + event.Op&writeOrCreateMask != 0) || + (currentConfigFile != "" && currentConfigFile != realConfigFile) { + realConfigFile = currentConfigFile + err := v.ReadInConfig() + if err != nil { + log.Printf("error reading config file: %v\n", err) + } + if v.onConfigChange != nil { v.onConfigChange(event) } + } else if filepath.Clean(event.Name) == configFile && + event.Op&fsnotify.Remove&fsnotify.Remove != 0 { + eventsWG.Done() + return } - case err := <-watcher.Errors: - log.Println("error:", err) + + case err, ok := <-watcher.Errors: + if ok { // 'Errors' channel is not closed + log.Printf("watcher error: %v\n", err) + } + eventsWG.Done() + return } } }() - watcher.Add(configDir) - <-done + initWG.Done() // done initalizing the watch in this go routine, so the parent routine can move on... + eventsWG.Wait() // now, wait for event loop to end in this go-routine... }() + initWG.Wait() // make sure that the go routine above fully ended before returning } // SetConfigFile explicitly defines the path, name and extension of the config file. @@ -330,6 +376,14 @@ func (v *Viper) mergeWithEnvPrefix(in string) string { return strings.ToUpper(in) } +// AllowEmptyEnv tells Viper to consider set, +// but empty environment variables as valid values instead of falling back. +// For backward compatibility reasons this is false by default. +func AllowEmptyEnv(allowEmptyEnv bool) { v.AllowEmptyEnv(allowEmptyEnv) } +func (v *Viper) AllowEmptyEnv(allowEmptyEnv bool) { + v.allowEmptyEnv = allowEmptyEnv +} + // TODO: should getEnv logic be moved into find(). Can generalize the use of // rewriting keys many things, Ex: Get('someKey') -> some_key // (camel case to snake case for JSON keys perhaps) @@ -337,11 +391,14 @@ func (v *Viper) mergeWithEnvPrefix(in string) string { // getEnv is a wrapper around os.Getenv which replaces characters in the original // key. This allows env vars which have different keys than the config object // keys. -func (v *Viper) getEnv(key string) string { +func (v *Viper) getEnv(key string) (string, bool) { if v.envKeyReplacer != nil { key = v.envKeyReplacer.Replace(key) } - return os.Getenv(key) + + val, ok := os.LookupEnv(key) + + return val, ok && (v.allowEmptyEnv || val != "") } // ConfigFileUsed returns the file used to populate the config registry. @@ -568,10 +625,9 @@ func (v *Viper) isPathShadowedInFlatMap(path []string, mi interface{}) string { // "foo.bar.baz" in a lower-priority map func (v *Viper) isPathShadowedInAutoEnv(path []string) string { var parentKey string - var val string for i := 1; i < len(path); i++ { parentKey = strings.Join(path[0:i], v.keyDelim) - if val = v.getEnv(v.mergeWithEnvPrefix(parentKey)); val != "" { + if _, ok := v.getEnv(v.mergeWithEnvPrefix(parentKey)); ok { return parentKey } } @@ -631,7 +687,7 @@ func (v *Viper) Get(key string) interface{} { return cast.ToBool(val) case string: return cast.ToString(val) - case int64, int32, int16, int8, int: + case int32, int16, int8, int: return cast.ToInt(val) case uint: return cast.ToUint(val) @@ -639,6 +695,8 @@ func (v *Viper) Get(key string) interface{} { return cast.ToUint32(val) case uint64: return cast.ToUint64(val) + case int64: + return cast.ToInt64(val) case float64, float32: return cast.ToFloat64(val) case time.Time: @@ -769,38 +827,38 @@ func (v *Viper) GetSizeInBytes(key string) uint { } // UnmarshalKey takes a single key and unmarshals it into a Struct. -func UnmarshalKey(key string, rawVal interface{}) error { return v.UnmarshalKey(key, rawVal) } -func (v *Viper) UnmarshalKey(key string, rawVal interface{}) error { - err := decode(v.Get(key), defaultDecoderConfig(rawVal)) +func UnmarshalKey(key string, rawVal interface{}, opts ...DecoderConfigOption) error { + return v.UnmarshalKey(key, rawVal, opts...) +} +func (v *Viper) UnmarshalKey(key string, rawVal interface{}, opts ...DecoderConfigOption) error { + err := decode(v.Get(key), defaultDecoderConfig(rawVal, opts...)) if err != nil { return err } - v.insensitiviseMaps() - return nil } // Unmarshal unmarshals the config into a Struct. Make sure that the tags // on the fields of the structure are properly set. -func Unmarshal(rawVal interface{}) error { return v.Unmarshal(rawVal) } -func (v *Viper) Unmarshal(rawVal interface{}) error { - err := decode(v.AllSettings(), defaultDecoderConfig(rawVal)) +func Unmarshal(rawVal interface{}, opts ...DecoderConfigOption) error { + return v.Unmarshal(rawVal, opts...) +} +func (v *Viper) Unmarshal(rawVal interface{}, opts ...DecoderConfigOption) error { + err := decode(v.AllSettings(), defaultDecoderConfig(rawVal, opts...)) if err != nil { return err } - v.insensitiviseMaps() - return nil } // defaultDecoderConfig returns default mapsstructure.DecoderConfig with suppot // of time.Duration values & string slices -func defaultDecoderConfig(output interface{}) *mapstructure.DecoderConfig { - return &mapstructure.DecoderConfig{ +func defaultDecoderConfig(output interface{}, opts ...DecoderConfigOption) *mapstructure.DecoderConfig { + c := &mapstructure.DecoderConfig{ Metadata: nil, Result: output, WeaklyTypedInput: true, @@ -809,6 +867,10 @@ func defaultDecoderConfig(output interface{}) *mapstructure.DecoderConfig { mapstructure.StringToSliceHookFunc(","), ), } + for _, opt := range opts { + opt(c) + } + return c } // A wrapper around mapstructure.Decode that mimics the WeakDecode functionality @@ -832,8 +894,6 @@ func (v *Viper) UnmarshalExact(rawVal interface{}) error { return err } - v.insensitiviseMaps() - return nil } @@ -964,7 +1024,7 @@ func (v *Viper) find(lcaseKey string) interface{} { if v.automaticEnvApplied { // even if it hasn't been registered, if automaticEnv is used, // check any Get request - if val = v.getEnv(v.mergeWithEnvPrefix(lcaseKey)); val != "" { + if val, ok := v.getEnv(v.mergeWithEnvPrefix(lcaseKey)); ok { return val } if nested && v.isPathShadowedInAutoEnv(path) != "" { @@ -973,7 +1033,7 @@ func (v *Viper) find(lcaseKey string) interface{} { } envkey, exists := v.env[lcaseKey] if exists { - if val = v.getEnv(envkey); val != "" { + if val, ok := v.getEnv(envkey); ok { return val } } @@ -1138,7 +1198,7 @@ func (v *Viper) SetDefault(key string, value interface{}) { deepestMap[lastKey] = value } -// Set sets the value for the key in the override regiser. +// Set sets the value for the key in the override register. // Set is case-insensitive for a key. // Will be used instead of values obtained via // flags, config file, ENV, default, or key/value store. @@ -1219,13 +1279,21 @@ func (v *Viper) ReadConfig(in io.Reader) error { // MergeConfig merges a new configuration with an existing config. func MergeConfig(in io.Reader) error { return v.MergeConfig(in) } func (v *Viper) MergeConfig(in io.Reader) error { - if v.config == nil { - v.config = make(map[string]interface{}) - } cfg := make(map[string]interface{}) if err := v.unmarshalReader(in, cfg); err != nil { return err } + return v.MergeConfigMap(cfg) +} + +// MergeConfigMap merges the configuration from the map given with an existing config. +// Note that the map given may be modified. +func MergeConfigMap(cfg map[string]interface{}) error { return v.MergeConfigMap(cfg) } +func (v *Viper) MergeConfigMap(cfg map[string]interface{}) error { + if v.config == nil { + v.config = make(map[string]interface{}) + } + insensitiviseMap(cfg) mergeMaps(cfg, v.config, nil) return nil } @@ -1286,7 +1354,7 @@ func (v *Viper) writeConfig(filename string, force bool) error { return fmt.Errorf("File: %s exists. Use WriteConfig to overwrite.", filename) } } - f, err := v.fs.OpenFile(filename, flags, os.FileMode(0644)) + f, err := v.fs.OpenFile(filename, flags, v.configPermissions) if err != nil { return err } @@ -1531,13 +1599,6 @@ func (v *Viper) WatchRemoteConfigOnChannel() error { return v.watchKeyValueConfigOnChannel() } -func (v *Viper) insensitiviseMaps() { - insensitiviseMap(v.config) - insensitiviseMap(v.defaults) - insensitiviseMap(v.override) - insensitiviseMap(v.kvstore) -} - // Retrieve the first found remote configuration. func (v *Viper) getKeyValueConfig() error { if RemoteConfig == nil { @@ -1730,6 +1791,12 @@ func (v *Viper) SetConfigType(in string) { } } +// SetConfigPermissions sets the permissions for the config file. +func SetConfigPermissions(perm os.FileMode) { v.SetConfigPermissions(perm) } +func (v *Viper) SetConfigPermissions(perm os.FileMode) { + v.configPermissions = perm.Perm() +} + func (v *Viper) getConfigType() string { if v.configType != "" { return v.configType diff --git a/viper_test.go b/viper_test.go index 745acee..f43a38b 100644 --- a/viper_test.go +++ b/viper_test.go @@ -7,22 +7,29 @@ package viper import ( "bytes" + "encoding/json" "fmt" "io" "io/ioutil" "os" + "os/exec" "path" "reflect" + "runtime" "sort" "strings" + "sync" "testing" "time" + "github.com/fsnotify/fsnotify" + "github.com/mitchellh/mapstructure" "github.com/spf13/afero" "github.com/spf13/cast" "github.com/spf13/pflag" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) var yamlExample = []byte(`Hacker: true @@ -381,6 +388,36 @@ func TestEnv(t *testing.T) { } +func TestEmptyEnv(t *testing.T) { + initJSON() + + BindEnv("type") // Empty environment variable + BindEnv("name") // Bound, but not set environment variable + + os.Clearenv() + + os.Setenv("TYPE", "") + + assert.Equal(t, "donut", Get("type")) + assert.Equal(t, "Cake", Get("name")) +} + +func TestEmptyEnv_Allowed(t *testing.T) { + initJSON() + + AllowEmptyEnv(true) + + BindEnv("type") // Empty environment variable + BindEnv("name") // Bound, but not set environment variable + + os.Clearenv() + + os.Setenv("TYPE", "") + + assert.Equal(t, "", Get("type")) + assert.Equal(t, "Cake", Get("name")) +} + func TestEnvPrefix(t *testing.T) { initJSON() @@ -503,6 +540,42 @@ func TestUnmarshal(t *testing.T) { assert.Equal(t, &config{Name: "Steve", Port: 1234, Duration: time.Second + time.Millisecond}, &C) } +func TestUnmarshalWithDecoderOptions(t *testing.T) { + Set("credentials", "{\"foo\":\"bar\"}") + + opt := DecodeHook(mapstructure.ComposeDecodeHookFunc( + mapstructure.StringToTimeDurationHookFunc(), + mapstructure.StringToSliceHookFunc(","), + // Custom Decode Hook Function + func(rf reflect.Kind, rt reflect.Kind, data interface{}) (interface{}, error) { + if rf != reflect.String || rt != reflect.Map { + return data, nil + } + m := map[string]string{} + raw := data.(string) + if raw == "" { + return m, nil + } + return m, json.Unmarshal([]byte(raw), &m) + }, + )) + + type config struct { + Credentials map[string]string + } + + var C config + + err := Unmarshal(&C, opt) + if err != nil { + t.Fatalf("unable to decode into struct, %v", err) + } + + assert.Equal(t, &config{ + Credentials: map[string]string{"foo": "bar"}, + }, &C) +} + func TestBindPFlags(t *testing.T) { v := New() // create independent Viper object flagSet := pflag.NewFlagSet("test", pflag.ContinueOnError) @@ -540,24 +613,28 @@ func TestBindPFlags(t *testing.T) { } func TestBindPFlagsStringSlice(t *testing.T) { - for _, testValue := range []struct { + tests := []struct { Expected []string Value string }{ - {[]string{}, ""}, + {nil, ""}, {[]string{"jeden"}, "jeden"}, {[]string{"dwa", "trzy"}, "dwa,trzy"}, - {[]string{"cztery", "piec , szesc"}, "cztery,\"piec , szesc\""}} { + {[]string{"cztery", "piec , szesc"}, "cztery,\"piec , szesc\""}, + } + + v := New() // create independent Viper object + defaultVal := []string{"default"} + v.SetDefault("stringslice", defaultVal) + + for _, testValue := range tests { + flagSet := pflag.NewFlagSet("test", pflag.ContinueOnError) + flagSet.StringSlice("stringslice", testValue.Expected, "test") for _, changed := range []bool{true, false} { - v := New() // create independent Viper object - flagSet := pflag.NewFlagSet("test", pflag.ContinueOnError) - flagSet.StringSlice("stringslice", testValue.Expected, "test") - flagSet.Visit(func(f *pflag.Flag) { - if len(testValue.Value) > 0 { - f.Value.Set(testValue.Value) - f.Changed = changed - } + flagSet.VisitAll(func(f *pflag.Flag) { + f.Value.Set(testValue.Value) + f.Changed = changed }) err := v.BindPFlags(flagSet) @@ -572,7 +649,11 @@ func TestBindPFlagsStringSlice(t *testing.T) { if err := v.Unmarshal(val); err != nil { t.Fatalf("%+#v cannot unmarshal: %s", testValue.Value, err) } - assert.Equal(t, testValue.Expected, val.StringSlice) + if changed { + assert.Equal(t, testValue.Expected, val.StringSlice) + } else { + assert.Equal(t, defaultVal, val.StringSlice) + } } } } @@ -1065,10 +1146,6 @@ func TestMergeConfig(t *testing.T) { t.Fatalf("pop != 37890, = %d", pop) } - if pop := v.GetInt("hello.lagrenum"); pop != 765432101234567 { - t.Fatalf("lagrenum != 765432101234567, = %d", pop) - } - if pop := v.GetInt32("hello.pop"); pop != int32(37890) { t.Fatalf("pop != 37890, = %d", pop) } @@ -1105,10 +1182,6 @@ func TestMergeConfig(t *testing.T) { t.Fatalf("pop != 45000, = %d", pop) } - if pop := v.GetInt("hello.lagrenum"); pop != 7654321001234567 { - t.Fatalf("lagrenum != 7654321001234567, = %d", pop) - } - if pop := v.GetInt32("hello.pop"); pop != int32(45000) { t.Fatalf("pop != 45000, = %d", pop) } @@ -1170,6 +1243,48 @@ func TestMergeConfigNoMerge(t *testing.T) { } } +func TestMergeConfigMap(t *testing.T) { + v := New() + v.SetConfigType("yml") + if err := v.ReadConfig(bytes.NewBuffer(yamlMergeExampleTgt)); err != nil { + t.Fatal(err) + } + + assert := func(i int) { + large := v.GetInt("hello.lagrenum") + pop := v.GetInt("hello.pop") + if large != 765432101234567 { + t.Fatal("Got large num:", large) + } + + if pop != i { + t.Fatal("Got pop:", pop) + } + } + + assert(37890) + + update := map[string]interface{}{ + "Hello": map[string]interface{}{ + "Pop": 1234, + }, + "World": map[interface{}]interface{}{ + "Rock": 345, + }, + } + + if err := v.MergeConfigMap(update); err != nil { + t.Fatal(err) + } + + if rock := v.GetInt("world.rock"); rock != 345 { + t.Fatal("Got rock:", rock) + } + + assert(1234) + +} + func TestUnmarshalingWithAliases(t *testing.T) { v := New() v.SetDefault("ID", 1) @@ -1389,6 +1504,111 @@ func doTestCaseInsensitive(t *testing.T, typ, config string) { } +func newViperWithConfigFile(t *testing.T) (*Viper, string, func()) { + watchDir, err := ioutil.TempDir("", "") + require.Nil(t, err) + configFile := path.Join(watchDir, "config.yaml") + err = ioutil.WriteFile(configFile, []byte("foo: bar\n"), 0640) + require.Nil(t, err) + cleanup := func() { + os.RemoveAll(watchDir) + } + v := New() + v.SetConfigFile(configFile) + err = v.ReadInConfig() + require.Nil(t, err) + require.Equal(t, "bar", v.Get("foo")) + return v, configFile, cleanup +} + +func newViperWithSymlinkedConfigFile(t *testing.T) (*Viper, string, string, func()) { + watchDir, err := ioutil.TempDir("", "") + require.Nil(t, err) + dataDir1 := path.Join(watchDir, "data1") + err = os.Mkdir(dataDir1, 0777) + require.Nil(t, err) + realConfigFile := path.Join(dataDir1, "config.yaml") + t.Logf("Real config file location: %s\n", realConfigFile) + err = ioutil.WriteFile(realConfigFile, []byte("foo: bar\n"), 0640) + require.Nil(t, err) + cleanup := func() { + os.RemoveAll(watchDir) + } + // now, symlink the tm `data1` dir to `data` in the baseDir + os.Symlink(dataDir1, path.Join(watchDir, "data")) + // and link the `/datadir1/config.yaml` to `/config.yaml` + configFile := path.Join(watchDir, "config.yaml") + os.Symlink(path.Join(watchDir, "data", "config.yaml"), configFile) + t.Logf("Config file location: %s\n", path.Join(watchDir, "config.yaml")) + // init Viper + v := New() + v.SetConfigFile(configFile) + err = v.ReadInConfig() + require.Nil(t, err) + require.Equal(t, "bar", v.Get("foo")) + return v, watchDir, configFile, cleanup +} + +func TestWatchFile(t *testing.T) { + if runtime.GOOS == "linux" { + // TODO(bep) FIX ME + t.Skip("Skip test on Linux ...") + } + + t.Run("file content changed", func(t *testing.T) { + // given a `config.yaml` file being watched + v, configFile, cleanup := newViperWithConfigFile(t) + defer cleanup() + _, err := os.Stat(configFile) + require.NoError(t, err) + t.Logf("test config file: %s\n", configFile) + wg := sync.WaitGroup{} + wg.Add(1) + v.OnConfigChange(func(in fsnotify.Event) { + t.Logf("config file changed") + wg.Done() + }) + v.WatchConfig() + // when overwriting the file and waiting for the custom change notification handler to be triggered + err = ioutil.WriteFile(configFile, []byte("foo: baz\n"), 0640) + wg.Wait() + // then the config value should have changed + require.Nil(t, err) + assert.Equal(t, "baz", v.Get("foo")) + }) + + t.Run("link to real file changed (à la Kubernetes)", func(t *testing.T) { + // skip if not executed on Linux + if runtime.GOOS != "linux" { + t.Skipf("Skipping test as symlink replacements don't work on non-linux environment...") + } + v, watchDir, _, _ := newViperWithSymlinkedConfigFile(t) + // defer cleanup() + wg := sync.WaitGroup{} + v.WatchConfig() + v.OnConfigChange(func(in fsnotify.Event) { + t.Logf("config file changed") + wg.Done() + }) + wg.Add(1) + // when link to another `config.yaml` file + dataDir2 := path.Join(watchDir, "data2") + err := os.Mkdir(dataDir2, 0777) + require.Nil(t, err) + configFile2 := path.Join(dataDir2, "config.yaml") + err = ioutil.WriteFile(configFile2, []byte("foo: baz\n"), 0640) + require.Nil(t, err) + // change the symlink using the `ln -sfn` command + err = exec.Command("ln", "-sfn", dataDir2, path.Join(watchDir, "data")).Run() + require.Nil(t, err) + wg.Wait() + // then + require.Nil(t, err) + assert.Equal(t, "baz", v.Get("foo")) + }) + +} + func BenchmarkGetBool(b *testing.B) { key := "BenchmarkGetBool" v = New()