From f070baa3b309781d7800296581836ce3ea74a451 Mon Sep 17 00:00:00 2001 From: Jeffrey DeFond Date: Wed, 10 Apr 2019 17:05:11 -0700 Subject: [PATCH] Revert "Merge branch 'master' of https://github.com/spf13/viper into feature/issue-175" This reverts commit e6afb3cfc843d3ee9a477c576acce19ce5b981f9, reversing changes made to b80be0b847fc216498643429e3bcc7a685bc454c. --- .gitignore | 7 +- .travis.yml | 10 +- README.md | 106 ++------- flags_test.go | 1 + go.mod | 24 -- go.sum | 35 --- nohup.out | 1 + remote/remote.go | 42 +--- util.go | 67 +++++- util_test.go | 1 + viper.go | 563 +++++++++-------------------------------------- viper_test.go | 505 +----------------------------------------- 12 files changed, 209 insertions(+), 1153 deletions(-) delete mode 100644 go.mod delete mode 100644 go.sum create mode 100644 nohup.out diff --git a/.gitignore b/.gitignore index 01b5c44..352a34a 100644 --- a/.gitignore +++ b/.gitignore @@ -21,9 +21,4 @@ _testmain.go *.exe *.test -*.bench - -.vscode - -# exclude dependencies in the `/vendor` folder -vendor +*.bench \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index bb83057..e793edb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,13 +1,10 @@ go_import_path: github.com/spf13/viper language: go - -env: - global: - - GO111MODULE="on" - go: - - 1.11.x + - 1.5.4 + - 1.6.3 + - 1.7 - tip os: @@ -21,7 +18,6 @@ matrix: script: - go install ./... - - diff -u <(echo -n) <(gofmt -d .) - go test -v ./... after_success: diff --git a/README.md b/README.md index 0208eac..25181df 100644 --- a/README.md +++ b/README.md @@ -6,19 +6,18 @@ Many Go projects are built using Viper including: * [Hugo](http://gohugo.io) * [EMC RexRay](http://rexray.readthedocs.org/en/stable/) -* [Imgur’s Incus](https://github.com/Imgur/incus) +* [Imgur's Incus](https://github.com/Imgur/incus) * [Nanobox](https://github.com/nanobox-io/nanobox)/[Nanopack](https://github.com/nanopack) * [Docker Notary](https://github.com/docker/Notary) * [BloomApi](https://www.bloomapi.com/) * [doctl](https://github.com/digitalocean/doctl) -* [Clairctl](https://github.com/jgsqware/clairctl) [![Build Status](https://travis-ci.org/spf13/viper.svg)](https://travis-ci.org/spf13/viper) [![Join the chat at https://gitter.im/spf13/viper](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/spf13/viper?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) [![GoDoc](https://godoc.org/github.com/spf13/viper?status.svg)](https://godoc.org/github.com/spf13/viper) ## What is Viper? -Viper is a complete configuration solution for Go applications including 12-Factor apps. It is designed +Viper is a complete configuration solution for go applications including 12 factor apps. It is designed to work within an application, and can handle all types of configuration needs and formats. It supports: @@ -69,7 +68,7 @@ Viper configuration keys are case insensitive. ### Establishing Defaults A good configuration system will support default values. A default value is not -required for a key, but it’s useful in the event that a key hasn’t been set via +required for a key, but it's useful in the event that a key hasn’t been set via config file, environment variable, remote configuration or flag. Examples: @@ -117,10 +116,10 @@ Optionally you can provide a function for Viper to run each time a change occurs **Make sure you add all of the configPaths prior to calling `WatchConfig()`** ```go -viper.WatchConfig() -viper.OnConfigChange(func(e fsnotify.Event) { - fmt.Println("Config file changed:", e.Name) -}) + viper.WatchConfig() + viper.OnConfigChange(func(e fsnotify.Event) { + fmt.Println("Config file changed:", e.Name) + }) ``` ### Reading Config from io.Reader @@ -179,20 +178,19 @@ viper.GetBool("verbose") // true ### Working with Environment Variables Viper has full support for environment variables. This enables 12 factor -applications out of the box. There are five methods that exist to aid working +applications out of the box. There are four methods that exist to aid working with ENV: * `AutomaticEnv()` * `BindEnv(string...) : error` * `SetEnvPrefix(string)` - * `SetEnvKeyReplacer(string...) *strings.Replacer` - * `AllowEmptyEnvVar(bool)` + * `SetEnvReplacer(string...) *strings.Replacer` _When working with ENV variables, it’s important to recognize that Viper treats ENV variables as case sensitive._ Viper provides a mechanism to try to ensure that ENV variables are unique. By -using `SetEnvPrefix`, you can tell Viper to use a prefix while reading from +using `SetEnvPrefix`, you can tell Viper to use add a prefix while reading from the environment variables. Both `BindEnv` and `AutomaticEnv` will use this prefix. @@ -213,15 +211,11 @@ time a `viper.Get` request is made. It will apply the following rules. It will check for a environment variable with a name matching the key uppercased and prefixed with the `EnvPrefix` if set. -`SetEnvKeyReplacer` allows you to use a `strings.Replacer` object to rewrite Env +`SetEnvReplacer` allows you to use a `strings.Replacer` object to rewrite Env keys to an extent. This is useful if you want to use `-` or something in your `Get()` calls, but want your environmental variables to use `_` delimiters. An example of using it can be found in `viper_test.go`. -By default empty environment variables are considered unset and will fall back to -the next configuration source. To treat empty environment variables as set, use -the `AllowEmptyEnv` method. - #### Env example ```go @@ -242,7 +236,7 @@ Like `BindEnv`, the value is not set when the binding method is called, but when it is accessed. This means you can bind as early as you want, even in an `init()` function. -For individual flags, the `BindPFlag()` method provides this functionality. +The `BindPFlag()` method provides this functionality. Example: @@ -251,19 +245,6 @@ serverCmd.Flags().Int("port", 1138, "Port to run Application server on") viper.BindPFlag("port", serverCmd.Flags().Lookup("port")) ``` -You can also bind an existing set of pflags (pflag.FlagSet): - -Example: - -```go -pflag.Int("flagname", 1234, "help message for flagname") - -pflag.Parse() -viper.BindPFlags(pflag.CommandLine) - -i := viper.GetInt("flagname") // retrieve values from viper instead of pflag -``` - The use of [pflag](https://github.com/spf13/pflag/) in Viper does not preclude the use of other packages that use the [flag](https://golang.org/pkg/flag/) package from the standard library. The pflag package can handle the flags @@ -282,23 +263,15 @@ import ( ) func main() { - - // using standard library "flag" package - flag.Int("flagname", 1234, "help message for flagname") - pflag.CommandLine.AddGoFlagSet(flag.CommandLine) pflag.Parse() - viper.BindPFlags(pflag.CommandLine) - - i := viper.GetInt("flagname") // retrieve value from viper - - ... + ... } ``` #### Flag interfaces -Viper provides two Go interfaces to bind other flag systems if you don’t use `Pflags`. +Viper provides two Go interfaces to bind other flag systems if you don't use `Pflags`. `FlagValue` represents a single flag. This is a very simple example on how to implement this interface: @@ -378,33 +351,12 @@ how to use Consul. ### Remote Key/Value Store Example - Unencrypted -#### etcd ```go viper.AddRemoteProvider("etcd", "http://127.0.0.1:4001","/config/hugo.json") viper.SetConfigType("json") // because there is no file extension in a stream of bytes, supported extensions are "json", "toml", "yaml", "yml", "properties", "props", "prop" err := viper.ReadRemoteConfig() ``` -#### Consul -You need to set a key to Consul key/value storage with JSON value containing your desired config. -For example, create a Consul key/value store key `MY_CONSUL_KEY` with value: - -```json -{ - "port": 8080, - "hostname": "myhostname.com" -} -``` - -```go -viper.AddRemoteProvider("consul", "localhost:8500", "MY_CONSUL_KEY") -viper.SetConfigType("json") // Need to explicitly set this to json -err := viper.ReadRemoteConfig() - -fmt.Println(viper.Get("port")) // 8080 -fmt.Println(viper.Get("hostname")) // myhostname.com -``` - ### Remote Key/Value Store Example - Encrypted ```go @@ -449,7 +401,7 @@ go func(){ ## Getting Values From Viper -In Viper, there are a few ways to get a value depending on the value’s type. +In Viper, there are a few ways to get a value depending on the value's type. The following functions and methods exist: * `Get(key string) : interface{}` @@ -463,7 +415,6 @@ The following functions and methods exist: * `GetTime(key string) : time.Time` * `GetDuration(key string) : time.Duration` * `IsSet(key string) : bool` - * `AllSettings() : map[string]interface{}` One important thing to recognize is that each Get function will return a zero value if it’s not found. To check if a given key exists, the `IsSet()` method @@ -580,7 +531,7 @@ func NewCache(cfg *Viper) *Cache {...} ``` which creates a cache based on config information formatted as `subv`. -Now it’s easy to create these 2 caches separately as: +Now it's easy to create these 2 caches separately as: ```go cfg1 := viper.Sub("app.cache1") @@ -617,27 +568,6 @@ if err != nil { } ``` -### Marshalling to string - -You may need to marhsal all the settings held in viper into a string rather than write them to a file. -You can use your favorite format's marshaller with the config returned by `AllSettings()`. - -```go -import ( - yaml "gopkg.in/yaml.v2" - // ... -) - -func yamlStringSettings() string { - c := viper.AllSettings() - bs, err := yaml.Marshal(c) - if err != nil { - t.Fatalf("unable to marshal config to YAML: %v", err) - } - return string(bs) -} -``` - ## Viper or Vipers? Viper comes ready to use out of the box. There is no configuration or @@ -645,13 +575,13 @@ initialization needed to begin using Viper. Since most applications will want to use a single central repository for their configuration, the viper package provides this. It is similar to a singleton. -In all of the examples above, they demonstrate using viper in its singleton +In all of the examples above, they demonstrate using viper in it's singleton style approach. ### Working with multiple vipers You can also create many different vipers for use in your application. Each will -have its own unique set of configurations and values. Each can read from a +have it’s own unique set of configurations and values. Each can read from a different config file, key value store, etc. All of the functions that viper package supports are mirrored as methods on a viper. diff --git a/flags_test.go b/flags_test.go index 0b976b6..5bffca3 100644 --- a/flags_test.go +++ b/flags_test.go @@ -62,4 +62,5 @@ func TestBindFlagValue(t *testing.T) { flag.Changed = true //hack for pflag usage assert.Equal(t, "testing_mutate", Get("testvalue")) + } diff --git a/go.mod b/go.mod deleted file mode 100644 index 86e801c..0000000 --- a/go.mod +++ /dev/null @@ -1,24 +0,0 @@ -module github.com/spf13/viper - -require ( - github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6 // indirect - github.com/coreos/etcd v3.3.10+incompatible // indirect - github.com/coreos/go-etcd v2.0.0+incompatible // indirect - github.com/coreos/go-semver v0.2.0 // indirect - github.com/fsnotify/fsnotify v1.4.7 - github.com/hashicorp/hcl v1.0.0 - github.com/magiconair/properties v1.8.0 - github.com/mitchellh/mapstructure v1.1.2 - github.com/pelletier/go-toml v1.2.0 - github.com/spf13/afero v1.1.2 - github.com/spf13/cast v1.3.0 - github.com/spf13/jwalterweatherman v1.0.0 - github.com/spf13/pflag v1.0.3 - github.com/stretchr/testify v1.2.2 - github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8 // indirect - github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77 - golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9 // indirect - golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a // indirect - golang.org/x/text v0.3.0 // indirect - gopkg.in/yaml.v2 v2.2.2 -) diff --git a/go.sum b/go.sum deleted file mode 100644 index 5c9fb7d..0000000 --- a/go.sum +++ /dev/null @@ -1,35 +0,0 @@ -github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= -github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= -github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY= -github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= -github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= -github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= -github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a h1:1n5lsVfiQW3yfsRGu98756EH1YthsFqr/5mxHduZW2A= -golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/nohup.out b/nohup.out new file mode 100644 index 0000000..8973bf2 --- /dev/null +++ b/nohup.out @@ -0,0 +1 @@ +QProcess::start: Process is already running diff --git a/remote/remote.go b/remote/remote.go index 810d070..faaf3b3 100644 --- a/remote/remote.go +++ b/remote/remote.go @@ -8,11 +8,10 @@ package remote import ( "bytes" - "io" - "os" - "github.com/spf13/viper" crypt "github.com/xordataexchange/crypt/config" + "io" + "os" ) type remoteConfigProvider struct{} @@ -34,45 +33,17 @@ func (rc remoteConfigProvider) Watch(rp viper.RemoteProvider) (io.Reader, error) if err != nil { return nil, err } - resp, err := cm.Get(rp.Path()) + resp := <-cm.Watch(rp.Path(), nil) + err = resp.Error if err != nil { return nil, err } - return bytes.NewReader(resp), nil -} - -func (rc remoteConfigProvider) WatchChannel(rp viper.RemoteProvider) (<-chan *viper.RemoteResponse, chan bool) { - cm, err := getConfigManager(rp) - if err != nil { - return nil, nil - } - quit := make(chan bool) - quitwc := make(chan bool) - viperResponsCh := make(chan *viper.RemoteResponse) - cryptoResponseCh := cm.Watch(rp.Path(), quit) - // need this function to convert the Channel response form crypt.Response to viper.Response - go func(cr <-chan *crypt.Response, vr chan<- *viper.RemoteResponse, quitwc <-chan bool, quit chan<- bool) { - for { - select { - case <-quitwc: - quit <- true - return - case resp := <-cr: - vr <- &viper.RemoteResponse{ - Error: resp.Error, - Value: resp.Value, - } - - } - - } - }(cryptoResponseCh, viperResponsCh, quitwc, quit) - - return viperResponsCh, quitwc + return bytes.NewReader(resp.Value), nil } func getConfigManager(rp viper.RemoteProvider) (crypt.ConfigManager, error) { + var cm crypt.ConfigManager var err error @@ -98,6 +69,7 @@ func getConfigManager(rp viper.RemoteProvider) (crypt.ConfigManager, error) { return nil, err } return cm, nil + } func init() { diff --git a/util.go b/util.go index 952cad4..3ebada9 100644 --- a/util.go +++ b/util.go @@ -11,16 +11,22 @@ package viper import ( + "bytes" + "encoding/json" "fmt" + "io" "os" "path/filepath" "runtime" "strings" "unicode" - "github.com/spf13/afero" + "github.com/hashicorp/hcl" + "github.com/magiconair/properties" + toml "github.com/pelletier/go-toml" "github.com/spf13/cast" jww "github.com/spf13/jwalterweatherman" + "gopkg.in/yaml.v2" ) // ConfigParseError denotes failing to parse configuration file. @@ -115,8 +121,8 @@ func absPathify(inPath string) string { } // Check if File / Directory Exists -func exists(fs afero.Fs, path string) (bool, error) { - _, err := fs.Stat(path) +func exists(path string) (bool, error) { + _, err := v.fs.Stat(path) if err == nil { return true, nil } @@ -146,6 +152,61 @@ func userHomeDir() string { return os.Getenv("HOME") } +func unmarshallConfigReader(in io.Reader, c map[string]interface{}, configType string) error { + buf := new(bytes.Buffer) + buf.ReadFrom(in) + + switch strings.ToLower(configType) { + case "yaml", "yml": + if err := yaml.Unmarshal(buf.Bytes(), &c); err != nil { + return ConfigParseError{err} + } + + case "json": + if err := json.Unmarshal(buf.Bytes(), &c); err != nil { + return ConfigParseError{err} + } + + case "hcl": + obj, err := hcl.Parse(string(buf.Bytes())) + if err != nil { + return ConfigParseError{err} + } + if err = hcl.DecodeObject(&c, obj); err != nil { + return ConfigParseError{err} + } + + case "toml": + tree, err := toml.LoadReader(buf) + if err != nil { + return ConfigParseError{err} + } + tmap := tree.ToMap() + for k, v := range tmap { + c[k] = v + } + + case "properties", "props", "prop": + var p *properties.Properties + var err error + if p, err = properties.Load(buf.Bytes(), properties.UTF8); err != nil { + return ConfigParseError{err} + } + for _, key := range p.Keys() { + value, _ := p.Get(key) + // recursively build nested maps + path := strings.Split(key, ".") + lastKey := strings.ToLower(path[len(path)-1]) + deepestMap := deepSearch(c, path[0:len(path)-1]) + // set innermost value + deepestMap[lastKey] = value + } + } + + insensitiviseMap(c) + return nil +} + func safeMul(a, b uint) uint { c := a * b if a > 1 && b > 1 && c/b != a { diff --git a/util_test.go b/util_test.go index 0af80bb..5949e09 100644 --- a/util_test.go +++ b/util_test.go @@ -16,6 +16,7 @@ import ( ) func TestCopyAndInsensitiviseMap(t *testing.T) { + var ( given = map[string]interface{}{ "Foo": 32, diff --git a/viper.go b/viper.go index a3d37f8..1bc46f5 100644 --- a/viper.go +++ b/viper.go @@ -21,8 +21,6 @@ package viper import ( "bytes" - "encoding/csv" - "encoding/json" "fmt" "io" "log" @@ -30,40 +28,18 @@ import ( "path/filepath" "reflect" "strings" - "sync" "time" - yaml "gopkg.in/yaml.v2" - "github.com/fsnotify/fsnotify" - "github.com/hashicorp/hcl" - "github.com/hashicorp/hcl/hcl/printer" - "github.com/magiconair/properties" "github.com/mitchellh/mapstructure" - toml "github.com/pelletier/go-toml" "github.com/spf13/afero" "github.com/spf13/cast" jww "github.com/spf13/jwalterweatherman" "github.com/spf13/pflag" ) -// ConfigMarshalError happens when failing to marshal the configuration. -type ConfigMarshalError struct { - err error -} - -// Error returns the formatted configuration error. -func (e ConfigMarshalError) Error() string { - return fmt.Sprintf("While marshaling config: %s", e.err.Error()) -} - var v *Viper -type RemoteResponse struct { - Value []byte - Error error -} - func init() { v = New() } @@ -71,7 +47,6 @@ func init() { type remoteConfigFactory interface { Get(rp RemoteProvider) (io.Reader, error) Watch(rp RemoteProvider) (io.Reader, error) - WatchChannel(rp RemoteProvider) (<-chan *RemoteResponse, chan bool) } // RemoteConfig is optional, see the remote package @@ -87,7 +62,8 @@ func (str UnsupportedConfigError) Error() string { } // UnsupportedRemoteProviderError denotes encountering an unsupported remote -// provider. Currently only etcd and Consul are supported. +// provider. Currently only etcd and Consul are +// supported. type UnsupportedRemoteProviderError string // Error returns the formatted remote provider error. @@ -114,23 +90,6 @@ func (fnfe ConfigFileNotFoundError) Error() string { return fmt.Sprintf("Config File %q Not Found in %q", fnfe.name, fnfe.locations) } -// A DecoderConfigOption can be passed to viper.Unmarshal to configure -// mapstructure.DecoderConfig options -type DecoderConfigOption func(*mapstructure.DecoderConfig) - -// DecodeHook returns a DecoderConfigOption which overrides the default -// DecoderConfig.DecodeHook value, the default is: -// -// mapstructure.ComposeDecodeHookFunc( -// mapstructure.StringToTimeDurationHookFunc(), -// mapstructure.StringToSliceHookFunc(","), -// ) -func DecodeHook(hook mapstructure.DecodeHookFunc) DecoderConfigOption { - return func(c *mapstructure.DecoderConfig) { - c.DecodeHook = hook - } -} - // Viper is a prioritized configuration registry. It // maintains a set of configuration sources, fetches // values to populate those, and provides them according @@ -180,15 +139,13 @@ type Viper struct { remoteProviders []*defaultRemoteProvider // Name of file to look for inside the path - configName string - configFile string - configType string - configPermissions os.FileMode - envPrefix string + configName string + configFile string + configType string + envPrefix string automaticEnvApplied bool envKeyReplacer *strings.Replacer - allowEmptyEnv bool config map[string]interface{} override map[string]interface{} @@ -199,10 +156,6 @@ type Viper struct { aliases map[string]string typeByDefValue bool - // Store read properties on the object so that we can write back in order with comments. - // This will only be used if the configuration read is a properties file. - properties *properties.Properties - onConfigChange func(fsnotify.Event) } @@ -211,7 +164,6 @@ func New() *Viper { v := new(Viper) v.keyDelim = "." v.configName = "config" - v.configPermissions = os.FileMode(0644) v.fs = afero.NewOsFs() v.config = make(map[string]interface{}) v.override = make(map[string]interface{}) @@ -230,7 +182,7 @@ func New() *Viper { // can use it in their testing as well. func Reset() { v = New() - SupportedExts = []string{"json", "toml", "yaml", "yml", "properties", "props", "prop", "hcl"} + SupportedExts = []string{"json", "toml", "yaml", "yml", "hcl"} SupportedRemoteProviders = []string{"etcd", "consul"} } @@ -280,77 +232,54 @@ func (v *Viper) OnConfigChange(run func(in fsnotify.Event)) { } func WatchConfig() { v.WatchConfig() } - func (v *Viper) WatchConfig() { - initWG := sync.WaitGroup{} - initWG.Add(1) go func() { watcher, err := fsnotify.NewWatcher() if err != nil { log.Fatal(err) } defer watcher.Close() + // we have to watch the entire directory to pick up renames/atomic saves in a cross-platform way filename, err := v.getConfigFile() if err != nil { - log.Printf("error: %v\n", err) + log.Println("error:", err) return } configFile := filepath.Clean(filename) configDir, _ := filepath.Split(configFile) - realConfigFile, _ := filepath.EvalSymlinks(filename) - eventsWG := sync.WaitGroup{} - eventsWG.Add(1) + done := make(chan bool) go func() { for { select { - case event, ok := <-watcher.Events: - if !ok { // 'Events' channel is closed - eventsWG.Done() - return - } - currentConfigFile, _ := filepath.EvalSymlinks(filename) - // we only care about the config file with the following cases: - // 1 - if the config file was modified or created - // 2 - if the real path to the config file changed (eg: k8s ConfigMap replacement) - const writeOrCreateMask = fsnotify.Write | fsnotify.Create - if (filepath.Clean(event.Name) == configFile && - event.Op&writeOrCreateMask != 0) || - (currentConfigFile != "" && currentConfigFile != realConfigFile) { - realConfigFile = currentConfigFile - err := v.ReadInConfig() - if err != nil { - log.Printf("error reading config file: %v\n", err) + case event := <-watcher.Events: + // we only care about the config file + if filepath.Clean(event.Name) == configFile { + if event.Op&fsnotify.Write == fsnotify.Write || event.Op&fsnotify.Create == fsnotify.Create { + err := v.ReadInConfig() + if err != nil { + log.Println("error:", err) + } + if v.onConfigChange != nil { + v.onConfigChange(event) + } } - if v.onConfigChange != nil { - v.onConfigChange(event) - } - } else if filepath.Clean(event.Name) == configFile && - event.Op&fsnotify.Remove&fsnotify.Remove != 0 { - eventsWG.Done() - return } - - case err, ok := <-watcher.Errors: - if ok { // 'Errors' channel is not closed - log.Printf("watcher error: %v\n", err) - } - eventsWG.Done() - return + case err := <-watcher.Errors: + log.Println("error:", err) } } }() + watcher.Add(configDir) - initWG.Done() // done initalizing the watch in this go routine, so the parent routine can move on... - eventsWG.Wait() // now, wait for event loop to end in this go-routine... + <-done }() - initWG.Wait() // make sure that the go routine above fully ended before returning } -// SetConfigFile explicitly defines the path, name and extension of the config file. -// Viper will use this and not check any of the config paths. +// SetConfigFile explicitly defines the path, name and extension of the config file +// Viper will use this and not check any of the config paths func SetConfigFile(in string) { v.SetConfigFile(in) } func (v *Viper) SetConfigFile(in string) { if in != "" { @@ -359,8 +288,8 @@ func (v *Viper) SetConfigFile(in string) { } // SetEnvPrefix defines a prefix that ENVIRONMENT variables will use. -// E.g. if your prefix is "spf", the env registry will look for env -// variables that start with "SPF_". +// E.g. if your prefix is "spf", the env registry +// will look for env. variables that start with "SPF_" func SetEnvPrefix(in string) { v.SetEnvPrefix(in) } func (v *Viper) SetEnvPrefix(in string) { if in != "" { @@ -376,32 +305,21 @@ func (v *Viper) mergeWithEnvPrefix(in string) string { return strings.ToUpper(in) } -// AllowEmptyEnv tells Viper to consider set, -// but empty environment variables as valid values instead of falling back. -// For backward compatibility reasons this is false by default. -func AllowEmptyEnv(allowEmptyEnv bool) { v.AllowEmptyEnv(allowEmptyEnv) } -func (v *Viper) AllowEmptyEnv(allowEmptyEnv bool) { - v.allowEmptyEnv = allowEmptyEnv -} - // TODO: should getEnv logic be moved into find(). Can generalize the use of // rewriting keys many things, Ex: Get('someKey') -> some_key -// (camel case to snake case for JSON keys perhaps) +// (cammel case to snake case for JSON keys perhaps) // getEnv is a wrapper around os.Getenv which replaces characters in the original -// key. This allows env vars which have different keys than the config object -// keys. -func (v *Viper) getEnv(key string) (string, bool) { +// key. This allows env vars which have different keys then the config object +// keys +func (v *Viper) getEnv(key string) string { if v.envKeyReplacer != nil { key = v.envKeyReplacer.Replace(key) } - - val, ok := os.LookupEnv(key) - - return val, ok && (v.allowEmptyEnv || val != "") + return os.Getenv(key) } -// ConfigFileUsed returns the file used to populate the config registry. +// ConfigFileUsed returns the file used to populate the config registry func ConfigFileUsed() string { return v.ConfigFileUsed() } func (v *Viper) ConfigFileUsed() string { return v.configFile } @@ -625,9 +543,10 @@ func (v *Viper) isPathShadowedInFlatMap(path []string, mi interface{}) string { // "foo.bar.baz" in a lower-priority map func (v *Viper) isPathShadowedInAutoEnv(path []string) string { var parentKey string + var val string for i := 1; i < len(path); i++ { parentKey = strings.Join(path[0:i], v.keyDelim) - if _, ok := v.getEnv(v.mergeWithEnvPrefix(parentKey)); ok { + if val = v.getEnv(v.mergeWithEnvPrefix(parentKey)); val != "" { return parentKey } } @@ -673,41 +592,32 @@ func (v *Viper) Get(key string) interface{} { return nil } + valType := val if v.typeByDefValue { // TODO(bep) this branch isn't covered by a single test. - valType := val path := strings.Split(lcaseKey, v.keyDelim) defVal := v.searchMap(v.defaults, path) if defVal != nil { valType = defVal } - - switch valType.(type) { - case bool: - return cast.ToBool(val) - case string: - return cast.ToString(val) - case int32, int16, int8, int: - return cast.ToInt(val) - case uint: - return cast.ToUint(val) - case uint32: - return cast.ToUint32(val) - case uint64: - return cast.ToUint64(val) - case int64: - return cast.ToInt64(val) - case float64, float32: - return cast.ToFloat64(val) - case time.Time: - return cast.ToTime(val) - case time.Duration: - return cast.ToDuration(val) - case []string: - return cast.ToStringSlice(val) - } } + switch valType.(type) { + case bool: + return cast.ToBool(val) + case string: + return cast.ToString(val) + case int64, int32, int16, int8, int: + return cast.ToInt(val) + case float64, float32: + return cast.ToFloat64(val) + case time.Time: + return cast.ToTime(val) + case time.Duration: + return cast.ToDuration(val) + case []string: + return cast.ToStringSlice(val) + } return val } @@ -746,36 +656,12 @@ func (v *Viper) GetInt(key string) int { return cast.ToInt(v.Get(key)) } -// GetInt32 returns the value associated with the key as an integer. -func GetInt32(key string) int32 { return v.GetInt32(key) } -func (v *Viper) GetInt32(key string) int32 { - return cast.ToInt32(v.Get(key)) -} - // GetInt64 returns the value associated with the key as an integer. func GetInt64(key string) int64 { return v.GetInt64(key) } func (v *Viper) GetInt64(key string) int64 { return cast.ToInt64(v.Get(key)) } -// GetUint returns the value associated with the key as an unsigned integer. -func GetUint(key string) uint { return v.GetUint(key) } -func (v *Viper) GetUint(key string) uint { - return cast.ToUint(v.Get(key)) -} - -// GetUint32 returns the value associated with the key as an unsigned integer. -func GetUint32(key string) uint32 { return v.GetUint32(key) } -func (v *Viper) GetUint32(key string) uint32 { - return cast.ToUint32(v.Get(key)) -} - -// GetUint64 returns the value associated with the key as an unsigned integer. -func GetUint64(key string) uint64 { return v.GetUint64(key) } -func (v *Viper) GetUint64(key string) uint64 { - return cast.ToUint64(v.Get(key)) -} - // GetFloat64 returns the value associated with the key as a float64. func GetFloat64(key string) float64 { return v.GetFloat64(key) } func (v *Viper) GetFloat64(key string) float64 { @@ -827,50 +713,35 @@ func (v *Viper) GetSizeInBytes(key string) uint { } // UnmarshalKey takes a single key and unmarshals it into a Struct. -func UnmarshalKey(key string, rawVal interface{}, opts ...DecoderConfigOption) error { - return v.UnmarshalKey(key, rawVal, opts...) -} -func (v *Viper) UnmarshalKey(key string, rawVal interface{}, opts ...DecoderConfigOption) error { - err := decode(v.Get(key), defaultDecoderConfig(rawVal, opts...)) - - if err != nil { - return err - } - - return nil +func UnmarshalKey(key string, rawVal interface{}) error { return v.UnmarshalKey(key, rawVal) } +func (v *Viper) UnmarshalKey(key string, rawVal interface{}) error { + return mapstructure.Decode(v.Get(key), rawVal) } // Unmarshal unmarshals the config into a Struct. Make sure that the tags // on the fields of the structure are properly set. -func Unmarshal(rawVal interface{}, opts ...DecoderConfigOption) error { - return v.Unmarshal(rawVal, opts...) -} -func (v *Viper) Unmarshal(rawVal interface{}, opts ...DecoderConfigOption) error { - err := decode(v.AllSettings(), defaultDecoderConfig(rawVal, opts...)) +func Unmarshal(rawVal interface{}) error { return v.Unmarshal(rawVal) } +func (v *Viper) Unmarshal(rawVal interface{}) error { + err := decode(v.AllSettings(), defaultDecoderConfig(rawVal)) if err != nil { return err } + v.insensitiviseMaps() + return nil } // defaultDecoderConfig returns default mapsstructure.DecoderConfig with suppot -// of time.Duration values & string slices -func defaultDecoderConfig(output interface{}, opts ...DecoderConfigOption) *mapstructure.DecoderConfig { - c := &mapstructure.DecoderConfig{ +// of time.Duration values +func defaultDecoderConfig(output interface{}) *mapstructure.DecoderConfig { + return &mapstructure.DecoderConfig{ Metadata: nil, Result: output, WeaklyTypedInput: true, - DecodeHook: mapstructure.ComposeDecodeHookFunc( - mapstructure.StringToTimeDurationHookFunc(), - mapstructure.StringToSliceHookFunc(","), - ), + DecodeHook: mapstructure.StringToTimeDurationHookFunc(), } - for _, opt := range opts { - opt(c) - } - return c } // A wrapper around mapstructure.Decode that mimics the WeakDecode functionality @@ -894,6 +765,8 @@ func (v *Viper) UnmarshalExact(rawVal interface{}) error { return err } + v.insensitiviseMaps() + return nil } @@ -928,7 +801,7 @@ func (v *Viper) BindFlagValues(flags FlagValueSet) (err error) { } // BindFlagValue binds a specific key to a FlagValue. -// Example (where serverCmd is a Cobra instance): +// Example(where serverCmd is a Cobra instance): // // serverCmd.Flags().Int("port", 1138, "Port to run Application server on") // Viper.BindFlagValue("port", serverCmd.Flags().Lookup("port")) @@ -1009,9 +882,7 @@ func (v *Viper) find(lcaseKey string) interface{} { return cast.ToBool(flag.ValueString()) case "stringSlice": s := strings.TrimPrefix(flag.ValueString(), "[") - s = strings.TrimSuffix(s, "]") - res, _ := readAsCSV(s) - return res + return strings.TrimSuffix(s, "]") default: return flag.ValueString() } @@ -1024,7 +895,7 @@ func (v *Viper) find(lcaseKey string) interface{} { if v.automaticEnvApplied { // even if it hasn't been registered, if automaticEnv is used, // check any Get request - if val, ok := v.getEnv(v.mergeWithEnvPrefix(lcaseKey)); ok { + if val = v.getEnv(v.mergeWithEnvPrefix(lcaseKey)); val != "" { return val } if nested && v.isPathShadowedInAutoEnv(path) != "" { @@ -1033,7 +904,7 @@ func (v *Viper) find(lcaseKey string) interface{} { } envkey, exists := v.env[lcaseKey] if exists { - if val, ok := v.getEnv(envkey); ok { + if val = v.getEnv(envkey); val != "" { return val } } @@ -1078,9 +949,7 @@ func (v *Viper) find(lcaseKey string) interface{} { return cast.ToBool(flag.ValueString()) case "stringSlice": s := strings.TrimPrefix(flag.ValueString(), "[") - s = strings.TrimSuffix(s, "]") - res, _ := readAsCSV(s) - return res + return strings.TrimSuffix(s, "]") default: return flag.ValueString() } @@ -1090,15 +959,6 @@ func (v *Viper) find(lcaseKey string) interface{} { return nil } -func readAsCSV(val string) ([]string, error) { - if val == "" { - return []string{}, nil - } - stringReader := strings.NewReader(val) - csvReader := csv.NewReader(stringReader) - return csvReader.Read() -} - // IsSet checks to see if the key has been set in any of the data locations. // IsSet is case-insensitive for a key. func IsSet(key string) bool { return v.IsSet(key) } @@ -1198,7 +1058,7 @@ func (v *Viper) SetDefault(key string, value interface{}) { deepestMap[lastKey] = value } -// Set sets the value for the key in the override register. +// Set sets the value for the key in the override regiser. // Set is case-insensitive for a key. // Will be used instead of values obtained via // flags, config file, ENV, default, or key/value store. @@ -1230,21 +1090,14 @@ func (v *Viper) ReadInConfig() error { return UnsupportedConfigError(v.getConfigType()) } - jww.DEBUG.Println("Reading file: ", filename) file, err := afero.ReadFile(v.fs, filename) if err != nil { return err } - config := make(map[string]interface{}) + v.config = make(map[string]interface{}) - err = v.unmarshalReader(bytes.NewReader(file), config) - if err != nil { - return err - } - - v.config = config - return nil + return v.unmarshalReader(bytes.NewReader(file), v.config) } // MergeInConfig merges a new configuration with an existing config. @@ -1279,214 +1132,17 @@ func (v *Viper) ReadConfig(in io.Reader) error { // MergeConfig merges a new configuration with an existing config. func MergeConfig(in io.Reader) error { return v.MergeConfig(in) } func (v *Viper) MergeConfig(in io.Reader) error { + if v.config == nil { + v.config = make(map[string]interface{}) + } cfg := make(map[string]interface{}) if err := v.unmarshalReader(in, cfg); err != nil { return err } - return v.MergeConfigMap(cfg) -} - -// MergeConfigMap merges the configuration from the map given with an existing config. -// Note that the map given may be modified. -func MergeConfigMap(cfg map[string]interface{}) error { return v.MergeConfigMap(cfg) } -func (v *Viper) MergeConfigMap(cfg map[string]interface{}) error { - if v.config == nil { - v.config = make(map[string]interface{}) - } - insensitiviseMap(cfg) mergeMaps(cfg, v.config, nil) return nil } -// WriteConfig writes the current configuration to a file. -func WriteConfig() error { return v.WriteConfig() } -func (v *Viper) WriteConfig() error { - filename, err := v.getConfigFile() - if err != nil { - return err - } - return v.writeConfig(filename, true) -} - -// SafeWriteConfig writes current configuration to file only if the file does not exist. -func SafeWriteConfig() error { return v.SafeWriteConfig() } -func (v *Viper) SafeWriteConfig() error { - filename, err := v.getConfigFile() - if err != nil { - return err - } - return v.writeConfig(filename, false) -} - -// WriteConfigAs writes current configuration to a given filename. -func WriteConfigAs(filename string) error { return v.WriteConfigAs(filename) } -func (v *Viper) WriteConfigAs(filename string) error { - return v.writeConfig(filename, true) -} - -// SafeWriteConfigAs writes current configuration to a given filename if it does not exist. -func SafeWriteConfigAs(filename string) error { return v.SafeWriteConfigAs(filename) } -func (v *Viper) SafeWriteConfigAs(filename string) error { - return v.writeConfig(filename, false) -} - -func writeConfig(filename string, force bool) error { return v.writeConfig(filename, force) } -func (v *Viper) writeConfig(filename string, force bool) error { - jww.INFO.Println("Attempting to write configuration to file.") - ext := filepath.Ext(filename) - if len(ext) <= 1 { - return fmt.Errorf("Filename: %s requires valid extension.", filename) - } - configType := ext[1:] - if !stringInSlice(configType, SupportedExts) { - return UnsupportedConfigError(configType) - } - if v.config == nil { - v.config = make(map[string]interface{}) - } - var flags int - if force == true { - flags = os.O_CREATE | os.O_TRUNC | os.O_WRONLY - } else { - if _, err := os.Stat(filename); os.IsNotExist(err) { - flags = os.O_WRONLY - } else { - return fmt.Errorf("File: %s exists. Use WriteConfig to overwrite.", filename) - } - } - f, err := v.fs.OpenFile(filename, flags, v.configPermissions) - if err != nil { - return err - } - return v.marshalWriter(f, configType) -} - -// Unmarshal a Reader into a map. -// Should probably be an unexported function. -func unmarshalReader(in io.Reader, c map[string]interface{}) error { - return v.unmarshalReader(in, c) -} -func (v *Viper) unmarshalReader(in io.Reader, c map[string]interface{}) error { - buf := new(bytes.Buffer) - buf.ReadFrom(in) - - switch strings.ToLower(v.getConfigType()) { - case "yaml", "yml": - if err := yaml.Unmarshal(buf.Bytes(), &c); err != nil { - return ConfigParseError{err} - } - - case "json": - if err := json.Unmarshal(buf.Bytes(), &c); err != nil { - return ConfigParseError{err} - } - - case "hcl": - obj, err := hcl.Parse(string(buf.Bytes())) - if err != nil { - return ConfigParseError{err} - } - if err = hcl.DecodeObject(&c, obj); err != nil { - return ConfigParseError{err} - } - - case "toml": - tree, err := toml.LoadReader(buf) - if err != nil { - return ConfigParseError{err} - } - tmap := tree.ToMap() - for k, v := range tmap { - c[k] = v - } - - case "properties", "props", "prop": - v.properties = properties.NewProperties() - var err error - if v.properties, err = properties.Load(buf.Bytes(), properties.UTF8); err != nil { - return ConfigParseError{err} - } - for _, key := range v.properties.Keys() { - value, _ := v.properties.Get(key) - // recursively build nested maps - path := strings.Split(key, ".") - lastKey := strings.ToLower(path[len(path)-1]) - deepestMap := deepSearch(c, path[0:len(path)-1]) - // set innermost value - deepestMap[lastKey] = value - } - } - - insensitiviseMap(c) - return nil -} - -// Marshal a map into Writer. -func marshalWriter(f afero.File, configType string) error { - return v.marshalWriter(f, configType) -} -func (v *Viper) marshalWriter(f afero.File, configType string) error { - c := v.AllSettings() - switch configType { - case "json": - b, err := json.MarshalIndent(c, "", " ") - if err != nil { - return ConfigMarshalError{err} - } - _, err = f.WriteString(string(b)) - if err != nil { - return ConfigMarshalError{err} - } - - case "hcl": - b, err := json.Marshal(c) - ast, err := hcl.Parse(string(b)) - if err != nil { - return ConfigMarshalError{err} - } - err = printer.Fprint(f, ast.Node) - if err != nil { - return ConfigMarshalError{err} - } - - case "prop", "props", "properties": - if v.properties == nil { - v.properties = properties.NewProperties() - } - p := v.properties - for _, key := range v.AllKeys() { - _, _, err := p.Set(key, v.GetString(key)) - if err != nil { - return ConfigMarshalError{err} - } - } - _, err := p.WriteComment(f, "#", properties.UTF8) - if err != nil { - return ConfigMarshalError{err} - } - - case "toml": - t, err := toml.TreeFromMap(c) - if err != nil { - return ConfigMarshalError{err} - } - s := t.String() - if _, err := f.WriteString(s); err != nil { - return ConfigMarshalError{err} - } - - case "yaml", "yml": - b, err := yaml.Marshal(c) - if err != nil { - return ConfigMarshalError{err} - } - if _, err = f.WriteString(string(b)); err != nil { - return ConfigMarshalError{err} - } - } - return nil -} - func keyExists(k string, m map[string]interface{}) string { lk := strings.ToLower(k) for mk := range m { @@ -1595,8 +1251,21 @@ func (v *Viper) WatchRemoteConfig() error { return v.watchKeyValueConfig() } -func (v *Viper) WatchRemoteConfigOnChannel() error { - return v.watchKeyValueConfigOnChannel() +// Unmarshall a Reader into a map. +// Should probably be an unexported function. +func unmarshalReader(in io.Reader, c map[string]interface{}) error { + return v.unmarshalReader(in, c) +} + +func (v *Viper) unmarshalReader(in io.Reader, c map[string]interface{}) error { + return unmarshallConfigReader(in, c, v.getConfigType()) +} + +func (v *Viper) insensitiviseMaps() { + insensitiviseMap(v.config) + insensitiviseMap(v.defaults) + insensitiviseMap(v.override) + insensitiviseMap(v.kvstore) } // Retrieve the first found remote configuration. @@ -1625,23 +1294,6 @@ func (v *Viper) getRemoteConfig(provider RemoteProvider) (map[string]interface{} return v.kvstore, err } -// Retrieve the first found remote configuration. -func (v *Viper) watchKeyValueConfigOnChannel() error { - for _, rp := range v.remoteProviders { - respc, _ := RemoteConfig.WatchChannel(rp) - //Todo: Add quit channel - go func(rc <-chan *RemoteResponse) { - for { - b := <-rc - reader := bytes.NewReader(b.Value) - v.unmarshalReader(reader, v.kvstore) - } - }(respc) - return nil - } - return RemoteConfigError("No Files Found") -} - // Retrieve the first found remote configuration. func (v *Viper) watchKeyValueConfig() error { for _, rp := range v.remoteProviders { @@ -1791,12 +1443,6 @@ func (v *Viper) SetConfigType(in string) { } } -// SetConfigPermissions sets the permissions for the config file. -func SetConfigPermissions(perm os.FileMode) { v.SetConfigPermissions(perm) } -func (v *Viper) SetConfigPermissions(perm os.FileMode) { - v.configPermissions = perm.Perm() -} - func (v *Viper) getConfigType() string { if v.configType != "" { return v.configType @@ -1817,21 +1463,25 @@ func (v *Viper) getConfigType() string { } func (v *Viper) getConfigFile() (string, error) { - if v.configFile == "" { - cf, err := v.findConfigFile() - if err != nil { - return "", err - } - v.configFile = cf + // if explicitly set, then use it + if v.configFile != "" { + return v.configFile, nil } - return v.configFile, nil + + cf, err := v.findConfigFile() + if err != nil { + return "", err + } + + v.configFile = cf + return v.getConfigFile() } func (v *Viper) searchInPath(in string) (filename string) { jww.DEBUG.Println("Searching for config in ", in) for _, ext := range SupportedExts { jww.DEBUG.Println("Checking for", filepath.Join(in, v.configName+"."+ext)) - if b, _ := exists(v.fs, filepath.Join(in, v.configName+"."+ext)); b { + if b, _ := exists(filepath.Join(in, v.configName+"."+ext)); b { jww.DEBUG.Println("Found: ", filepath.Join(in, v.configName+"."+ext)) return filepath.Join(in, v.configName+"."+ext) } @@ -1843,6 +1493,7 @@ func (v *Viper) searchInPath(in string) (filename string) { // Search all configPaths for any config file. // Returns the first path that exists (and is a config file). func (v *Viper) findConfigFile() (string, error) { + jww.INFO.Println("Searching for config in ", v.configPaths) for _, cp := range v.configPaths { diff --git a/viper_test.go b/viper_test.go index f43a38b..cd7b65c 100644 --- a/viper_test.go +++ b/viper_test.go @@ -7,29 +7,21 @@ package viper import ( "bytes" - "encoding/json" "fmt" "io" "io/ioutil" "os" - "os/exec" "path" "reflect" - "runtime" "sort" "strings" - "sync" "testing" "time" - "github.com/fsnotify/fsnotify" - "github.com/mitchellh/mapstructure" - "github.com/spf13/afero" "github.com/spf13/cast" "github.com/spf13/pflag" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) var yamlExample = []byte(`Hacker: true @@ -270,7 +262,7 @@ func TestDefault(t *testing.T) { assert.Equal(t, "leather", Get("clothing.jacket")) } -func TestUnmarshaling(t *testing.T) { +func TestUnmarshalling(t *testing.T) { SetConfigType("yaml") r := bytes.NewReader(yamlExample) @@ -388,36 +380,6 @@ func TestEnv(t *testing.T) { } -func TestEmptyEnv(t *testing.T) { - initJSON() - - BindEnv("type") // Empty environment variable - BindEnv("name") // Bound, but not set environment variable - - os.Clearenv() - - os.Setenv("TYPE", "") - - assert.Equal(t, "donut", Get("type")) - assert.Equal(t, "Cake", Get("name")) -} - -func TestEmptyEnv_Allowed(t *testing.T) { - initJSON() - - AllowEmptyEnv(true) - - BindEnv("type") // Empty environment variable - BindEnv("name") // Bound, but not set environment variable - - os.Clearenv() - - os.Setenv("TYPE", "") - - assert.Equal(t, "", Get("type")) - assert.Equal(t, "Cake", Get("name")) -} - func TestEnvPrefix(t *testing.T) { initJSON() @@ -455,7 +417,7 @@ func TestAutoEnvWithPrefix(t *testing.T) { assert.Equal(t, "13", Get("bar")) } -func TestSetEnvKeyReplacer(t *testing.T) { +func TestSetEnvReplacer(t *testing.T) { Reset() AutomaticEnv() @@ -540,42 +502,6 @@ func TestUnmarshal(t *testing.T) { assert.Equal(t, &config{Name: "Steve", Port: 1234, Duration: time.Second + time.Millisecond}, &C) } -func TestUnmarshalWithDecoderOptions(t *testing.T) { - Set("credentials", "{\"foo\":\"bar\"}") - - opt := DecodeHook(mapstructure.ComposeDecodeHookFunc( - mapstructure.StringToTimeDurationHookFunc(), - mapstructure.StringToSliceHookFunc(","), - // Custom Decode Hook Function - func(rf reflect.Kind, rt reflect.Kind, data interface{}) (interface{}, error) { - if rf != reflect.String || rt != reflect.Map { - return data, nil - } - m := map[string]string{} - raw := data.(string) - if raw == "" { - return m, nil - } - return m, json.Unmarshal([]byte(raw), &m) - }, - )) - - type config struct { - Credentials map[string]string - } - - var C config - - err := Unmarshal(&C, opt) - if err != nil { - t.Fatalf("unable to decode into struct, %v", err) - } - - assert.Equal(t, &config{ - Credentials: map[string]string{"foo": "bar"}, - }, &C) -} - func TestBindPFlags(t *testing.T) { v := New() // create independent Viper object flagSet := pflag.NewFlagSet("test", pflag.ContinueOnError) @@ -612,52 +538,6 @@ func TestBindPFlags(t *testing.T) { } -func TestBindPFlagsStringSlice(t *testing.T) { - tests := []struct { - Expected []string - Value string - }{ - {nil, ""}, - {[]string{"jeden"}, "jeden"}, - {[]string{"dwa", "trzy"}, "dwa,trzy"}, - {[]string{"cztery", "piec , szesc"}, "cztery,\"piec , szesc\""}, - } - - v := New() // create independent Viper object - defaultVal := []string{"default"} - v.SetDefault("stringslice", defaultVal) - - for _, testValue := range tests { - flagSet := pflag.NewFlagSet("test", pflag.ContinueOnError) - flagSet.StringSlice("stringslice", testValue.Expected, "test") - - for _, changed := range []bool{true, false} { - flagSet.VisitAll(func(f *pflag.Flag) { - f.Value.Set(testValue.Value) - f.Changed = changed - }) - - err := v.BindPFlags(flagSet) - if err != nil { - t.Fatalf("error binding flag set, %v", err) - } - - type TestStr struct { - StringSlice []string - } - val := &TestStr{} - if err := v.Unmarshal(val); err != nil { - t.Fatalf("%+#v cannot unmarshal: %s", testValue.Value, err) - } - if changed { - assert.Equal(t, testValue.Expected, val.StringSlice) - } else { - assert.Equal(t, defaultVal, val.StringSlice) - } - } - } -} - func TestBindPFlag(t *testing.T) { var testString = "testing" var testValue = newStringValue(testString, &testString) @@ -929,195 +809,10 @@ func TestSub(t *testing.T) { assert.Equal(t, (*Viper)(nil), subv) } -var hclWriteExpected = []byte(`"foos" = { - "foo" = { - "key" = 1 - } - - "foo" = { - "key" = 2 - } - - "foo" = { - "key" = 3 - } - - "foo" = { - "key" = 4 - } -} - -"id" = "0001" - -"name" = "Cake" - -"ppu" = 0.55 - -"type" = "donut"`) - -func TestWriteConfigHCL(t *testing.T) { - v := New() - fs := afero.NewMemMapFs() - v.SetFs(fs) - v.SetConfigName("c") - v.SetConfigType("hcl") - err := v.ReadConfig(bytes.NewBuffer(hclExample)) - if err != nil { - t.Fatal(err) - } - if err := v.WriteConfigAs("c.hcl"); err != nil { - t.Fatal(err) - } - read, err := afero.ReadFile(fs, "c.hcl") - if err != nil { - t.Fatal(err) - } - assert.Equal(t, hclWriteExpected, read) -} - -var jsonWriteExpected = []byte(`{ - "batters": { - "batter": [ - { - "type": "Regular" - }, - { - "type": "Chocolate" - }, - { - "type": "Blueberry" - }, - { - "type": "Devil's Food" - } - ] - }, - "id": "0001", - "name": "Cake", - "ppu": 0.55, - "type": "donut" -}`) - -func TestWriteConfigJson(t *testing.T) { - v := New() - fs := afero.NewMemMapFs() - v.SetFs(fs) - v.SetConfigName("c") - v.SetConfigType("json") - err := v.ReadConfig(bytes.NewBuffer(jsonExample)) - if err != nil { - t.Fatal(err) - } - if err := v.WriteConfigAs("c.json"); err != nil { - t.Fatal(err) - } - read, err := afero.ReadFile(fs, "c.json") - if err != nil { - t.Fatal(err) - } - assert.Equal(t, jsonWriteExpected, read) -} - -var propertiesWriteExpected = []byte(`p_id = 0001 -p_type = donut -p_name = Cake -p_ppu = 0.55 -p_batters.batter.type = Regular -`) - -func TestWriteConfigProperties(t *testing.T) { - v := New() - fs := afero.NewMemMapFs() - v.SetFs(fs) - v.SetConfigName("c") - v.SetConfigType("properties") - err := v.ReadConfig(bytes.NewBuffer(propertiesExample)) - if err != nil { - t.Fatal(err) - } - if err := v.WriteConfigAs("c.properties"); err != nil { - t.Fatal(err) - } - read, err := afero.ReadFile(fs, "c.properties") - if err != nil { - t.Fatal(err) - } - assert.Equal(t, propertiesWriteExpected, read) -} - -func TestWriteConfigTOML(t *testing.T) { - fs := afero.NewMemMapFs() - v := New() - v.SetFs(fs) - v.SetConfigName("c") - v.SetConfigType("toml") - err := v.ReadConfig(bytes.NewBuffer(tomlExample)) - if err != nil { - t.Fatal(err) - } - if err := v.WriteConfigAs("c.toml"); err != nil { - t.Fatal(err) - } - - // The TOML String method does not order the contents. - // Therefore, we must read the generated file and compare the data. - v2 := New() - v2.SetFs(fs) - v2.SetConfigName("c") - v2.SetConfigType("toml") - v2.SetConfigFile("c.toml") - err = v2.ReadInConfig() - if err != nil { - t.Fatal(err) - } - - assert.Equal(t, v.GetString("title"), v2.GetString("title")) - assert.Equal(t, v.GetString("owner.bio"), v2.GetString("owner.bio")) - assert.Equal(t, v.GetString("owner.dob"), v2.GetString("owner.dob")) - assert.Equal(t, v.GetString("owner.organization"), v2.GetString("owner.organization")) -} - -var yamlWriteExpected = []byte(`age: 35 -beard: true -clothing: - jacket: leather - pants: - size: large - trousers: denim -eyes: brown -hacker: true -hobbies: -- skateboarding -- snowboarding -- go -name: steve -`) - -func TestWriteConfigYAML(t *testing.T) { - v := New() - fs := afero.NewMemMapFs() - v.SetFs(fs) - v.SetConfigName("c") - v.SetConfigType("yaml") - err := v.ReadConfig(bytes.NewBuffer(yamlExample)) - if err != nil { - t.Fatal(err) - } - if err := v.WriteConfigAs("c.yaml"); err != nil { - t.Fatal(err) - } - read, err := afero.ReadFile(fs, "c.yaml") - if err != nil { - t.Fatal(err) - } - assert.Equal(t, yamlWriteExpected, read) -} - var yamlMergeExampleTgt = []byte(` hello: pop: 37890 lagrenum: 765432101234567 - num2pow63: 9223372036854775808 world: - us - uk @@ -1146,26 +841,14 @@ func TestMergeConfig(t *testing.T) { t.Fatalf("pop != 37890, = %d", pop) } - if pop := v.GetInt32("hello.pop"); pop != int32(37890) { - t.Fatalf("pop != 37890, = %d", pop) + if pop := v.GetInt("hello.lagrenum"); pop != 765432101234567 { + t.Fatalf("lagrenum != 765432101234567, = %d", pop) } if pop := v.GetInt64("hello.lagrenum"); pop != int64(765432101234567) { t.Fatalf("int64 lagrenum != 765432101234567, = %d", pop) } - if pop := v.GetUint("hello.pop"); pop != 37890 { - t.Fatalf("uint pop != 37890, = %d", pop) - } - - if pop := v.GetUint32("hello.pop"); pop != 37890 { - t.Fatalf("uint32 pop != 37890, = %d", pop) - } - - if pop := v.GetUint64("hello.num2pow63"); pop != 9223372036854775808 { - t.Fatalf("uint64 num2pow63 != 9223372036854775808, = %d", pop) - } - if world := v.GetStringSlice("hello.world"); len(world) != 4 { t.Fatalf("len(world) != 4, = %d", len(world)) } @@ -1182,8 +865,8 @@ func TestMergeConfig(t *testing.T) { t.Fatalf("pop != 45000, = %d", pop) } - if pop := v.GetInt32("hello.pop"); pop != int32(45000) { - t.Fatalf("pop != 45000, = %d", pop) + if pop := v.GetInt("hello.lagrenum"); pop != 7654321001234567 { + t.Fatalf("lagrenum != 7654321001234567, = %d", pop) } if pop := v.GetInt64("hello.lagrenum"); pop != int64(7654321001234567) { @@ -1243,48 +926,6 @@ func TestMergeConfigNoMerge(t *testing.T) { } } -func TestMergeConfigMap(t *testing.T) { - v := New() - v.SetConfigType("yml") - if err := v.ReadConfig(bytes.NewBuffer(yamlMergeExampleTgt)); err != nil { - t.Fatal(err) - } - - assert := func(i int) { - large := v.GetInt("hello.lagrenum") - pop := v.GetInt("hello.pop") - if large != 765432101234567 { - t.Fatal("Got large num:", large) - } - - if pop != i { - t.Fatal("Got pop:", pop) - } - } - - assert(37890) - - update := map[string]interface{}{ - "Hello": map[string]interface{}{ - "Pop": 1234, - }, - "World": map[interface{}]interface{}{ - "Rock": 345, - }, - } - - if err := v.MergeConfigMap(update); err != nil { - t.Fatal(err) - } - - if rock := v.GetInt("world.rock"); rock != 345 { - t.Fatal("Got rock:", rock) - } - - assert(1234) - -} - func TestUnmarshalingWithAliases(t *testing.T) { v := New() v.SetDefault("ID", 1) @@ -1461,35 +1102,6 @@ func TestCaseInsensitiveSet(t *testing.T) { } } -func TestParseNested(t *testing.T) { - type duration struct { - Delay time.Duration - } - - type item struct { - Name string - Delay time.Duration - Nested duration - } - - config := `[[parent]] - delay="100ms" - [parent.nested] - delay="200ms" -` - initConfig("toml", config) - - var items []item - err := v.UnmarshalKey("parent", &items) - if err != nil { - t.Fatalf("unable to decode into struct, %v", err) - } - - assert.Equal(t, 1, len(items)) - assert.Equal(t, 100*time.Millisecond, items[0].Delay) - assert.Equal(t, 200*time.Millisecond, items[0].Nested.Delay) -} - func doTestCaseInsensitive(t *testing.T, typ, config string) { initConfig(typ, config) Set("RfD", true) @@ -1504,111 +1116,6 @@ func doTestCaseInsensitive(t *testing.T, typ, config string) { } -func newViperWithConfigFile(t *testing.T) (*Viper, string, func()) { - watchDir, err := ioutil.TempDir("", "") - require.Nil(t, err) - configFile := path.Join(watchDir, "config.yaml") - err = ioutil.WriteFile(configFile, []byte("foo: bar\n"), 0640) - require.Nil(t, err) - cleanup := func() { - os.RemoveAll(watchDir) - } - v := New() - v.SetConfigFile(configFile) - err = v.ReadInConfig() - require.Nil(t, err) - require.Equal(t, "bar", v.Get("foo")) - return v, configFile, cleanup -} - -func newViperWithSymlinkedConfigFile(t *testing.T) (*Viper, string, string, func()) { - watchDir, err := ioutil.TempDir("", "") - require.Nil(t, err) - dataDir1 := path.Join(watchDir, "data1") - err = os.Mkdir(dataDir1, 0777) - require.Nil(t, err) - realConfigFile := path.Join(dataDir1, "config.yaml") - t.Logf("Real config file location: %s\n", realConfigFile) - err = ioutil.WriteFile(realConfigFile, []byte("foo: bar\n"), 0640) - require.Nil(t, err) - cleanup := func() { - os.RemoveAll(watchDir) - } - // now, symlink the tm `data1` dir to `data` in the baseDir - os.Symlink(dataDir1, path.Join(watchDir, "data")) - // and link the `/datadir1/config.yaml` to `/config.yaml` - configFile := path.Join(watchDir, "config.yaml") - os.Symlink(path.Join(watchDir, "data", "config.yaml"), configFile) - t.Logf("Config file location: %s\n", path.Join(watchDir, "config.yaml")) - // init Viper - v := New() - v.SetConfigFile(configFile) - err = v.ReadInConfig() - require.Nil(t, err) - require.Equal(t, "bar", v.Get("foo")) - return v, watchDir, configFile, cleanup -} - -func TestWatchFile(t *testing.T) { - if runtime.GOOS == "linux" { - // TODO(bep) FIX ME - t.Skip("Skip test on Linux ...") - } - - t.Run("file content changed", func(t *testing.T) { - // given a `config.yaml` file being watched - v, configFile, cleanup := newViperWithConfigFile(t) - defer cleanup() - _, err := os.Stat(configFile) - require.NoError(t, err) - t.Logf("test config file: %s\n", configFile) - wg := sync.WaitGroup{} - wg.Add(1) - v.OnConfigChange(func(in fsnotify.Event) { - t.Logf("config file changed") - wg.Done() - }) - v.WatchConfig() - // when overwriting the file and waiting for the custom change notification handler to be triggered - err = ioutil.WriteFile(configFile, []byte("foo: baz\n"), 0640) - wg.Wait() - // then the config value should have changed - require.Nil(t, err) - assert.Equal(t, "baz", v.Get("foo")) - }) - - t.Run("link to real file changed (à la Kubernetes)", func(t *testing.T) { - // skip if not executed on Linux - if runtime.GOOS != "linux" { - t.Skipf("Skipping test as symlink replacements don't work on non-linux environment...") - } - v, watchDir, _, _ := newViperWithSymlinkedConfigFile(t) - // defer cleanup() - wg := sync.WaitGroup{} - v.WatchConfig() - v.OnConfigChange(func(in fsnotify.Event) { - t.Logf("config file changed") - wg.Done() - }) - wg.Add(1) - // when link to another `config.yaml` file - dataDir2 := path.Join(watchDir, "data2") - err := os.Mkdir(dataDir2, 0777) - require.Nil(t, err) - configFile2 := path.Join(dataDir2, "config.yaml") - err = ioutil.WriteFile(configFile2, []byte("foo: baz\n"), 0640) - require.Nil(t, err) - // change the symlink using the `ln -sfn` command - err = exec.Command("ln", "-sfn", dataDir2, path.Join(watchDir, "data")).Run() - require.Nil(t, err) - wg.Wait() - // then - require.Nil(t, err) - assert.Equal(t, "baz", v.Get("foo")) - }) - -} - func BenchmarkGetBool(b *testing.B) { key := "BenchmarkGetBool" v = New()