diff --git a/.travis.yml b/.travis.yml index a578cbf..ae1f68e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,5 @@ language: go go: - - 1.2 - 1.3 - release - tip diff --git a/README.md b/README.md index a06bedb..47b1798 100644 --- a/README.md +++ b/README.md @@ -1,44 +1,47 @@ viper [![Build Status](https://travis-ci.org/spf13/viper.svg)](https://travis-ci.org/spf13/viper) ===== +[![Join the chat at https://gitter.im/spf13/viper](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/spf13/viper?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) + Go configuration with fangs ## What is Viper? -Viper is a complete configuration solution for go applications. It has -been designed to work within an application to handle all types of -configuration. It supports +Viper is a complete configuration solution for go applications. It is designed +to work within an application, and can handle all types of configuration needs +and formats. It supports: * setting defaults -* reading from yaml, toml and json config files +* reading from JSON, TOML, and YAML config files * reading from environment variables -* reading from remote config systems (Etcd or Consul) +* reading from remote config systems (Etcd or Consul), and watching changes * reading from command line flags +* reading from buffer * setting explicit values -It can be thought of as a registry for all of your applications +Viper can be thought of as a registry for all of your applications configuration needs. ## Why Viper? -When building a modern application you don’t want to have to worry about -configuration file formats, you want to focus on building awesome software. +When building a modern application, you don’t want to worry about +configuration file formats; you want to focus on building awesome software. Viper is here to help with that. Viper does the following for you: -1. Find, load and marshal a configuration file in YAML, TOML or JSON. +1. Find, load, and marshal a configuration file in JSON, TOML, or YAML. 2. Provide a mechanism to set default values for your different - configuration options -3. Provide a mechanism to set override values for options specified - through command line flags. -4. Provide an alias system to easily rename parameters without breaking - existing code. -5. Make it easy to tell the difference between when a user has provided - a command line or config file which is the same as the default. + configuration options. +3. Provide a mechanism to set override values for options specified through + command line flags. +4. Provide an alias system to easily rename parameters without breaking existing + code. +5. Make it easy to tell the difference between when a user has provided a + command line or config file which is the same as the default. -Viper uses the following precedence order. Each item takes precedence -over the item below it: +Viper uses the following precedence order. Each item takes precedence over the +item below it: * explicit call to Set * flag @@ -53,248 +56,413 @@ Viper configuration keys are case insensitive. ### Establishing Defaults -A good configuration system will support default values. A default value -is not required for a key, but can establish a default to be used in the -event that the key hasn’t be set via config file, environment variable, -remote configuration or flag. +A good configuration system will support default values. A default value is not +required for a key, but it's useful in the event that a key hasn’t be set via +config file, environment variable, remote configuration or flag. Examples: - viper.SetDefault("ContentDir", "content") - viper.SetDefault("LayoutDir", "layouts") - viper.SetDefault("Indexes", map[string]string{"tag": "tags", "category": "categories"}) +```go +viper.SetDefault("ContentDir", "content") +viper.SetDefault("LayoutDir", "layouts") +viper.SetDefault("Taxonomies", map[string]string{"tag": "tags", "category": "categories"}) +``` ### Reading Config Files -If you want to support a config file, Viper requires a minimal -configuration so it knows where to look for the config file. Viper -supports yaml, toml and json files. Viper can search multiple paths, but -currently a single viper only supports a single config file. +Viper requires minimal configuration so it knows where to look for config files. +Viper supports JSON, TOML and YAML files. Viper can search multiple paths, but +currently a single Viper instance only supports a single configuration file. - viper.SetConfigName("config") // name of config file (without extension) - viper.AddConfigPath("/etc/appname/") // path to look for the config file in - viper.AddConfigPath("$HOME/.appname") // call multiple times to add many search paths - viper.ReadInConfig() // Find and read the config file +```go +viper.SetConfigName("config") // name of config file (without extension) +viper.AddConfigPath("/etc/appname/") // path to look for the config file in +viper.AddConfigPath("$HOME/.appname") // call multiple times to add many search paths +err := viper.ReadInConfig() // Find and read the config file +if err != nil { // Handle errors reading the config file + panic(fmt.Errorf("Fatal error config file: %s \n", err)) +} +``` + +### Reading Config from io.Reader + +Viper predefines many configuration sources such as files, environment +variables, flags, and remote K/V store, but you are not bound to them. You can +also implement your own required configuration source and feed it to viper. + +```go +viper.SetConfigType("yaml") // or viper.SetConfigType("YAML") + +// any approach to require this configuration into your program. +var yamlExample = []byte(` +Hacker: true +name: steve +hobbies: +- skateboarding +- snowboarding +- go +clothing: + jacket: leather + trousers: denim +age: 35 +eyes : brown +beard: true +`) + +viper.ReadConfig(bytes.NewBuffer(yamlExample)) + +viper.Get("name") // this would be "steve" +``` ### Setting Overrides These could be from a command line flag, or from your own application logic. - viper.Set("Verbose", true) - viper.Set("LogFile", LogFile) +```go +viper.Set("Verbose", true) +viper.Set("LogFile", LogFile) +``` ### Registering and Using Aliases Aliases permit a single value to be referenced by multiple keys - viper.RegisterAlias("loud", "Verbose") +```go +viper.RegisterAlias("loud", "Verbose") - viper.Set("verbose", true) // same result as next line - viper.Set("loud", true) // same result as prior line +viper.Set("verbose", true) // same result as next line +viper.Set("loud", true) // same result as prior line - viper.GetBool("loud") // true - viper.GetBool("verbose") // true +viper.GetBool("loud") // true +viper.GetBool("verbose") // true +``` ### Working with Environment Variables Viper has full support for environment variables. This enables 12 factor -applications out of the box. There are three methods that exist to aid -with working with ENV: +applications out of the box. There are four methods that exist to aid working +with ENV: - * AutomaticEnv() - * BindEnv(string...) : error - * SetEnvPrefix(string) + * `AutomaticEnv()` + * `BindEnv(string...) : error` + * `SetEnvPrefix(string)` + * `SetEnvReplacer(string...) *strings.Replacer` -_When working with ENV variables it’s important to recognize that Viper +_When working with ENV variables, it’s important to recognize that Viper treats ENV variables as case sensitive._ -Viper provides a mechanism to try to ensure that ENV variables are -unique. By using SetEnvPrefix you can tell Viper to use add a prefix -while reading from the environment variables. Both BindEnv and -AutomaticEnv will use this prefix. +Viper provides a mechanism to try to ensure that ENV variables are unique. By +using `SetEnvPrefix`, you can tell Viper to use add a prefix while reading from +the environment variables. Both `BindEnv` and `AutomaticEnv` will use this +prefix. -BindEnv takes one or two parameters. The first parameter is the key -name, the second is the name of the environment variable. The name of -the environment variable is case sensitive. If the ENV variable name is -not provided then Viper will automatically assume that the key name -matches the ENV variable name but the ENV variable is IN ALL CAPS. When -you explicitly provide the env variable name it **Does not** -automatically add the prefix. +`BindEnv` takes one or two parameters. The first parameter is the key name, the +second is the name of the environment variable. The name of the environment +variable is case sensitive. If the ENV variable name is not provided, then +Viper will automatically assume that the key name matches the ENV variable name, +but the ENV variable is IN ALL CAPS. When you explicitly provide the ENV +variable name, it **does not** automatically add the prefix. -One important thing to recognize when working with ENV variables is that -the value will be read each time it is accessed. It does not fix the -value when the BindEnv is called. +One important thing to recognize when working with ENV variables is that the +value will be read each time it is accessed. Viper does not fix the value when +the `BindEnv` is called. -AutomaticEnv is a powerful helper especially when combined with -SetEnvPrefix. When called, Viper will check for an environment variable -any time a viper.Get request is made. It will apply the following rules. -It will check for a environment variable with a name matching the key -uppercased and prefixed with the EnvPrefix if set. +`AutomaticEnv` is a powerful helper especially when combined with +`SetEnvPrefix`. When called, Viper will check for an environment variable any +time a `viper.Get` request is made. It will apply the following rules. It will +check for a environment variable with a name matching the key uppercased and +prefixed with the `EnvPrefix` if set. + +`SetEnvReplacer` allows you to use a `strings.Replacer` object to rewrite Env +keys to an extent. This is useful if you want to use `-` or something in your +`Get()` calls, but want your environmental variables to use `_` delimiters. An +example of using it can be found in `viper_test.go`. #### Env example - SetEnvPrefix("spf") // will be uppercased automatically - BindEnv("id") +```go +SetEnvPrefix("spf") // will be uppercased automatically +BindEnv("id") - os.Setenv("SPF_ID", "13") // typically done outside of the app - - id := Get("id")) // 13 +os.Setenv("SPF_ID", "13") // typically done outside of the app +id := Get("id") // 13 +``` ### Working with Flags -Viper has the ability to bind to flags. Specifically Viper supports -Pflags as used in the [Cobra](http://github.com/spf13/cobra) library. +Viper has the ability to bind to flags. Specifically, Viper supports `Pflags` +as used in the [Cobra](https://github.com/spf13/cobra) library. -Like BindEnv the value is not set when the binding method is called, but -when it is accessed. This means you can bind as early as you want, even -in an init() function. +Like `BindEnv`, the value is not set when the binding method is called, but when +it is accessed. This means you can bind as early as you want, even in an +`init()` function. -The BindPFlag() method provides this functionality. +The `BindPFlag()` method provides this functionality. Example: - serverCmd.Flags().Int("port", 1138, "Port to run Application server on") - viper.BindPFlag("port", serverCmd.Flags().Lookup("port")) - +```go +serverCmd.Flags().Int("port", 1138, "Port to run Application server on") +viper.BindPFlag("port", serverCmd.Flags().Lookup("port")) +``` ### Remote Key/Value Store Support -Viper will read a config string (as JSON, TOML, or YAML) retrieved from a -path in a Key/Value store such as Etcd or Consul. These values take precedence -over default values, but are overriden by configuration values retrieved from disk, + +To enable remote support in Viper, do a blank import of the `viper/remote` +package: + +`import _ github.com/spf13/viper/remote` + +Viper will read a config string (as JSON, TOML, or YAML) retrieved from a path +in a Key/Value store such as Etcd or Consul. These values take precedence over +default values, but are overridden by configuration values retrieved from disk, flags, or environment variables. -Viper uses [crypt](https://github.com/xordataexchange/crypt) to retrieve configuration -from the k/v store, which means that you can store your configuration values -encrypted and have them automatically decrypted if you have the correct -gpg keyring. Encryption is optional. +Viper uses [crypt](https://github.com/xordataexchange/crypt) to retrieve +configuration from the K/V store, which means that you can store your +configuration values encrypted and have them automatically decrypted if you have +the correct gpg keyring. Encryption is optional. You can use remote configuration in conjunction with local configuration, or -independently of it. +independently of it. -`crypt` has a command-line helper that you can use to put configurations -in your k/v store. `crypt` defaults to etcd on http://127.0.0.1:4001. +`crypt` has a command-line helper that you can use to put configurations in your +K/V store. `crypt` defaults to etcd on http://127.0.0.1:4001. - go get github.com/xordataexchange/crypt/bin/crypt - crypt set -plaintext /config/hugo.json /Users/hugo/settings/config.json +```bash +$ go get github.com/xordataexchange/crypt/bin/crypt +$ crypt set -plaintext /config/hugo.json /Users/hugo/settings/config.json +``` Confirm that your value was set: - crypt get -plaintext /config/hugo.json +```bash +$ crypt get -plaintext /config/hugo.json +``` -See the `crypt` documentation for examples of how to set encrypted values, or how -to use Consul. +See the `crypt` documentation for examples of how to set encrypted values, or +how to use Consul. ### Remote Key/Value Store Example - Unencrypted - viper.AddRemoteProvider("etcd", "http://127.0.0.1:4001","/config/hugo.json") - viper.SetConfigType("json") // because there is no file extension in a stream of bytes - err := viper.ReadRemoteConfig() +```go +viper.AddRemoteProvider("etcd", "http://127.0.0.1:4001","/config/hugo.json") +viper.SetConfigType("json") // because there is no file extension in a stream of bytes +err := viper.ReadRemoteConfig() +``` ### Remote Key/Value Store Example - Encrypted - viper.AddSecureRemoteProvider("etcd","http://127.0.0.1:4001","/config/hugo.json","/etc/secrets/mykeyring.gpg") - viper.SetConfigType("json") // because there is no file extension in a stream of bytes - err := viper.ReadRemoteConfig() +```go +viper.AddSecureRemoteProvider("etcd","http://127.0.0.1:4001","/config/hugo.json","/etc/secrets/mykeyring.gpg") +viper.SetConfigType("json") // because there is no file extension in a stream of bytes +err := viper.ReadRemoteConfig() +``` +### Watching Changes in Etcd - Unencrypted + +```go +// alternatively, you can create a new viper instance. +var runtime_viper = viper.New() + +runtime_viper.AddRemoteProvider("etcd", "http://127.0.0.1:4001", "/config/hugo.yml") +runtime_viper.SetConfigType("yaml") // because there is no file extension in a stream of bytes + +// read from remote config the first time. +err := runtime_viper.ReadRemoteConfig() + +// marshal config +runtime_viper.Marshal(&runtime_conf) + +// open a goroutine to wath remote changes forever +go func(){ + for { + time.Sleep(time.Second * 5) // delay after each request + + // currenlty, only tested with etcd support + err := runtime_viper.WatchRemoteConfig() + if err != nil { + log.Errorf("unable to read remote config: %v", err) + continue + } + + // marshal new config into our runtime config struct. you can also use channel + // to implement a signal to notify the system of the changes + runtime_viper.Marshal(&runtime_conf) + } +}() +``` ## Getting Values From Viper -In Viper there are a few ways to get a value depending on what type of value you want to retrieved. -The following functions and methods exist: +In Viper, there are a few ways to get a value depending on the value's type. +The following functions and methods exist: - * Get(key string) : interface{} - * GetBool(key string) : bool - * GetFloat64(key string) : float64 - * GetInt(key string) : int - * GetString(key string) : string - * GetStringMap(key string) : map[string]interface{} - * GetStringMapString(key string) : map[string]string - * GetStringSlice(key string) : []string - * GetTime(key string) : time.Time - * IsSet(key string) : bool + * `Get(key string) : interface{}` + * `GetBool(key string) : bool` + * `GetFloat64(key string) : float64` + * `GetInt(key string) : int` + * `GetString(key string) : string` + * `GetStringMap(key string) : map[string]interface{}` + * `GetStringMapString(key string) : map[string]string` + * `GetStringSlice(key string) : []string` + * `GetTime(key string) : time.Time` + * `GetDuration(key string) : time.Duration` + * `IsSet(key string) : bool` -One important thing to recognize is that each Get function will return -it’s zero value if it’s not found. To check if a given key exists, the IsSet() -method has been provided. +One important thing to recognize is that each Get function will return a zero +value if it’s not found. To check if a given key exists, the `IsSet()` method +has been provided. Example: +```go +viper.GetString("logfile") // case-insensitive Setting & Getting +if viper.GetBool("verbose") { + fmt.Println("verbose enabled") +} +``` +### Accessing nested keys - viper.GetString("logfile") // case insensitive Setting & Getting - if viper.GetBool("verbose") { - fmt.Println("verbose enabled") - } +The accessor methods also accept formatted paths to deeply nested keys. For +example, if the following JSON file is loaded: + +```json +{ + "host": { + "address": "localhost", + "port": 5799 + }, + "datastore": { + "metric": { + "host": "127.0.0.1", + "port": 3099 + }, + "warehouse": { + "host": "198.0.0.1", + "port": 2112 + } + } +} + +``` + +Viper can access a nested field by passing a `.` delimited path of keys: + +```go +GetString("datastore.metric.host") // (returns "127.0.0.1") +``` + +This obeys the precedence rules established above; the search for the root key +(in this example, `datastore`) will cascade through the remaining configuration +registries until found. The search for the sub-keys (`metric` and `host`), +however, will not. + +For example, if the `metric` key was not defined in the configuration loaded +from file, but was defined in the defaults, Viper would return the zero value. + +On the other hand, if the primary key was not defined, Viper would go through +the remaining registries looking for it. + +Lastly, if there exists a key that matches the delimited key path, its value +will be returned instead. E.g. + +```json +{ + "datastore.metric.host": "0.0.0.0", + "host": { + "address": "localhost", + "port": 5799 + }, + "datastore": { + "metric": { + "host": "127.0.0.1", + "port": 3099 + }, + "warehouse": { + "host": "198.0.0.1", + "port": 2112 + } + } +} + +GetString("datastore.metric.host") //returns "0.0.0.0" +``` ### Marshaling -You also have the option of Marshaling all or a specific value to a struct, map, etc. +You also have the option of Marshaling all or a specific value to a struct, map, +etc. There are two methods to do this: - * Marshal(rawVal interface{}) : error - * MarshalKey(key string, rawVal interface{}) : error + * `Marshal(rawVal interface{}) : error` + * `MarshalKey(key string, rawVal interface{}) : error` Example: - type config struct { - Port int - Name string - } +```go +type config struct { + Port int + Name string +} - var C config - - err := Marshal(&C) - if err != nil { - t.Fatalf("unable to decode into struct, %v", err) - } +var C config +err := Marshal(&C) +if err != nil { + t.Fatalf("unable to decode into struct, %v", err) +} +``` ## Viper or Vipers? Viper comes ready to use out of the box. There is no configuration or -initialization needed to begin using Viper. Since most applications will -want to use a single central repository for their configuration the -viper package provides this. It is similar to a singleton. +initialization needed to begin using Viper. Since most applications will want +to use a single central repository for their configuration, the viper package +provides this. It is similar to a singleton. -In all of the examples above they demonstrate using viper in it’s -singleton style approach. +In all of the examples above, they demonstrate using viper in it's singleton +style approach. ### Working with multiple vipers -You can also create many different vipers for use in your application. -Each will have it’s own unique set of configurations and values. Each -can read from a different config file, key value store, etc. All of the -functions that viper package supports are mirrored as methods on a viper. +You can also create many different vipers for use in your application. Each will +have it’s own unique set of configurations and values. Each can read from a +different config file, key value store, etc. All of the functions that viper +package supports are mirrored as methods on a viper. Example: - x := viper.New() - y := viper.New() +```go +x := viper.New() +y := viper.New() - x.SetDefault("ContentDir", "content") - y.SetDefault("ContentDir", "foobar") +x.SetDefault("ContentDir", "content") +y.SetDefault("ContentDir", "foobar") - ... +//... +``` -When working with multiple vipers it is up to the user to keep track of -the different vipers. +When working with multiple vipers, it is up to the user to keep track of the +different vipers. ## Q & A Q: Why not INI files? -A: Ini files are pretty awful. There’s no standard format and they are hard to -validate. Viper is designed to work with YAML, TOML or JSON files. If someone -really wants to add this feature, I’d be happy to merge it. It’s easy to -specify which formats your application will permit. +A: Ini files are pretty awful. There’s no standard format, and they are hard to +validate. Viper is designed to work with JSON, TOML or YAML files. If someone +really wants to add this feature, I’d be happy to merge it. It’s easy to specify +which formats your application will permit. -Q: Why is it called "viper"? +Q: Why is it called “Viper”? -A: Viper is designed to be a companion to -[Cobra](http://github.com/spf13/cobra). While both can operate completely +A: Viper is designed to be a [companion](http://en.wikipedia.org/wiki/Viper_(G.I._Joe)) +to [Cobra](https://github.com/spf13/cobra). While both can operate completely independently, together they make a powerful pair to handle much of your application foundation needs. -Q: Why is it called "Cobra"? +Q: Why is it called “Cobra”? -A: Is there a better name for a commander? +A: Is there a better name for a [commander](http://en.wikipedia.org/wiki/Cobra_Commander)? diff --git a/remote/remote.go b/remote/remote.go new file mode 100644 index 0000000..faaf3b3 --- /dev/null +++ b/remote/remote.go @@ -0,0 +1,77 @@ +// Copyright © 2015 Steve Francia . +// +// Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file. + +// Package remote integrates the remote features of Viper. +package remote + +import ( + "bytes" + "github.com/spf13/viper" + crypt "github.com/xordataexchange/crypt/config" + "io" + "os" +) + +type remoteConfigProvider struct{} + +func (rc remoteConfigProvider) Get(rp viper.RemoteProvider) (io.Reader, error) { + cm, err := getConfigManager(rp) + if err != nil { + return nil, err + } + b, err := cm.Get(rp.Path()) + if err != nil { + return nil, err + } + return bytes.NewReader(b), nil +} + +func (rc remoteConfigProvider) Watch(rp viper.RemoteProvider) (io.Reader, error) { + cm, err := getConfigManager(rp) + if err != nil { + return nil, err + } + resp := <-cm.Watch(rp.Path(), nil) + err = resp.Error + if err != nil { + return nil, err + } + + return bytes.NewReader(resp.Value), nil +} + +func getConfigManager(rp viper.RemoteProvider) (crypt.ConfigManager, error) { + + var cm crypt.ConfigManager + var err error + + if rp.SecretKeyring() != "" { + kr, err := os.Open(rp.SecretKeyring()) + defer kr.Close() + if err != nil { + return nil, err + } + if rp.Provider() == "etcd" { + cm, err = crypt.NewEtcdConfigManager([]string{rp.Endpoint()}, kr) + } else { + cm, err = crypt.NewConsulConfigManager([]string{rp.Endpoint()}, kr) + } + } else { + if rp.Provider() == "etcd" { + cm, err = crypt.NewStandardEtcdConfigManager([]string{rp.Endpoint()}) + } else { + cm, err = crypt.NewStandardConsulConfigManager([]string{rp.Endpoint()}) + } + } + if err != nil { + return nil, err + } + return cm, nil + +} + +func init() { + viper.RemoteConfig = &remoteConfigProvider{} +} diff --git a/util.go b/util.go index 1786f7c..a605e62 100644 --- a/util.go +++ b/util.go @@ -19,13 +19,26 @@ import ( "path/filepath" "runtime" "strings" + "unicode" "github.com/BurntSushi/toml" + "github.com/magiconair/properties" + "github.com/spf13/cast" jww "github.com/spf13/jwalterweatherman" - "gopkg.in/yaml.v1" + "gopkg.in/yaml.v2" ) -func insensativiseMap(m map[string]interface{}) { +// Denotes failing to parse configuration file. +type ConfigParseError struct { + err error +} + +// Returns the formatted configuration error. +func (pe ConfigParseError) Error() string { + return fmt.Sprintf("While parsing config: %s", pe.err.Error()) +} + +func insensitiviseMap(m map[string]interface{}) { for key, val := range m { lower := strings.ToLower(key) if key != lower { @@ -116,26 +129,81 @@ func findCWD() (string, error) { return path, nil } -func marshallConfigReader(in io.Reader, c map[string]interface{}, configType string) { +func marshallConfigReader(in io.Reader, c map[string]interface{}, configType string) error { buf := new(bytes.Buffer) buf.ReadFrom(in) - switch configType { + switch strings.ToLower(configType) { case "yaml", "yml": if err := yaml.Unmarshal(buf.Bytes(), &c); err != nil { - jww.ERROR.Fatalf("Error parsing config: %s", err) + return ConfigParseError{err} } case "json": if err := json.Unmarshal(buf.Bytes(), &c); err != nil { - jww.ERROR.Fatalf("Error parsing config: %s", err) + return ConfigParseError{err} } case "toml": if _, err := toml.Decode(buf.String(), &c); err != nil { - jww.ERROR.Fatalf("Error parsing config: %s", err) + return ConfigParseError{err} + } + + case "properties", "props", "prop": + var p *properties.Properties + var err error + if p, err = properties.Load(buf.Bytes(), properties.UTF8); err != nil { + return ConfigParseError{err} + } + for _, key := range p.Keys() { + value, _ := p.Get(key) + c[key] = value } } - insensativiseMap(c) + insensitiviseMap(c) + return nil +} + +func safeMul(a, b uint) uint { + c := a * b + if a > 1 && b > 1 && c/b != a { + return 0 + } + return c +} + +// parseSizeInBytes converts strings like 1GB or 12 mb into an unsigned integer number of bytes +func parseSizeInBytes(sizeStr string) uint { + sizeStr = strings.TrimSpace(sizeStr) + lastChar := len(sizeStr) - 1 + multiplier := uint(1) + + if lastChar > 0 { + if sizeStr[lastChar] == 'b' || sizeStr[lastChar] == 'B' { + if lastChar > 1 { + switch unicode.ToLower(rune(sizeStr[lastChar-1])) { + case 'k': + multiplier = 1 << 10 + sizeStr = strings.TrimSpace(sizeStr[:lastChar-1]) + case 'm': + multiplier = 1 << 20 + sizeStr = strings.TrimSpace(sizeStr[:lastChar-1]) + case 'g': + multiplier = 1 << 30 + sizeStr = strings.TrimSpace(sizeStr[:lastChar-1]) + default: + multiplier = 1 + sizeStr = strings.TrimSpace(sizeStr[:lastChar]) + } + } + } + } + + size := cast.ToInt(sizeStr) + if size < 0 { + size = 0 + } + + return safeMul(uint(size), multiplier) } diff --git a/viper.go b/viper.go index 3071df0..c90e837 100644 --- a/viper.go +++ b/viper.go @@ -10,6 +10,7 @@ // Each item takes precedence over the item below it: +// overrides // flag // env // config @@ -34,7 +35,6 @@ import ( "github.com/spf13/cast" jww "github.com/spf13/jwalterweatherman" "github.com/spf13/pflag" - crypt "github.com/xordataexchange/crypt/config" ) var v *Viper @@ -43,32 +43,96 @@ func init() { v = New() } +type remoteConfigFactory interface { + Get(rp RemoteProvider) (io.Reader, error) + Watch(rp RemoteProvider) (io.Reader, error) +} + +// RemoteConfig is optional, see the remote package +var RemoteConfig remoteConfigFactory + +// Denotes encountering an unsupported +// configuration filetype. type UnsupportedConfigError string +// Returns the formatted configuration error. func (str UnsupportedConfigError) Error() string { return fmt.Sprintf("Unsupported Config Type %q", string(str)) } +// Denotes encountering an unsupported remote +// provider. Currently only Etcd and Consul are +// supported. type UnsupportedRemoteProviderError string +// Returns the formatted remote provider error. func (str UnsupportedRemoteProviderError) Error() string { return fmt.Sprintf("Unsupported Remote Provider Type %q", string(str)) } +// Denotes encountering an error while trying to +// pull the configuration from the remote provider. type RemoteConfigError string +// Returns the formatted remote provider error func (rce RemoteConfigError) Error() string { return fmt.Sprintf("Remote Configurations Error: %s", string(rce)) } -// A viper is a unexported struct. Use New() to create a new instance of viper -// or use the functions for a "global instance" +// Denotes failing to find configuration file. +type ConfigFileNotFoundError struct { + name, locations string +} + +// Returns the formatted configuration error. +func (fnfe ConfigFileNotFoundError) Error() string { + return fmt.Sprintf("Config File %q Not Found in %q", fnfe.name, fnfe.locations) +} + +// Viper is a prioritized configuration registry. It +// maintains a set of configuration sources, fetches +// values to populate those, and provides them according +// to the source's priority. +// The priority of the sources is the following: +// 1. overrides +// 2. flags +// 3. env. variables +// 4. config file +// 5. key/value store +// 6. defaults +// +// For example, if values from the following sources were loaded: +// +// Defaults : { +// "secret": "", +// "user": "default", +// "endpoint": "https://localhost" +// } +// Config : { +// "user": "root" +// "secret": "defaultsecret" +// } +// Env : { +// "secret": "somesecretkey" +// } +// +// The resulting config will have the following values: +// +// { +// "secret": "somesecretkey", +// "user": "root", +// "endpoint": "https://localhost" +// } type Viper struct { + // Delimiter that separates a list of keys + // used to access a nested value in one go + keyDelim string + // A set of paths to look for the config file in configPaths []string // A set of remote providers to search for the configuration - remoteProviders []*remoteProvider + remoteProviders []*defaultRemoteProvider // Name of file to look for inside the path configName string @@ -77,6 +141,7 @@ type Viper struct { envPrefix string automaticEnvApplied bool + envKeyReplacer *strings.Replacer config map[string]interface{} override map[string]interface{} @@ -87,9 +152,10 @@ type Viper struct { aliases map[string]string } -// The prescribed way to create a new Viper +// Returns an initialized Viper instance. func New() *Viper { v := new(Viper) + v.keyDelim = "." v.configName = "config" v.config = make(map[string]interface{}) v.override = make(map[string]interface{}) @@ -102,21 +168,53 @@ func New() *Viper { return v } -// remoteProvider stores the configuration necessary -// to connect to a remote key/value store. -// Optional secretKeyring to unencrypt encrypted values -// can be provided. -type remoteProvider struct { +// Intended for testing, will reset all to default settings. +// In the public interface for the viper package so applications +// can use it in their testing as well. +func Reset() { + v = New() + SupportedExts = []string{"json", "toml", "yaml", "yml"} + SupportedRemoteProviders = []string{"etcd", "consul"} +} + +type defaultRemoteProvider struct { provider string endpoint string path string secretKeyring string } -// universally supported extensions -var SupportedExts []string = []string{"json", "toml", "yaml", "yml"} +func (rp defaultRemoteProvider) Provider() string { + return rp.provider +} -// universally supported remote providers +func (rp defaultRemoteProvider) Endpoint() string { + return rp.endpoint +} + +func (rp defaultRemoteProvider) Path() string { + return rp.path +} + +func (rp defaultRemoteProvider) SecretKeyring() string { + return rp.secretKeyring +} + +// RemoteProvider stores the configuration necessary +// to connect to a remote key/value store. +// Optional secretKeyring to unencrypt encrypted values +// can be provided. +type RemoteProvider interface { + Provider() string + Endpoint() string + Path() string + SecretKeyring() string +} + +// Universally supported extensions. +var SupportedExts []string = []string{"json", "toml", "yaml", "yml", "properties", "props", "prop"} + +// Universally supported remote providers. var SupportedRemoteProviders []string = []string{"etcd", "consul"} // Explicitly define the path, name and extension of the config file @@ -129,6 +227,8 @@ func (v *Viper) SetConfigFile(in string) { } // Define a prefix that ENVIRONMENT variables will use. +// E.g. if your prefix is "spf", the env registry +// will look for env. variables that start with "SPF_" func SetEnvPrefix(in string) { v.SetEnvPrefix(in) } func (v *Viper) SetEnvPrefix(in string) { if in != "" { @@ -144,11 +244,25 @@ func (v *Viper) mergeWithEnvPrefix(in string) string { return strings.ToUpper(in) } -// Return the config file used +// TODO: should getEnv logic be moved into find(). Can generalize the use of +// rewriting keys many things, Ex: Get('someKey') -> some_key +// (cammel case to snake case for JSON keys perhaps) + +// getEnv s a wrapper around os.Getenv which replaces characters in the original +// key. This allows env vars which have different keys then the config object +// keys +func (v *Viper) getEnv(key string) string { + if v.envKeyReplacer != nil { + key = v.envKeyReplacer.Replace(key) + } + return os.Getenv(key) +} + +// Return the file used to populate the config registry func ConfigFileUsed() string { return v.ConfigFileUsed() } func (v *Viper) ConfigFileUsed() string { return v.configFile } -// Add a path for viper to search for the config file in. +// Add a path for Viper to search for the config file in. // Can be called multiple times to define multiple search paths. func AddConfigPath(in string) { v.AddConfigPath(in) } func (v *Viper) AddConfigPath(in string) { @@ -178,7 +292,7 @@ func (v *Viper) AddRemoteProvider(provider, endpoint, path string) error { } if provider != "" && endpoint != "" { jww.INFO.Printf("adding %s:%s to remote provider list", provider, endpoint) - rp := &remoteProvider{ + rp := &defaultRemoteProvider{ endpoint: endpoint, provider: provider, path: path, @@ -210,10 +324,11 @@ func (v *Viper) AddSecureRemoteProvider(provider, endpoint, path, secretkeyring } if provider != "" && endpoint != "" { jww.INFO.Printf("adding %s:%s to remote provider list", provider, endpoint) - rp := &remoteProvider{ - endpoint: endpoint, - provider: provider, - path: path, + rp := &defaultRemoteProvider{ + endpoint: endpoint, + provider: provider, + path: path, + secretKeyring: secretkeyring, } if !v.providerPathExists(rp) { v.remoteProviders = append(v.remoteProviders, rp) @@ -222,7 +337,7 @@ func (v *Viper) AddSecureRemoteProvider(provider, endpoint, path, secretkeyring return nil } -func (v *Viper) providerPathExists(p *remoteProvider) bool { +func (v *Viper) providerPathExists(p *defaultRemoteProvider) bool { for _, y := range v.remoteProviders { if reflect.DeepEqual(y, p) { return true @@ -231,20 +346,50 @@ func (v *Viper) providerPathExists(p *remoteProvider) bool { return false } +func (v *Viper) searchMap(source map[string]interface{}, path []string) interface{} { + + if len(path) == 0 { + return source + } + + if next, ok := source[path[0]]; ok { + switch next.(type) { + case map[interface{}]interface{}: + return v.searchMap(cast.ToStringMap(next), path[1:]) + case map[string]interface{}: + // Type assertion is safe here since it is only reached + // if the type of `next` is the same as the type being asserted + return v.searchMap(next.(map[string]interface{}), path[1:]) + default: + return next + } + } else { + return nil + } +} + // Viper is essentially repository for configurations // Get can retrieve any value given the key to use // Get has the behavior of returning the value associated with the first // place from where it is set. Viper will check in the following order: -// flag, env, config file, key/value store, default +// override, flag, env, config file, key/value store, default // // Get returns an interface. For a specific value use one of the Get____ methods. func Get(key string) interface{} { return v.Get(key) } func (v *Viper) Get(key string) interface{} { - key = strings.ToLower(key) - val := v.find(key) + path := strings.Split(key, v.keyDelim) + + val := v.find(strings.ToLower(key)) if val == nil { - return nil + source := v.find(path[0]) + if source == nil { + return nil + } + + if reflect.TypeOf(source).Kind() == reflect.Map { + val = v.searchMap(cast.ToStringMap(source), path[1:]) + } } switch val.(type) { @@ -258,87 +403,125 @@ func (v *Viper) Get(key string) interface{} { return cast.ToFloat64(val) case time.Time: return cast.ToTime(val) + case time.Duration: + return cast.ToDuration(val) case []string: return val } return val } +// Returns the value associated with the key as a string func GetString(key string) string { return v.GetString(key) } func (v *Viper) GetString(key string) string { return cast.ToString(v.Get(key)) } +// Returns the value associated with the key asa boolean func GetBool(key string) bool { return v.GetBool(key) } func (v *Viper) GetBool(key string) bool { return cast.ToBool(v.Get(key)) } +// Returns the value associated with the key as an integer func GetInt(key string) int { return v.GetInt(key) } func (v *Viper) GetInt(key string) int { return cast.ToInt(v.Get(key)) } +// Returns the value associated with the key as a float64 func GetFloat64(key string) float64 { return v.GetFloat64(key) } func (v *Viper) GetFloat64(key string) float64 { return cast.ToFloat64(v.Get(key)) } +// Returns the value associated with the key as time func GetTime(key string) time.Time { return v.GetTime(key) } func (v *Viper) GetTime(key string) time.Time { return cast.ToTime(v.Get(key)) } +// Returns the value associated with the key as a duration +func GetDuration(key string) time.Duration { return v.GetDuration(key) } +func (v *Viper) GetDuration(key string) time.Duration { + return cast.ToDuration(v.Get(key)) +} + +// Returns the value associated with the key as a slice of strings func GetStringSlice(key string) []string { return v.GetStringSlice(key) } func (v *Viper) GetStringSlice(key string) []string { return cast.ToStringSlice(v.Get(key)) } +// Returns the value associated with the key as a map of interfaces func GetStringMap(key string) map[string]interface{} { return v.GetStringMap(key) } func (v *Viper) GetStringMap(key string) map[string]interface{} { return cast.ToStringMap(v.Get(key)) } +// Returns the value associated with the key as a map of strings func GetStringMapString(key string) map[string]string { return v.GetStringMapString(key) } func (v *Viper) GetStringMapString(key string) map[string]string { return cast.ToStringMapString(v.Get(key)) } +// Returns the size of the value associated with the given key +// in bytes. +func GetSizeInBytes(key string) uint { return v.GetSizeInBytes(key) } +func (v *Viper) GetSizeInBytes(key string) uint { + sizeStr := cast.ToString(v.Get(key)) + return parseSizeInBytes(sizeStr) +} + // Takes a single key and marshals it into a Struct func MarshalKey(key string, rawVal interface{}) error { return v.MarshalKey(key, rawVal) } func (v *Viper) MarshalKey(key string, rawVal interface{}) error { return mapstructure.Decode(v.Get(key), rawVal) } -// Marshals the config into a Struct +// Marshals the config into a Struct. Make sure that the tags +// on the fields of the structure are properly set. func Marshal(rawVal interface{}) error { return v.Marshal(rawVal) } func (v *Viper) Marshal(rawVal interface{}) error { - err := mapstructure.Decode(v.defaults, rawVal) - if err != nil { - return err - } - err = mapstructure.Decode(v.config, rawVal) - if err != nil { - return err - } - err = mapstructure.Decode(v.override, rawVal) - if err != nil { - return err - } - err = mapstructure.Decode(v.kvstore, rawVal) + err := mapstructure.WeakDecode(v.AllSettings(), rawVal) + if err != nil { return err } - v.insensativiseMaps() + v.insensitiviseMaps() return nil } +// Bind a full flag set to the configuration, using each flag's long +// name as the config key. +func BindPFlags(flags *pflag.FlagSet) (err error) { return v.BindPFlags(flags) } +func (v *Viper) BindPFlags(flags *pflag.FlagSet) (err error) { + flags.VisitAll(func(flag *pflag.Flag) { + if err != nil { + // an error has been encountered in one of the previous flags + return + } + + err = v.BindPFlag(flag.Name, flag) + switch flag.Value.Type() { + case "int", "int8", "int16", "int32", "int64": + v.SetDefault(flag.Name, cast.ToInt(flag.Value.String())) + case "bool": + v.SetDefault(flag.Name, cast.ToBool(flag.Value.String())) + default: + v.SetDefault(flag.Name, flag.Value.String()) + } + }) + return +} + // Bind a specific key to a flag (as used by cobra) +// Example(where serverCmd is a Cobra instance): // // serverCmd.Flags().Int("port", 1138, "Port to run Application server on") -// viper.BindPFlag("port", serverCmd.Flags().Lookup("port")) +// Viper.BindPFlag("port", serverCmd.Flags().Lookup("port")) // func BindPFlag(key string, flag *pflag.Flag) (err error) { return v.BindPFlag(key, flag) } func (v *Viper) BindPFlag(key string, flag *pflag.Flag) (err error) { @@ -349,16 +532,16 @@ func (v *Viper) BindPFlag(key string, flag *pflag.Flag) (err error) { switch flag.Value.Type() { case "int", "int8", "int16", "int32", "int64": - SetDefault(key, cast.ToInt(flag.Value.String())) + v.SetDefault(key, cast.ToInt(flag.Value.String())) case "bool": - SetDefault(key, cast.ToBool(flag.Value.String())) + v.SetDefault(key, cast.ToBool(flag.Value.String())) default: - SetDefault(key, flag.Value.String()) + v.SetDefault(key, flag.Value.String()) } return nil } -// Binds a viper key to a ENV variable +// Binds a Viper key to a ENV variable // ENV variables are case sensitive // If only a key is provided, it will use the env key matching the key, uppercased. // EnvPrefix will be used when set when env name is not provided. @@ -411,7 +594,7 @@ func (v *Viper) find(key string) interface{} { if v.automaticEnvApplied { // even if it hasn't been registered, if automaticEnv is used, // check any Get request - if val = os.Getenv(v.mergeWithEnvPrefix(key)); val != "" { + if val = v.getEnv(v.mergeWithEnvPrefix(key)); val != "" { jww.TRACE.Println(key, "found in environment with val:", val) return val } @@ -420,7 +603,7 @@ func (v *Viper) find(key string) interface{} { envkey, exists := v.env[key] if exists { jww.TRACE.Println(key, "registered as env var", envkey) - if val = os.Getenv(envkey); val != "" { + if val = v.getEnv(envkey); val != "" { jww.TRACE.Println(envkey, "found in environment with val:", val) return val } else { @@ -456,13 +639,21 @@ func (v *Viper) IsSet(key string) bool { return t != nil } -// Have viper check ENV variables for all +// Have Viper check ENV variables for all // keys set in config, default & flags func AutomaticEnv() { v.AutomaticEnv() } func (v *Viper) AutomaticEnv() { v.automaticEnvApplied = true } +// SetEnvKeyReplacer sets the strings.Replacer on the viper object +// Useful for mapping an environmental variable to a key that does +// not match it. +func SetEnvKeyReplacer(r *strings.Replacer) { v.SetEnvKeyReplacer(r) } +func (v *Viper) SetEnvKeyReplacer(r *strings.Replacer) { + v.envKeyReplacer = r +} + // Aliases provide another accessor for the same key. // This enables one to change a name without breaking the application func RegisterAlias(alias string, key string) { v.RegisterAlias(alias, key) } @@ -531,9 +722,9 @@ func (v *Viper) SetDefault(key string, value interface{}) { v.defaults[key] = value } -// The user provided value (via flag) +// Sets the value for the key in the override regiser. // Will be used instead of values obtained via -// config file, ENV, default, or key/value store +// flags, config file, ENV, default, or key/value store func Set(key string, value interface{}) { v.Set(key, value) } func (v *Viper) Set(key string, value interface{}) { // If alias passed in, then set the proper override @@ -555,8 +746,15 @@ func (v *Viper) ReadInConfig() error { return err } - v.marshalReader(bytes.NewReader(file), v.config) - return nil + v.config = make(map[string]interface{}) + + return v.marshalReader(bytes.NewReader(file), v.config) +} + +func ReadConfig(in io.Reader) error { return v.ReadConfig(in) } +func (v *Viper) ReadConfig(in io.Reader) error { + v.config = make(map[string]interface{}) + return v.marshalReader(in, v.config) } // ReadInRawConfig reads the given raw bytes to load configuration. It is @@ -568,6 +766,14 @@ func (v *Viper) ReadInRawConfig(raw []byte) error { return nil } +// func ReadBufConfig(buf *bytes.Buffer) error { return v.ReadBufConfig(buf) } +// func (v *Viper) ReadBufConfig(buf *bytes.Buffer) error { +// v.config = make(map[string]interface{}) +// return v.marshalReader(buf, v.config) +// } + +// Attempts to get configuration from a remote source +// and read it in the remote configuration registry. func ReadRemoteConfig() error { return v.ReadRemoteConfig() } func (v *Viper) ReadRemoteConfig() error { err := v.getKeyValueConfig() @@ -577,22 +783,38 @@ func (v *Viper) ReadRemoteConfig() error { return nil } -// Marshall a Reader into a map -// Should probably be an unexported function -func marshalReader(in io.Reader, c map[string]interface{}) { v.marshalReader(in, c) } -func (v *Viper) marshalReader(in io.Reader, c map[string]interface{}) { - marshallConfigReader(in, c, v.getConfigType()) +func WatchRemoteConfig() error { return v.WatchRemoteConfig() } +func (v *Viper) WatchRemoteConfig() error { + err := v.watchKeyValueConfig() + if err != nil { + return err + } + return nil } -func (v *Viper) insensativiseMaps() { - insensativiseMap(v.config) - insensativiseMap(v.defaults) - insensativiseMap(v.override) - insensativiseMap(v.kvstore) +// Marshall a Reader into a map +// Should probably be an unexported function +func marshalReader(in io.Reader, c map[string]interface{}) error { + return v.marshalReader(in, c) +} + +func (v *Viper) marshalReader(in io.Reader, c map[string]interface{}) error { + return marshallConfigReader(in, c, v.getConfigType()) +} + +func (v *Viper) insensitiviseMaps() { + insensitiviseMap(v.config) + insensitiviseMap(v.defaults) + insensitiviseMap(v.override) + insensitiviseMap(v.kvstore) } // retrieve the first found remote configuration func (v *Viper) getKeyValueConfig() error { + if RemoteConfig == nil { + return RemoteConfigError("Enable the remote features by doing a blank import of the viper/remote package: '_ github.com/spf13/viper/remote'") + } + for _, rp := range v.remoteProviders { val, err := v.getRemoteConfig(rp) if err != nil { @@ -604,37 +826,35 @@ func (v *Viper) getKeyValueConfig() error { return RemoteConfigError("No Files Found") } -func (v *Viper) getRemoteConfig(provider *remoteProvider) (map[string]interface{}, error) { - var cm crypt.ConfigManager - var err error +func (v *Viper) getRemoteConfig(provider *defaultRemoteProvider) (map[string]interface{}, error) { - if provider.secretKeyring != "" { - kr, err := os.Open(provider.secretKeyring) - defer kr.Close() + reader, err := RemoteConfig.Get(provider) + if err != nil { + return nil, err + } + err = v.marshalReader(reader, v.kvstore) + return v.kvstore, err +} + +// retrieve the first found remote configuration +func (v *Viper) watchKeyValueConfig() error { + for _, rp := range v.remoteProviders { + val, err := v.watchRemoteConfig(rp) if err != nil { - return nil, err - } - if provider.provider == "etcd" { - cm, err = crypt.NewEtcdConfigManager([]string{provider.endpoint}, kr) - } else { - cm, err = crypt.NewConsulConfigManager([]string{provider.endpoint}, kr) - } - } else { - if provider.provider == "etcd" { - cm, err = crypt.NewStandardEtcdConfigManager([]string{provider.endpoint}) - } else { - cm, err = crypt.NewStandardConsulConfigManager([]string{provider.endpoint}) + continue } + v.kvstore = val + return nil } + return RemoteConfigError("No Files Found") +} + +func (v *Viper) watchRemoteConfig(provider *defaultRemoteProvider) (map[string]interface{}, error) { + reader, err := RemoteConfig.Watch(provider) if err != nil { return nil, err } - b, err := cm.Get(provider.path) - if err != nil { - return nil, err - } - reader := bytes.NewReader(b) - v.marshalReader(reader, v.kvstore) + err = v.marshalReader(reader, v.kvstore) return v.kvstore, err } @@ -687,6 +907,8 @@ func (v *Viper) SetConfigName(in string) { } } +// Sets the type of the configuration returned by the +// remote source, e.g. "json". func SetConfigType(in string) { v.SetConfigType(in) } func (v *Viper) SetConfigType(in string) { if in != "" { @@ -740,6 +962,7 @@ func (v *Viper) searchInPath(in string) (filename string) { // search all configPaths for any config file. // Returns the first path that exists (and is a config file) func (v *Viper) findConfigFile() (string, error) { + jww.INFO.Println("Searching for config in ", v.configPaths) for _, cp := range v.configPaths { @@ -748,34 +971,25 @@ func (v *Viper) findConfigFile() (string, error) { return file, nil } } - - cwd, _ := findCWD() - file := v.searchInPath(cwd) - if file != "" { - return file, nil - } - - // try the current working directory - wd, _ := os.Getwd() - file = v.searchInPath(wd) - if file != "" { - return file, nil - } - return "", fmt.Errorf("config file not found in: %s", v.configPaths) + return "", ConfigFileNotFoundError{v.configName, fmt.Sprintf("%s", v.configPaths)} } +// Prints all configuration registries for debugging +// purposes. func Debug() { v.Debug() } func (v *Viper) Debug() { - fmt.Println("Config:") - pretty.Println(v.config) - fmt.Println("Key/Value Store:") - pretty.Println(v.kvstore) - fmt.Println("Env:") - pretty.Println(v.env) - fmt.Println("Defaults:") - pretty.Println(v.defaults) - fmt.Println("Override:") - pretty.Println(v.override) fmt.Println("Aliases:") pretty.Println(v.aliases) + fmt.Println("Override:") + pretty.Println(v.override) + fmt.Println("PFlags") + pretty.Println(v.pflags) + fmt.Println("Env:") + pretty.Println(v.env) + fmt.Println("Key/Value Store:") + pretty.Println(v.kvstore) + fmt.Println("Config:") + pretty.Println(v.config) + fmt.Println("Defaults:") + pretty.Println(v.defaults) } diff --git a/viper_test.go b/viper_test.go index ce1aa3c..a5f2c52 100644 --- a/viper_test.go +++ b/viper_test.go @@ -8,8 +8,12 @@ package viper import ( "bytes" "fmt" + "io/ioutil" "os" + "path" + "reflect" "sort" + "strings" "testing" "time" @@ -26,6 +30,8 @@ hobbies: clothing: jacket: leather trousers: denim + pants: + size: large age: 35 eyes : brown beard: true @@ -54,12 +60,13 @@ var jsonExample = []byte(`{ } }`) -// Intended for testing, will reset all to default settings. -func Reset() { - v = New() - SupportedExts = []string{"json", "toml", "yaml", "yml"} - SupportedRemoteProviders = []string{"etcd", "consul"} -} +var propertiesExample = []byte(` +p_id: 0001 +p_type: donut +p_name: Cake +p_ppu: 0.55 +p_batters.batter.type: Regular +`) var remoteExample = []byte(`{ "id":"0002", @@ -77,6 +84,10 @@ func initConfigs() { r = bytes.NewReader(jsonExample) marshalReader(r, v.config) + SetConfigType("properties") + r = bytes.NewReader(propertiesExample) + marshalReader(r, v.config) + SetConfigType("toml") r = bytes.NewReader(tomlExample) marshalReader(r, v.config) @@ -102,6 +113,14 @@ func initJSON() { marshalReader(r, v.config) } +func initProperties() { + Reset() + SetConfigType("properties") + r := bytes.NewReader(propertiesExample) + + marshalReader(r, v.config) +} + func initTOML() { Reset() SetConfigType("toml") @@ -110,6 +129,47 @@ func initTOML() { marshalReader(r, v.config) } +// make directories for testing +func initDirs(t *testing.T) (string, string, func()) { + + var ( + testDirs = []string{`a a`, `b`, `c\c`, `D_`} + config = `improbable` + ) + + root, err := ioutil.TempDir("", "") + + cleanup := true + defer func() { + if cleanup { + os.Chdir("..") + os.RemoveAll(root) + } + }() + + assert.Nil(t, err) + + err = os.Chdir(root) + assert.Nil(t, err) + + for _, dir := range testDirs { + err = os.Mkdir(dir, 0750) + assert.Nil(t, err) + + err = ioutil.WriteFile( + path.Join(dir, config+".toml"), + []byte("key = \"value is "+dir+"\"\n"), + 0640) + assert.Nil(t, err) + } + + cleanup = false + return root, config, func() { + os.Chdir("..") + os.RemoveAll(root) + } +} + //stubs for PFlag Values type stringValue string @@ -150,7 +210,7 @@ func TestMarshalling(t *testing.T) { assert.False(t, InConfig("state")) assert.Equal(t, "steve", Get("name")) assert.Equal(t, []interface{}{"skateboarding", "snowboarding", "go"}, Get("hobbies")) - assert.Equal(t, map[interface{}]interface{}{"jacket": "leather", "trousers": "denim"}, Get("clothing")) + assert.Equal(t, map[interface{}]interface{}{"jacket": "leather", "trousers": "denim", "pants": map[interface{}]interface{}{"size": "large"}}, Get("clothing")) assert.Equal(t, 35, Get("age")) } @@ -191,6 +251,11 @@ func TestJSON(t *testing.T) { assert.Equal(t, "0001", Get("id")) } +func TestProperties(t *testing.T) { + initProperties() + assert.Equal(t, "0001", Get("p_id")) +} + func TestTOML(t *testing.T) { initTOML() assert.Equal(t, "TOML Example", Get("title")) @@ -228,6 +293,7 @@ func TestEnv(t *testing.T) { AutomaticEnv() assert.Equal(t, "crunk", Get("name")) + } func TestEnvPrefix(t *testing.T) { @@ -250,12 +316,41 @@ func TestEnvPrefix(t *testing.T) { assert.Equal(t, "crunk", Get("name")) } +func TestAutoEnv(t *testing.T) { + Reset() + + AutomaticEnv() + os.Setenv("FOO_BAR", "13") + assert.Equal(t, "13", Get("foo_bar")) +} + +func TestAutoEnvWithPrefix(t *testing.T) { + Reset() + + AutomaticEnv() + SetEnvPrefix("Baz") + os.Setenv("BAZ_BAR", "13") + assert.Equal(t, "13", Get("bar")) +} + +func TestSetEnvReplacer(t *testing.T) { + Reset() + + AutomaticEnv() + os.Setenv("REFRESH_INTERVAL", "30s") + + replacer := strings.NewReplacer("-", "_") + SetEnvKeyReplacer(replacer) + + assert.Equal(t, "30s", Get("refresh-interval")) +} + func TestAllKeys(t *testing.T) { initConfigs() - ks := sort.StringSlice{"title", "newkey", "owner", "name", "beard", "ppu", "batters", "hobbies", "clothing", "age", "hacker", "id", "type", "eyes"} + ks := sort.StringSlice{"title", "newkey", "owner", "name", "beard", "ppu", "batters", "hobbies", "clothing", "age", "hacker", "id", "type", "eyes", "p_id", "p_ppu", "p_batters.batter.type", "p_type", "p_name"} dob, _ := time.Parse(time.RFC3339, "1979-05-27T07:32:00Z") - all := map[string]interface{}{"owner": map[string]interface{}{"organization": "MongoDB", "Bio": "MongoDB Chief Developer Advocate & Hacker at Large", "dob": dob}, "title": "TOML Example", "ppu": 0.55, "eyes": "brown", "clothing": map[interface{}]interface{}{"trousers": "denim", "jacket": "leather"}, "id": "0001", "batters": map[string]interface{}{"batter": []interface{}{map[string]interface{}{"type": "Regular"}, map[string]interface{}{"type": "Chocolate"}, map[string]interface{}{"type": "Blueberry"}, map[string]interface{}{"type": "Devil's Food"}}}, "hacker": true, "beard": true, "hobbies": []interface{}{"skateboarding", "snowboarding", "go"}, "age": 35, "type": "donut", "newkey": "remote", "name": "Cake"} + all := map[string]interface{}{"owner": map[string]interface{}{"organization": "MongoDB", "Bio": "MongoDB Chief Developer Advocate & Hacker at Large", "dob": dob}, "title": "TOML Example", "ppu": 0.55, "eyes": "brown", "clothing": map[interface{}]interface{}{"trousers": "denim", "jacket": "leather", "pants": map[interface{}]interface{}{"size": "large"}}, "id": "0001", "batters": map[string]interface{}{"batter": []interface{}{map[string]interface{}{"type": "Regular"}, map[string]interface{}{"type": "Chocolate"}, map[string]interface{}{"type": "Blueberry"}, map[string]interface{}{"type": "Devil's Food"}}}, "hacker": true, "beard": true, "hobbies": []interface{}{"skateboarding", "snowboarding", "go"}, "age": 35, "type": "donut", "newkey": "remote", "name": "Cake", "p_id": "0001", "p_ppu": "0.55", "p_name": "Cake", "p_batters.batter.type": "Regular", "p_type": "donut"} var allkeys sort.StringSlice allkeys = AllKeys() @@ -309,6 +404,41 @@ func TestMarshal(t *testing.T) { assert.Equal(t, &C, &config{Name: "Steve", Port: 1234}) } +func TestBindPFlags(t *testing.T) { + flagSet := pflag.NewFlagSet("test", pflag.ContinueOnError) + + var testValues = map[string]*string{ + "host": nil, + "port": nil, + "endpoint": nil, + } + + var mutatedTestValues = map[string]string{ + "host": "localhost", + "port": "6060", + "endpoint": "/public", + } + + for name, _ := range testValues { + testValues[name] = flagSet.String(name, "", "test") + } + + err := BindPFlags(flagSet) + if err != nil { + t.Fatalf("error binding flag set, %v", err) + } + + flagSet.VisitAll(func(flag *pflag.Flag) { + flag.Value.Set(mutatedTestValues[flag.Name]) + flag.Changed = true + }) + + for name, expected := range mutatedTestValues { + assert.Equal(t, Get(name), expected) + } + +} + func TestBindPFlag(t *testing.T) { var testString = "testing" var testValue = newStringValue(testString, &testString) @@ -352,3 +482,164 @@ func TestBoundCaseSensitivity(t *testing.T) { assert.Equal(t, "green", Get("eyes")) } + +func TestSizeInBytes(t *testing.T) { + input := map[string]uint{ + "": 0, + "b": 0, + "12 bytes": 0, + "200000000000gb": 0, + "12 b": 12, + "43 MB": 43 * (1 << 20), + "10mb": 10 * (1 << 20), + "1gb": 1 << 30, + } + + for str, expected := range input { + assert.Equal(t, expected, parseSizeInBytes(str), str) + } +} + +func TestFindsNestedKeys(t *testing.T) { + initConfigs() + dob, _ := time.Parse(time.RFC3339, "1979-05-27T07:32:00Z") + + Set("super", map[string]interface{}{ + "deep": map[string]interface{}{ + "nested": "value", + }, + }) + + expected := map[string]interface{}{ + "super": map[string]interface{}{ + "deep": map[string]interface{}{ + "nested": "value", + }, + }, + "super.deep": map[string]interface{}{ + "nested": "value", + }, + "super.deep.nested": "value", + "owner.organization": "MongoDB", + "batters.batter": []interface{}{ + map[string]interface{}{ + "type": "Regular", + }, + map[string]interface{}{ + "type": "Chocolate", + }, + map[string]interface{}{ + "type": "Blueberry", + }, + map[string]interface{}{ + "type": "Devil's Food", + }, + }, + "hobbies": []interface{}{ + "skateboarding", "snowboarding", "go", + }, + "title": "TOML Example", + "newkey": "remote", + "batters": map[string]interface{}{ + "batter": []interface{}{ + map[string]interface{}{ + "type": "Regular", + }, + map[string]interface{}{ + "type": "Chocolate", + }, map[string]interface{}{ + "type": "Blueberry", + }, map[string]interface{}{ + "type": "Devil's Food", + }, + }, + }, + "eyes": "brown", + "age": 35, + "owner": map[string]interface{}{ + "organization": "MongoDB", + "Bio": "MongoDB Chief Developer Advocate & Hacker at Large", + "dob": dob, + }, + "owner.Bio": "MongoDB Chief Developer Advocate & Hacker at Large", + "type": "donut", + "id": "0001", + "name": "Cake", + "hacker": true, + "ppu": 0.55, + "clothing": map[interface{}]interface{}{ + "jacket": "leather", + "trousers": "denim", + "pants": map[interface{}]interface{}{ + "size": "large", + }, + }, + "clothing.jacket": "leather", + "clothing.pants.size": "large", + "clothing.trousers": "denim", + "owner.dob": dob, + "beard": true, + } + + for key, expectedValue := range expected { + + assert.Equal(t, expectedValue, v.Get(key)) + } + +} + +func TestReadBufConfig(t *testing.T) { + v := New() + v.SetConfigType("yaml") + v.ReadConfig(bytes.NewBuffer(yamlExample)) + t.Log(v.AllKeys()) + + assert.True(t, v.InConfig("name")) + assert.False(t, v.InConfig("state")) + assert.Equal(t, "steve", v.Get("name")) + assert.Equal(t, []interface{}{"skateboarding", "snowboarding", "go"}, v.Get("hobbies")) + assert.Equal(t, map[interface{}]interface{}{"jacket": "leather", "trousers": "denim", "pants": map[interface{}]interface{}{"size": "large"}}, v.Get("clothing")) + assert.Equal(t, 35, v.Get("age")) +} + +func TestDirsSearch(t *testing.T) { + + root, config, cleanup := initDirs(t) + defer cleanup() + + v := New() + v.SetConfigName(config) + v.SetDefault(`key`, `default`) + + entries, err := ioutil.ReadDir(root) + for _, e := range entries { + if e.IsDir() { + v.AddConfigPath(e.Name()) + } + } + + err = v.ReadInConfig() + assert.Nil(t, err) + + assert.Equal(t, `value is `+path.Base(v.configPaths[0]), v.GetString(`key`)) +} + +func TestWrongDirsSearchNotFound(t *testing.T) { + + _, config, cleanup := initDirs(t) + defer cleanup() + + v := New() + v.SetConfigName(config) + v.SetDefault(`key`, `default`) + + v.AddConfigPath(`whattayoutalkingbout`) + v.AddConfigPath(`thispathaintthere`) + + err := v.ReadInConfig() + assert.Equal(t, reflect.TypeOf(UnsupportedConfigError("")), reflect.TypeOf(err)) + + // Even though config did not load and the error might have + // been ignored by the client, the default still loads + assert.Equal(t, `default`, v.GetString(`key`)) +}