From 9f725cec5c67b47e88c0b926560988ef9a21b3c5 Mon Sep 17 00:00:00 2001 From: oliveagle Date: Fri, 8 May 2015 17:13:33 +0800 Subject: [PATCH 1/5] ReadBufConfig --- viper.go | 75 +++++++++++++++++++++++++++++++++++++++++++++++++++ viper_test.go | 14 ++++++++++ 2 files changed, 89 insertions(+) diff --git a/viper.go b/viper.go index 73bc6f3..bd41301 100644 --- a/viper.go +++ b/viper.go @@ -36,6 +36,8 @@ import ( jww "github.com/spf13/jwalterweatherman" "github.com/spf13/pflag" crypt "github.com/xordataexchange/crypt/config" + + // log "github.com/oliveagle/seelog" ) var v *Viper @@ -268,6 +270,8 @@ func (v *Viper) AddRemoteProvider(provider, endpoint, path string) error { v.remoteProviders = append(v.remoteProviders, rp) } } + + // log.Info("remoteProviders", v.remoteProviders) return nil } @@ -716,6 +720,13 @@ func (v *Viper) ReadInConfig() error { return nil } +func ReadBufConfig(buf *bytes.Buffer) error { return v.ReadBufConfig(buf) } +func (v *Viper) ReadBufConfig(buf *bytes.Buffer) error { + v.config = make(map[string]interface{}) + v.marshalReader(buf, v.config) + return nil +} + // Attempts to get configuration from a remote source // and read it in the remote configuration registry. func ReadRemoteConfig() error { return v.ReadRemoteConfig() } @@ -727,6 +738,15 @@ func (v *Viper) ReadRemoteConfig() error { return nil } +func WatchRemoteConfig() error { return v.WatchRemoteConfig() } +func (v *Viper) WatchRemoteConfig() error { + err := v.watchKeyValueConfig() + if err != nil { + return err + } + return nil +} + // Marshall a Reader into a map // Should probably be an unexported function func marshalReader(in io.Reader, c map[string]interface{}) { v.marshalReader(in, c) } @@ -788,6 +808,61 @@ func (v *Viper) getRemoteConfig(provider *remoteProvider) (map[string]interface{ return v.kvstore, err } +// retrieve the first found remote configuration +func (v *Viper) watchKeyValueConfig() error { + // log.Info("ahahahah==========================================") + // log.Info("remoteProviders", v.remoteProviders) + + for _, rp := range v.remoteProviders { + val, err := v.watchRemoteConfig(rp) + if err != nil { + continue + } + v.kvstore = val + return nil + } + return RemoteConfigError("No Files Found") +} + +func (v *Viper) watchRemoteConfig(provider *remoteProvider) (map[string]interface{}, error) { + // defer log.Flush() + // log.Info("ahahahah==========================================") + var cm crypt.ConfigManager + var err error + + if provider.secretKeyring != "" { + kr, err := os.Open(provider.secretKeyring) + defer kr.Close() + if err != nil { + return nil, err + } + if provider.provider == "etcd" { + cm, err = crypt.NewEtcdConfigManager([]string{provider.endpoint}, kr) + } else { + cm, err = crypt.NewConsulConfigManager([]string{provider.endpoint}, kr) + } + } else { + if provider.provider == "etcd" { + cm, err = crypt.NewStandardEtcdConfigManager([]string{provider.endpoint}) + } else { + cm, err = crypt.NewStandardConsulConfigManager([]string{provider.endpoint}) + } + } + if err != nil { + return nil, err + } + resp := <-cm.Watch(provider.path, nil) + // b, err := cm.Watch(provider.path, nil) + err = resp.Error + if err != nil { + return nil, err + } + + reader := bytes.NewReader(resp.Value) + v.marshalReader(reader, v.kvstore) + return v.kvstore, err +} + // Return all keys regardless where they are set func AllKeys() []string { return v.AllKeys() } func (v *Viper) AllKeys() []string { diff --git a/viper_test.go b/viper_test.go index 1039039..3f58820 100644 --- a/viper_test.go +++ b/viper_test.go @@ -537,3 +537,17 @@ func TestFindsNestedKeys(t *testing.T) { } } + +func TestReadBufConfig(t *testing.T) { + v := New() + v.SetConfigType("yaml") + v.ReadBufConfig(bytes.NewBuffer(yamlExample)) + t.Log(v.AllKeys()) + + assert.True(t, v.InConfig("name")) + assert.False(t, v.InConfig("state")) + assert.Equal(t, "steve", v.Get("name")) + assert.Equal(t, []interface{}{"skateboarding", "snowboarding", "go"}, v.Get("hobbies")) + assert.Equal(t, map[interface{}]interface{}{"jacket": "leather", "trousers": "denim"}, v.Get("clothing")) + assert.Equal(t, 35, v.Get("age")) +} From 1c1277d5b36da4dbcaebc846af1f0b56c8fcd78f Mon Sep 17 00:00:00 2001 From: oliveagle Date: Fri, 8 May 2015 17:18:00 +0800 Subject: [PATCH 2/5] clean a little: added watch and ReadBufConf --- viper.go | 9 --------- 1 file changed, 9 deletions(-) diff --git a/viper.go b/viper.go index bd41301..351e30c 100644 --- a/viper.go +++ b/viper.go @@ -36,8 +36,6 @@ import ( jww "github.com/spf13/jwalterweatherman" "github.com/spf13/pflag" crypt "github.com/xordataexchange/crypt/config" - - // log "github.com/oliveagle/seelog" ) var v *Viper @@ -270,8 +268,6 @@ func (v *Viper) AddRemoteProvider(provider, endpoint, path string) error { v.remoteProviders = append(v.remoteProviders, rp) } } - - // log.Info("remoteProviders", v.remoteProviders) return nil } @@ -810,9 +806,6 @@ func (v *Viper) getRemoteConfig(provider *remoteProvider) (map[string]interface{ // retrieve the first found remote configuration func (v *Viper) watchKeyValueConfig() error { - // log.Info("ahahahah==========================================") - // log.Info("remoteProviders", v.remoteProviders) - for _, rp := range v.remoteProviders { val, err := v.watchRemoteConfig(rp) if err != nil { @@ -825,8 +818,6 @@ func (v *Viper) watchKeyValueConfig() error { } func (v *Viper) watchRemoteConfig(provider *remoteProvider) (map[string]interface{}, error) { - // defer log.Flush() - // log.Info("ahahahah==========================================") var cm crypt.ConfigManager var err error From 9c9f7f12c8d87897b394946a48b4769694bd9c13 Mon Sep 17 00:00:00 2001 From: oliveagle Date: Mon, 11 May 2015 12:33:35 +0800 Subject: [PATCH 3/5] add README.md, and fix strings.ToLower(configType) --- README.md | 65 ++++++++++++++++++++++++++++++++++++++++++++++++++++++- util.go | 2 +- 2 files changed, 65 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 4e72e97..cd42a2f 100644 --- a/README.md +++ b/README.md @@ -14,8 +14,9 @@ configuration. It supports * setting defaults * reading from json, toml and yaml config files * reading from environment variables -* reading from remote config systems (Etcd or Consul) +* reading from remote config systems (Etcd or Consul), watching changes * reading from command line flags +* reading from buffer * setting explicit values It can be thought of as a registry for all of your applications @@ -81,6 +82,36 @@ currently a single viper only supports a single config file. panic(fmt.Errorf("Fatal error config file: %s \n", err)) } +### Reading Config from bytes.Buffer + +Viper predefined many configuration sources, such as files, environment variables, flags and +remote K/V store. But you are not bound to them. You can also implement your own way to +require configuration and feed it to viper. + +````go +viper.SetConfigType("yaml") // or viper.SetConfigType("YAML") + +// any approach to require this configuration into your program. +var yamlExample = []byte(` +Hacker: true +name: steve +hobbies: +- skateboarding +- snowboarding +- go +clothing: + jacket: leather + trousers: denim +age: 35 +eyes : brown +beard: true +`) + +viper.ReadBufConfig(bytes.NewBuffer(yamlExample)) + +viper.Get("name") // this would be "steve" +```` + ### Setting Overrides These could be from a command line flag, or from your own application logic. @@ -208,6 +239,38 @@ to use Consul. viper.SetConfigType("json") // because there is no file extension in a stream of bytes err := viper.ReadRemoteConfig() +### Watching Changes in Etcd - Unencrypted + + // alternatively, you can create a new viper instance. + var runtime_viper = viper.New() + + runtime_viper.AddRemoteProvider("etcd", "http://127.0.0.1:4001", "/config/hugo.yml") + runtime_viper.SetConfigType("yaml") // because there is no file extension in a stream of bytes + + // read from remote config the first time. + err := runtime_viper.ReadRemoteConfig() + + // marshal config + runtime_viper.Marshal(&runtime_conf) + + // open a goroutine to wath remote changes forever + go func(){ + for { + time.Sleep(time.Second * 5) // delay after each request + + // currenlty, only tested with etcd support + err := runtime_viper.WatchRemoteConfig() + if err != nil { + log.Errorf("unable to read remote config: %v", err) + continue + } + + // marshal new config into our runtime config struct. you can also use channel + // to implement a signal to notify the system of the changes + runtime_viper.Marshal(&runtime_conf) + } + }() + ## Getting Values From Viper diff --git a/util.go b/util.go index 6fef1a4..981f57b 100644 --- a/util.go +++ b/util.go @@ -128,7 +128,7 @@ func marshallConfigReader(in io.Reader, c map[string]interface{}, configType str buf := new(bytes.Buffer) buf.ReadFrom(in) - switch configType { + switch strings.ToLower(configType) { case "yaml", "yml": if err := yaml.Unmarshal(buf.Bytes(), &c); err != nil { jww.ERROR.Fatalf("Error parsing config: %s", err) From 5736fabdfa955468b811e700f35d79cacdeb76b1 Mon Sep 17 00:00:00 2001 From: oliveagle Date: Thu, 14 May 2015 17:40:59 +0800 Subject: [PATCH 4/5] replace bytes.Buffer with io.Reader --- viper.go | 13 ++++++++++--- viper_test.go | 2 +- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/viper.go b/viper.go index 351e30c..946f9c1 100644 --- a/viper.go +++ b/viper.go @@ -716,13 +716,20 @@ func (v *Viper) ReadInConfig() error { return nil } -func ReadBufConfig(buf *bytes.Buffer) error { return v.ReadBufConfig(buf) } -func (v *Viper) ReadBufConfig(buf *bytes.Buffer) error { +func ReadConfig(in io.Reader) error { return v.ReadConfig(in) } +func (v *Viper) ReadConfig(in io.Reader) error { v.config = make(map[string]interface{}) - v.marshalReader(buf, v.config) + v.marshalReader(in, v.config) return nil } +// func ReadBufConfig(buf *bytes.Buffer) error { return v.ReadBufConfig(buf) } +// func (v *Viper) ReadBufConfig(buf *bytes.Buffer) error { +// v.config = make(map[string]interface{}) +// v.marshalReader(buf, v.config) +// return nil +// } + // Attempts to get configuration from a remote source // and read it in the remote configuration registry. func ReadRemoteConfig() error { return v.ReadRemoteConfig() } diff --git a/viper_test.go b/viper_test.go index 3f58820..10e15af 100644 --- a/viper_test.go +++ b/viper_test.go @@ -541,7 +541,7 @@ func TestFindsNestedKeys(t *testing.T) { func TestReadBufConfig(t *testing.T) { v := New() v.SetConfigType("yaml") - v.ReadBufConfig(bytes.NewBuffer(yamlExample)) + v.ReadConfig(bytes.NewBuffer(yamlExample)) t.Log(v.AllKeys()) assert.True(t, v.InConfig("name")) From 23d0f4785a3fe6309155e4c29d167988d9189fc4 Mon Sep 17 00:00:00 2001 From: oliveagle Date: Fri, 15 May 2015 19:36:05 +0800 Subject: [PATCH 5/5] update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index cd42a2f..16d017b 100644 --- a/README.md +++ b/README.md @@ -82,7 +82,7 @@ currently a single viper only supports a single config file. panic(fmt.Errorf("Fatal error config file: %s \n", err)) } -### Reading Config from bytes.Buffer +### Reading Config from io.Reader Viper predefined many configuration sources, such as files, environment variables, flags and remote K/V store. But you are not bound to them. You can also implement your own way to @@ -107,7 +107,7 @@ eyes : brown beard: true `) -viper.ReadBufConfig(bytes.NewBuffer(yamlExample)) +viper.ReadConfig(bytes.NewBuffer(yamlExample)) viper.Get("name") // this would be "steve" ````