Added config

Signed-off-by: Joe Elliott <number101010@gmail.com>

.gitignore

@@ -1 +1 @@
-frigg
+./frigg

cmd/frigg/cfg/cfg.go

@@ -0,0 +1,61 @@
+package cfg
+
+import (
+	"reflect"
+
+	"github.com/pkg/errors"
+)
+
+// Source is a generic configuration source. This function may do whatever is
+// required to obtain the configuration. It is passed a pointer to the
+// destination, which will be something compatible to `json.Unmarshal`. The
+// obtained configuration may be written to this object, it may also contain
+// data from previous sources.
+type Source func(interface{}) error
+
+var (
+	ErrNotPointer = errors.New("dst is not a pointer")
+)
+
+// Unmarshal merges the values of the various configuration sources and sets them on
+// `dst`. The object must be compatible with `json.Unmarshal`.
+func Unmarshal(dst interface{}, sources ...Source) error {
+	if len(sources) == 0 {
+		panic("No sources supplied to cfg.Unmarshal(). This is most likely a programming issue and should never happen. Check the code!")
+	}
+	if reflect.ValueOf(dst).Kind() != reflect.Ptr {
+		return ErrNotPointer
+	}
+
+	for _, source := range sources {
+		if err := source(dst); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// Parse is a higher level wrapper for Unmarshal that automatically parses flags and a .yaml file
+func Parse(dst interface{}) error {
+	return dParse(dst,
+		Defaults(),
+		YAMLFlag("config.file", "", "yaml file to load"),
+		Flags(),
+	)
+}
+
+// dParse is the same as Parse, but with dependency injection for testing
+func dParse(dst interface{}, defaults, yaml, flags Source) error {
+	// check dst is a pointer
+	v := reflect.ValueOf(dst)
+	if v.Kind() != reflect.Ptr {
+		return ErrNotPointer
+	}
+
+	// unmarshal config
+	return Unmarshal(dst,
+		defaults,
+		yaml,
+		flags,
+	)
+}

cmd/frigg/cfg/cfg_test.go

@@ -0,0 +1,65 @@
+package cfg
+
+import (
+	"flag"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestParse(t *testing.T) {
+	yamlSource := dYAML([]byte(`
+server:
+  port: 2000
+  timeout: 60h
+tls:
+  key: YAML
+`))
+
+	fs := flag.NewFlagSet(t.Name(), flag.PanicOnError)
+	flagSource := dFlags(fs, []string{"-verbose", "-server.port=21"})
+
+	data := Data{}
+	err := dParse(&data,
+		dDefaults(fs),
+		yamlSource,
+		flagSource,
+	)
+	require.NoError(t, err)
+
+	assert.Equal(t, Data{
+		Verbose: true, // flag
+		Server: Server{
+			Port:    21,             // flag
+			Timeout: 60 * time.Hour, // defaults
+		},
+		TLS: TLS{
+			Cert: "DEFAULTCERT", // defaults
+			Key:  "YAML",        // yaml
+		},
+	}, data)
+}
+
+func TestParseWithInvalidYAML(t *testing.T) {
+	yamlSource := dYAML([]byte(`
+servers:
+  ports: 2000
+  timeoutz: 60h
+tls:
+  keey: YAML
+`))
+
+	fs := flag.NewFlagSet(t.Name(), flag.PanicOnError)
+	flagSource := dFlags(fs, []string{"-verbose", "-server.port=21"})
+
+	data := Data{}
+	err := dParse(&data,
+		dDefaults(fs),
+		yamlSource,
+		flagSource,
+	)
+	require.Error(t, err)
+	require.Equal(t, err.Error(), "yaml: unmarshal errors:\n  line 2: field servers not found in type cfg.Data\n  line 6: field keey not found in type cfg.TLS")
+}

cmd/frigg/cfg/data_test.go

@@ -0,0 +1,33 @@
+package cfg
+
+import (
+	"flag"
+	"time"
+)
+
+// Data is a test Data structure
+type Data struct {
+	Verbose bool   `yaml:"verbose"`
+	Server  Server `yaml:"server"`
+	TLS     TLS    `yaml:"tls"`
+}
+
+type Server struct {
+	Port    int           `yaml:"port"`
+	Timeout time.Duration `yaml:"timeout"`
+}
+
+type TLS struct {
+	Cert string `yaml:"cert"`
+	Key  string `yaml:"key"`
+}
+
+// RegisterFlags makes Data implement flagext.Registerer for using flags
+func (d *Data) RegisterFlags(fs *flag.FlagSet) {
+	fs.BoolVar(&d.Verbose, "verbose", false, "")
+	fs.IntVar(&d.Server.Port, "server.port", 80, "")
+	fs.DurationVar(&d.Server.Timeout, "server.timeout", 60*time.Second, "")
+
+	fs.StringVar(&d.TLS.Cert, "tls.cert", "DEFAULTCERT", "")
+	fs.StringVar(&d.TLS.Key, "tls.key", "DEFAULTKEY", "")
+}

cmd/frigg/cfg/files.go

@@ -0,0 +1,87 @@
+package cfg
+
+import (
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"os"
+
+	"github.com/pkg/errors"
+	"gopkg.in/yaml.v2"
+)
+
+// JSON returns a Source that opens the supplied `.json` file and loads it.
+func JSON(f *string) Source {
+	return func(dst interface{}) error {
+		if f == nil {
+			return nil
+		}
+
+		j, err := ioutil.ReadFile(*f)
+		if err != nil {
+			return err
+		}
+
+		err = dJSON(j)(dst)
+		return errors.Wrap(err, *f)
+	}
+}
+
+// dJSON returns a JSON source and allows dependency injection
+func dJSON(y []byte) Source {
+	return func(dst interface{}) error {
+		return json.Unmarshal(y, dst)
+	}
+}
+
+// YAML returns a Source that opens the supplied `.yaml` file and loads it.
+func YAML(f *string) Source {
+	return func(dst interface{}) error {
+		if f == nil {
+			return nil
+		}
+
+		y, err := ioutil.ReadFile(*f)
+		if err != nil {
+			return err
+		}
+
+		err = dYAML(y)(dst)
+		return errors.Wrap(err, *f)
+	}
+}
+
+// dYAML returns a YAML source and allows dependency injection
+func dYAML(y []byte) Source {
+	return func(dst interface{}) error {
+		return yaml.UnmarshalStrict(y, dst)
+	}
+}
+
+// YAMLFlag defines a `config.file` flag and loads this file
+func YAMLFlag(name, value, help string) Source {
+	return func(dst interface{}) error {
+		f := flag.String(name, value, help)
+
+		usage := flag.CommandLine.Usage
+		flag.CommandLine.Usage = func() { /* don't do anything by default, we will print usage ourselves, but only when requested. */ }
+
+		flag.CommandLine.Init(flag.CommandLine.Name(), flag.ContinueOnError)
+		err := flag.CommandLine.Parse(os.Args[1:])
+		if err == flag.ErrHelp {
+			// print available parameters to stdout, so that users can grep/less it easily
+			flag.CommandLine.SetOutput(os.Stdout)
+			usage()
+			os.Exit(2)
+		} else if err != nil {
+			fmt.Fprintln(flag.CommandLine.Output(), "Run with -help to get list of available parameters")
+			os.Exit(2)
+		}
+
+		if *f == "" {
+			f = nil
+		}
+		return YAML(f)(dst)
+	}
+}

cmd/frigg/cfg/flag.go

@@ -0,0 +1,108 @@
+package cfg
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"sort"
+	"strings"
+
+	"github.com/cortexproject/cortex/pkg/util/flagext"
+	"github.com/pkg/errors"
+)
+
+// Defaults registers flags to the command line using dst as the
+// flagext.Registerer
+func Defaults() Source {
+	return dDefaults(flag.CommandLine)
+}
+
+// dDefaults registers flags to the flagSet using dst as the flagext.Registerer
+func dDefaults(fs *flag.FlagSet) Source {
+	return func(dst interface{}) error {
+		r, ok := dst.(flagext.Registerer)
+		if !ok {
+			return errors.New("dst does not satisfy flagext.Registerer")
+		}
+
+		// already sets the defaults on r
+		r.RegisterFlags(fs)
+		return nil
+	}
+}
+
+// Flags parses the flag from the command line, setting only user-supplied
+// values on the flagext.Registerer passed to Defaults()
+func Flags() Source {
+	flag.Usage = categorizedUsage(flag.CommandLine)
+	return dFlags(flag.CommandLine, os.Args[1:])
+}
+
+// dFlags parses the flagset, applying all values set on the slice
+func dFlags(fs *flag.FlagSet, args []string) Source {
+	return func(dst interface{}) error {
+		// parse the final flagset
+		return fs.Parse(args)
+	}
+}
+
+func categorizedUsage(fs *flag.FlagSet) func() {
+	categories := make(map[string][]string)
+	return func() {
+		if fs.Name() == "" {
+			fmt.Fprintf(fs.Output(), "Usage:\n")
+		} else {
+			fmt.Fprintf(fs.Output(), "Usage of %s:\n", fs.Name())
+		}
+
+		fs.VisitAll(func(f *flag.Flag) {
+			id := ""
+			if strings.Contains(f.Name, ".") {
+				id = strings.Split(f.Name, ".")[0]
+			}
+
+			kind, usage := flag.UnquoteUsage(f)
+			if kind != "" {
+				kind = " " + kind
+			}
+			def := f.DefValue
+			if def != "" {
+				def = fmt.Sprintf(" (default %s)", def)
+			}
+			categories[id] = append(categories[id], fmt.Sprintf("   -%s%s:\n      %s%s", f.Name, kind, usage, def))
+		})
+
+		for name, flags := range categories {
+			if len(flags) == 1 {
+				categories[""] = append(categories[""], flags[0])
+				delete(categories, name)
+			}
+		}
+
+		for name := range categories {
+			sort.Strings(categories[name])
+		}
+
+		for _, u := range categories[""] {
+			fmt.Fprintln(fs.Output(), u)
+		}
+		fmt.Fprintln(fs.Output())
+
+		keys := make([]string, 0, len(categories))
+		for k := range categories {
+			keys = append(keys, k)
+		}
+		sort.Strings(keys)
+
+		for _, name := range keys {
+			if name == "" {
+				continue
+			}
+			fmt.Fprintf(fs.Output(), " %s:\n", strings.Title(name))
+			for _, u := range categories[name] {
+				fmt.Fprintln(fs.Output(), u)
+			}
+			fmt.Fprintln(fs.Output())
+		}
+	}
+}

cmd/frigg/cfg/flag_test.go

@@ -0,0 +1,58 @@
+package cfg
+
+import (
+	"flag"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestDefaults checks that defaults are correctly obtained from a
+// flagext.Registerer
+func TestDefaults(t *testing.T) {
+	data := Data{}
+	fs := flag.NewFlagSet(t.Name(), flag.PanicOnError)
+
+	err := Unmarshal(&data,
+		dDefaults(fs),
+	)
+
+	require.NoError(t, err)
+	assert.Equal(t, Data{
+		Verbose: false,
+		Server: Server{
+			Port:    80,
+			Timeout: 60 * time.Second,
+		},
+		TLS: TLS{
+			Cert: "DEFAULTCERT",
+			Key:  "DEFAULTKEY",
+		},
+	}, data)
+}
+
+// TestFlags checks that defaults and flag values (they can't be separated) are
+// correctly obtained from the command line
+func TestFlags(t *testing.T) {
+	data := Data{}
+	fs := flag.NewFlagSet(t.Name(), flag.PanicOnError)
+	err := Unmarshal(&data,
+		dDefaults(fs),
+		dFlags(fs, []string{"-server.timeout=10h", "-verbose"}),
+	)
+	require.NoError(t, err)
+
+	assert.Equal(t, Data{
+		Verbose: true,
+		Server: Server{
+			Port:    80,
+			Timeout: 10 * time.Hour,
+		},
+		TLS: TLS{
+			Cert: "DEFAULTCERT",
+			Key:  "DEFAULTKEY",
+		},
+	}, data)
+}

cmd/frigg/cfg/precedence_test.go

@@ -0,0 +1,70 @@
+package cfg
+
+import (
+	"flag"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// This file checks precedence rules are correctly working
+// The default precedence recommended by this package is the following:
+// flag defaults < yaml < user-set flags
+//
+// The following tests make sure that this is indeed correct
+
+const y = `
+verbose: true
+tls:
+  cert: YAML
+server:
+  port: 1234
+`
+
+func TestYAMLOverDefaults(t *testing.T) {
+	data := Data{}
+	fs := flag.NewFlagSet(t.Name(), flag.PanicOnError)
+	err := Unmarshal(&data,
+		dDefaults(fs),
+		dYAML([]byte(y)),
+	)
+
+	require.NoError(t, err)
+	assert.Equal(t, Data{
+		Verbose: true, // yaml
+		Server: Server{
+			Port:    1234,             // yaml
+			Timeout: 60 * time.Second, // default
+		},
+		TLS: TLS{
+			Cert: "YAML",       // yaml
+			Key:  "DEFAULTKEY", // default
+		},
+	}, data)
+}
+
+func TestFlagOverYAML(t *testing.T) {
+	data := Data{}
+	fs := flag.NewFlagSet(t.Name(), flag.PanicOnError)
+
+	err := Unmarshal(&data,
+		dDefaults(fs),
+		dYAML([]byte(y)),
+		dFlags(fs, []string{"-verbose=false", "-tls.cert=CLI"}),
+	)
+
+	require.NoError(t, err)
+	assert.Equal(t, Data{
+		Verbose: false, // flag
+		Server: Server{
+			Port:    1234,             // yaml
+			Timeout: 60 * time.Second, // default
+		},
+		TLS: TLS{
+			Cert: "CLI",        // flag
+			Key:  "DEFAULTKEY", // default
+		},
+	}, data)
+}