From aca8249875a4f2617f8cd4e9716434d7c1bd7934 Mon Sep 17 00:00:00 2001 From: Andrey Ivanov Date: Mon, 7 Dec 2020 20:20:23 +0300 Subject: [PATCH] Initial commit --- .gitignore | 15 +++++++ .golangci.yml | 21 +++++++++ .travis.yml | 38 ++++++++++++++++ Makefile | 13 ++++++ README.md | 4 +- go.mod | 11 +++++ go.sum | 38 ++++++++++++++++ pkg/config/README.md | 11 +++++ pkg/config/config.go | 92 +++++++++++++++++++++++++++++++++++++++ pkg/config/config_test.go | 86 ++++++++++++++++++++++++++++++++++++ pkg/config/types.go | 50 +++++++++++++++++++++ pkg/logger/logger.go | 74 +++++++++++++++++++++++++++++++ pkg/logger/logger_test.go | 50 +++++++++++++++++++++ 13 files changed, 502 insertions(+), 1 deletion(-) create mode 100644 .gitignore create mode 100644 .golangci.yml create mode 100644 .travis.yml create mode 100644 Makefile create mode 100644 go.mod create mode 100644 go.sum create mode 100644 pkg/config/README.md create mode 100644 pkg/config/config.go create mode 100644 pkg/config/config_test.go create mode 100644 pkg/config/types.go create mode 100644 pkg/logger/logger.go create mode 100644 pkg/logger/logger_test.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..52ba918 --- /dev/null +++ b/.gitignore @@ -0,0 +1,15 @@ +# Binaries for programs and plugins +*.dll +*.so +*.dylib + +# Test binary, built with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Dependency directories (remove the comment below to include it) +# vendor/ + +.idea diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..37cc5f7 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,21 @@ +run: + tests: false + +linters: + disable-all: false + enable-all: true + disable: + - gochecknoglobals + - gochecknoinits + - godox + - goerr113 + - gomnd + - lll + - nakedret + - testpackage + - wsl + - nlreturn + - whitespace + - bodyclose + - wrapcheck + - exhaustivestruct \ No newline at end of file diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..06b5098 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,38 @@ +language: go + +go: + - "1.14" + +os: + - linux + +git: + depth: 1 + quiet: true + submodules: false + +notifications: + email: true + +env: + global: + - GO111MODULE=on + - GOPROXY=https://proxy.golang.org + - BRANCH="${TRAVIS_PULL_REQUEST_BRANCH:-$TRAVIS_BRANCH}" + +stages: + - name: Tests + +jobs: + include: + - stage: "Tests" + name: "Makefile" + install: go mod download + script: + - make lint + - make fast-test + - make slow-test + if: (type = push) AND (type = pull_request) +branches: + only: + - master diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..0d06a4d --- /dev/null +++ b/Makefile @@ -0,0 +1,13 @@ +lint: install-lint-deps + golangci-lint run ./previewer/... ./internal/... + +fast-test: + go test -race -count 100 -timeout 30s -short ./internal/... + +slow-test: + go test -race -timeout 150s -run Slow ./internal/... + +install-lint-deps: + rm -rf $(shell go env GOPATH)/bin/golangci-lint + (which golangci-lint > /dev/null) || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin +.PHONY: fast-test slow-test lint \ No newline at end of file diff --git a/README.md b/README.md index 0538196..9d8a1e0 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,4 @@ # modules -Usable go modules +Usable go modules for Go apps + +* [/config - universal configuration module](./pkg/config/README.md) diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..5691c80 --- /dev/null +++ b/go.mod @@ -0,0 +1,11 @@ +module modules + +go 1.14 + +require ( + github.com/BurntSushi/toml v0.3.1 + github.com/amitrai48/logger v0.0.0-20190214092904-448001c055ec + github.com/pkg/errors v0.9.1 // indirect + github.com/stretchr/testify v1.6.1 + gopkg.in/yaml.v2 v2.4.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..7b69029 --- /dev/null +++ b/go.sum @@ -0,0 +1,38 @@ +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/amitrai48/logger v0.0.0-20190214092904-448001c055ec h1:tDOPo9NAXCjvoK35HgZyzQSNLmb3chZqN2tnO273Bro= +github.com/amitrai48/logger v0.0.0-20190214092904-448001c055ec/go.mod h1:RZEHP3cxXvQlMuMjkpdh6qXA4b0CpjxnUBNxOpR0r30= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/sirupsen/logrus v1.3.0 h1:hI/7Q+DtNZ2kINb6qt/lS+IyXnHQe9e90POfeewL/ME= +github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +go.uber.org/atomic v1.3.2 h1:2Oa65PReHzfn29GpvgsYwloV9AVFHPDk8tYxt2c2tr4= +go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/zap v1.9.1 h1:XCJQEf3W6eZaVwhRBof6ImoYGJSITeKWsyeh3HFu/5o= +go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793 h1:u+LnwYTOOW7Ukr/fppxEb1Nwz0AtPflrblfvUudpo+I= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33 h1:I6FyU15t786LL7oL/hn43zqTuEGr4PN7F4XJ1p4E3Y8= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= +gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/config/README.md b/pkg/config/README.md new file mode 100644 index 0000000..eafcc5c --- /dev/null +++ b/pkg/config/README.md @@ -0,0 +1,11 @@ +# /config + +Модуль для настройки конфигурации приложения. +Возможности: +* Чтение конфига из файла в формате TOML +* Чтение из переменных окружения +* Чтение из базы данных (параметры DSN дложны быть переданы через предыдущие два пункта) +* Мердж полученных данных с приоритетом последнего источника + + +[<- BACK to ROOT](../../README.md) \ No newline at end of file diff --git a/pkg/config/config.go b/pkg/config/config.go new file mode 100644 index 0000000..4fd8673 --- /dev/null +++ b/pkg/config/config.go @@ -0,0 +1,92 @@ +package config + +import ( + "fmt" + "io/ioutil" + "os" + "reflect" + "strconv" + "strings" + + "github.com/BurntSushi/toml" +) + +type Config struct { + ConfigFile string + EnvPrefix string + DSN string +} +/* +Логика конфига: + - В конструктор отдаем структуру - она заполняется исходя из логики и модификаторов конструктора + - Данные подтягиваются из файла + - Данные подтягиваются из переменных окружения + - Данные тянутся из базы +*/ + +func New(configFile string, str interface{}) error { + if configFile != "" { + f, err := os.Open(configFile) + if err != nil { + return fmt.Errorf("can't open config file: %w", err) + } + defer f.Close() + s, err := ioutil.ReadAll(f) + if err != nil { + return fmt.Errorf("can't read content of the config file : %w", err) + } + _, err = toml.Decode(string(s), str) + if err != nil { + return fmt.Errorf("can't parce config file : %w", err) + } + } + err := ApplyEnvVars(str, "APP") + if err != nil { + return fmt.Errorf("can't apply envvars to config :%w", err) + } + return nil + return nil +} + +func ApplyEnvVars(c interface{}, prefix string) error { + return applyEnvVar(reflect.ValueOf(c), reflect.TypeOf(c), -1, prefix) +} + +func applyEnvVar(v reflect.Value, t reflect.Type, counter int, prefix string) error { + if v.Kind() != reflect.Ptr { + return fmt.Errorf("not a pointer value") + } + f := reflect.StructField{} + if counter != -1 { + f = t.Field(counter) + } + v = reflect.Indirect(v) + fName := strings.ToUpper(f.Name) + env := os.Getenv(prefix + fName) + if env != "" { + switch v.Kind() { + case reflect.Int: + envI, err := strconv.Atoi(env) + if err != nil { + return fmt.Errorf("could not parse to int: %w", err) + } + v.SetInt(int64(envI)) + case reflect.String: + v.SetString(env) + case reflect.Bool: + envB, err := strconv.ParseBool(env) + if err != nil { + return fmt.Errorf("could not parse bool: %w", err) + } + v.SetBool(envB) + } + } + if v.Kind() == reflect.Struct { + for i := 0; i < v.NumField(); i++ { + if err := applyEnvVar(v.Field(i).Addr(), v.Type(), i, prefix+fName+"_"); err != nil { + return fmt.Errorf("could not apply env var: %w", err) + } + } + } + return nil +} diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go new file mode 100644 index 0000000..a6bd1b6 --- /dev/null +++ b/pkg/config/config_test.go @@ -0,0 +1,86 @@ +package config + +import ( + "github.com/stretchr/testify/require" + "io/ioutil" + "log" + "os" + "testing" +) + + +func TestNewConfig(t *testing.T) { + + badfile, err := ioutil.TempFile("", "conf.") + if err != nil { + log.Fatal(err) + } + defer os.Remove(badfile.Name()) + badfile.WriteString(`aefSD +sadfg +RFABND FYGUMG +V`) + badfile.Sync() + + goodfile, err := ioutil.TempFile("", "conf.") + if err != nil { + log.Fatal(err) + } + defer os.Remove(goodfile.Name()) + goodfile.WriteString(`[storage] +inMemory = true +SQLHost = "localhost"`) + goodfile.Sync() + + t.Run("No such file", func(t *testing.T) { + var c Calendar + e := New("adfergdth", &c) + require.Equal(t, Calendar{}, c) + require.Error(t, e) + }) + + t.Run("Bad file", func(t *testing.T) { + var c Calendar + e := New(badfile.Name(), &c) + require.Equal(t, Calendar{}, c) + require.Error(t, e) + }) + + t.Run("TOML reading", func(t *testing.T) { + var c Calendar + e := New(goodfile.Name(), &c) + require.Equal(t, true, c.Storage.InMemory) + require.Equal(t, "localhost", c.Storage.SQLHost) + require.NoError(t, e) + }) + + t.Run("ENV reading", func(t *testing.T) { + for k, v := range map[string]string{"APP_STRUCT1_VAR1": "val1", "APP_STRUCT1_VAR2": "val2", "APP_STRUCT2_VAR1": "val3", "APP_STRUCT2_VAR2": "val4", "APP_STRUCT3_VAR1": "val5", "APP_STRUCT3_VAR2": "val6"} { + require.NoError(t, os.Setenv(k, v)) + } + var str struct { + Struct1 struct { + Var1 string + Var2 string + } + Struct2 struct { + Var1 string + Var2 string + } + Struct3 struct { + Var1 string + Var2 string + } + } + + err := New("", &str) + require.NoError(t, err) + require.Equal(t, "val1", str.Struct1.Var1) + require.Equal(t, "val2", str.Struct1.Var2) + require.Equal(t, "val3", str.Struct2.Var1) + require.Equal(t, "val4", str.Struct2.Var2) + require.Equal(t, "val5", str.Struct3.Var1) + require.Equal(t, "val6", str.Struct3.Var2) + }) + +} diff --git a/pkg/config/types.go b/pkg/config/types.go new file mode 100644 index 0000000..b180406 --- /dev/null +++ b/pkg/config/types.go @@ -0,0 +1,50 @@ +package config + +type Calendar struct { + GRPC Server + HTTP Server + API Server + Logger Logger + Storage Storage +} + +type Scheduler struct { + Rabbitmq Rabbit + Storage Storage + Logger Logger +} + +type Sender struct { + Rabbitmq Rabbit + Logger Logger +} + +type Server struct { + Address string + Port string +} + +type Rabbit struct { + Login string + Pass string + Address string + Port string + Exchange string + Queue string + Key string +} + +type Logger struct { + File string + Level string + MuteStdout bool +} + +type Storage struct { + InMemory bool + SQLHost string + SQLPort string + SQLDbase string + SQLUser string + SQLPass string +} diff --git a/pkg/logger/logger.go b/pkg/logger/logger.go new file mode 100644 index 0000000..6aee48c --- /dev/null +++ b/pkg/logger/logger.go @@ -0,0 +1,74 @@ +package logger + +import ( + "errors" + "log" + "os" + "strings" + + amitralog "github.com/amitrai48/logger" +) + +type LoggerInterface interface { + Debugf(format string, args ...interface{}) + Infof(format string, args ...interface{}) + Warnf(format string, args ...interface{}) + Errorf(format string, args ...interface{}) + Fatalf(format string, args ...interface{}) +} + +type Logger struct { + amitralog.Logger +} + +type Config struct { + File string + Level string + MuteStdout bool +} + +var validLevel = map[string]bool{"debug": true, "info": true, "warn": true, "error": true, "fatal": true} + +func New(conf Config) (LoggerInterface, error) { + if conf.File == "" || !validLevel[strings.ToLower(conf.Level)] { + return nil, errors.New("invalid logger config") + } + + c := amitralog.Configuration{ + EnableConsole: !conf.MuteStdout, + ConsoleLevel: amitralog.Fatal, + ConsoleJSONFormat: false, + EnableFile: true, + FileLevel: strings.ToLower(conf.Level), + FileJSONFormat: true, + FileLocation: conf.File, + } + + if err := amitralog.NewLogger(c, amitralog.InstanceZapLogger); err != nil { + log.Fatalf("Could not instantiate log %s", err.Error()) + } + l := amitralog.WithFields(amitralog.Fields{"hw": "15"}) + l.Infof("logger start successful") + return l, nil +} + +func (l *Logger) Debugf(format string, args ...interface{}) { + l.Logger.Debugf(format, args) +} + +func (l *Logger) Infof(format string, args ...interface{}) { + l.Logger.Infof(format, args) +} + +func (l *Logger) Warnf(format string, args ...interface{}) { + l.Logger.Warnf(format, args) +} + +func (l *Logger) Errorf(format string, args ...interface{}) { + l.Logger.Errorf(format, args) +} + +func (l *Logger) Fatalf(format string, args ...interface{}) { + l.Logger.Fatalf(format, args) + os.Exit(2) +} diff --git a/pkg/logger/logger_test.go b/pkg/logger/logger_test.go new file mode 100644 index 0000000..edc4b28 --- /dev/null +++ b/pkg/logger/logger_test.go @@ -0,0 +1,50 @@ +package logger + +import ( + "github.com/stretchr/testify/require" + "io/ioutil" + oslog "log" + "os" + "strings" + "testing" +) + +func TestLoggerLogic(t *testing.T) { + tmpfile, err := ioutil.TempFile("", "log.") + if err != nil { + oslog.Fatal(err) + } + defer os.Remove(tmpfile.Name()) + + conf := Config{File: tmpfile.Name(), Level: "warn", MuteStdout: false} + log, err := New(conf) + if err != nil { + oslog.Fatal(err) + } + + t.Run("Messages arround the level", func(t *testing.T) { + log.Debugf("debug message") + log.Errorf("error message") + + res, err := ioutil.ReadAll(tmpfile) + if err != nil { + oslog.Fatal(err) + } + require.Less(t, strings.Index(string(res), "debug message"), 0) + require.Greater(t, strings.Index(string(res), "error message"), 0) + }) +} + +func TestLoggerNegative(t *testing.T) { + t.Run("Bad file name", func(t *testing.T) { + conf := Config{File: "", Level: "debug", MuteStdout: true} + _, err := New(conf) + require.Error(t, err, "invalid logger config") + }) + + t.Run("Bad level", func(t *testing.T) { + conf := Config{File: "asdafad", Level: "wegretryjt", MuteStdout: true} + _, err := New(conf) + require.Error(t, err, "invalid logger config") + }) +}