From 9d97b3e7324e0f955002c53b24deeb9cdf768be5 Mon Sep 17 00:00:00 2001 From: Andrey Ivanov Date: Fri, 21 Aug 2020 12:05:06 +0300 Subject: [PATCH 1/6] HW10 is completed --- hw10_program_optimization/go.mod | 2 +- hw10_program_optimization/stats.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hw10_program_optimization/go.mod b/hw10_program_optimization/go.mod index 3de4680..d0736cd 100644 --- a/hw10_program_optimization/go.mod +++ b/hw10_program_optimization/go.mod @@ -1,4 +1,4 @@ -module github.com/fixme_my_friend/hw10_program_optimization +module github.com/tiburon-777/HW_OTUS/hw10_program_optimization go 1.14 diff --git a/hw10_program_optimization/stats.go b/hw10_program_optimization/stats.go index 32a2185..b871102 100644 --- a/hw10_program_optimization/stats.go +++ b/hw10_program_optimization/stats.go @@ -29,7 +29,7 @@ func GetDomainStat(r io.Reader, domain string) (DomainStat, error) { return countDomains(u, domain) } -type users [100_000]User +type users [100000]User func getUsers(r io.Reader) (result users, err error) { content, err := ioutil.ReadAll(r) From 956abaaeca60ba3f15ba4a8aa7e7b343d1fb42d0 Mon Sep 17 00:00:00 2001 From: Andrey Ivanov Date: Fri, 21 Aug 2020 14:31:03 +0300 Subject: [PATCH 2/6] HW10 is completed --- hw10_program_optimization/README.md | 7 +++++++ hw10_program_optimization/stats.go | 32 ++++++++++++----------------- 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/hw10_program_optimization/README.md b/hw10_program_optimization/README.md index 939cbb4..1bd1c42 100644 --- a/hw10_program_optimization/README.md +++ b/hw10_program_optimization/README.md @@ -30,6 +30,13 @@ GetDomainStat(r, "edu") // {"quinu": 1} - удалять имеющийся лишний код (кроме функции `GetDomainStat`); - добавлять юнит-тесты. +**Обратите внимание на запуск TestGetDomainStat_Time_And_Memory** +```bash +go test -v -count=1 -timeout=30s -tags bench . +``` + +Здесь используется билд-тэг bench, чтобы отделить обычные тесты от тестов производительности. + ### Критерии оценки - Пайплайн зелёный и нет попытки «обмануть» систему - 4 балла - Добавлены юнит-тесты - до 3 баллов diff --git a/hw10_program_optimization/stats.go b/hw10_program_optimization/stats.go index b871102..5cee6f1 100644 --- a/hw10_program_optimization/stats.go +++ b/hw10_program_optimization/stats.go @@ -5,7 +5,6 @@ import ( "fmt" "io" "io/ioutil" - "regexp" "strings" ) @@ -29,38 +28,33 @@ func GetDomainStat(r io.Reader, domain string) (DomainStat, error) { return countDomains(u, domain) } -type users [100000]User +type Users []User -func getUsers(r io.Reader) (result users, err error) { +func getUsers(r io.Reader) (Users, error) { + result := make([]User, 10000) content, err := ioutil.ReadAll(r) if err != nil { - return + return result, err } lines := strings.Split(string(content), "\n") - for i, line := range lines { + for _, line := range lines { var user User if err = json.Unmarshal([]byte(line), &user); err != nil { - return + return result, err } - result[i] = user + result = append(result,user) } - return + return result, err } -func countDomains(u users, domain string) (DomainStat, error) { +func countDomains(u Users, domain string) (DomainStat, error) { result := make(DomainStat) - + i:=0 for _, user := range u { - matched, err := regexp.Match("\\."+domain, []byte(user.Email)) - if err != nil { - return nil, err - } - - if matched { - num := result[strings.ToLower(strings.SplitN(user.Email, "@", 2)[1])] - num++ - result[strings.ToLower(strings.SplitN(user.Email, "@", 2)[1])] = num + if strings.Contains(user.Email,"."+domain) { + result[strings.ToLower(strings.SplitN(user.Email, "@", 2)[1])]++ + i++ } } return result, nil From 3faacb8ba66f32e48751481a27eced64ab5ac5b6 Mon Sep 17 00:00:00 2001 From: Andrey Ivanov Date: Sun, 23 Aug 2020 08:40:57 +0300 Subject: [PATCH 3/6] =?UTF-8?q?HW10=20is=20completed=20(=D1=82=D0=B5=D1=81?= =?UTF-8?q?=D1=82=20=D0=BD=D0=B0=20CI)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- hw10_program_optimization/go.mod | 5 ++- hw10_program_optimization/go.sum | 10 ++++++ hw10_program_optimization/stats.go | 56 ++++++++++-------------------- 3 files changed, 33 insertions(+), 38 deletions(-) diff --git a/hw10_program_optimization/go.mod b/hw10_program_optimization/go.mod index d0736cd..81dbf2a 100644 --- a/hw10_program_optimization/go.mod +++ b/hw10_program_optimization/go.mod @@ -2,4 +2,7 @@ module github.com/tiburon-777/HW_OTUS/hw10_program_optimization go 1.14 -require github.com/stretchr/testify v1.5.1 +require ( + github.com/json-iterator/go v1.1.10 + github.com/stretchr/testify v1.5.1 +) diff --git a/hw10_program_optimization/go.sum b/hw10_program_optimization/go.sum index 331fa69..c1291ea 100644 --- a/hw10_program_optimization/go.sum +++ b/hw10_program_optimization/go.sum @@ -1,8 +1,18 @@ github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= diff --git a/hw10_program_optimization/stats.go b/hw10_program_optimization/stats.go index 5cee6f1..f4111fc 100644 --- a/hw10_program_optimization/stats.go +++ b/hw10_program_optimization/stats.go @@ -1,11 +1,11 @@ package hw10_program_optimization //nolint:golint,stylecheck import ( - "encoding/json" - "fmt" + "bufio" "io" - "io/ioutil" "strings" + + jsoniter "github.com/json-iterator/go" ) type User struct { @@ -20,42 +20,24 @@ type User struct { type DomainStat map[string]int +var json = jsoniter.ConfigCompatibleWithStandardLibrary + func GetDomainStat(r io.Reader, domain string) (DomainStat, error) { - u, err := getUsers(r) - if err != nil { - return nil, fmt.Errorf("get users error: %s", err) - } - return countDomains(u, domain) -} - -type Users []User - -func getUsers(r io.Reader) (Users, error) { - result := make([]User, 10000) - content, err := ioutil.ReadAll(r) - if err != nil { - return result, err - } - - lines := strings.Split(string(content), "\n") - for _, line := range lines { - var user User - if err = json.Unmarshal([]byte(line), &user); err != nil { - return result, err + result := make(DomainStat) + var err error + var user User + e := bufio.NewReader(r) + for { + line, _, err := e.ReadLine() + if err == io.EOF { + break + } + if err = json.Unmarshal(line, &user); err != nil { + return nil, err + } + if strings.Contains(user.Email, "."+domain) { + result[strings.ToLower(strings.SplitN(user.Email, "@", 2)[1])]++ } - result = append(result,user) } return result, err } - -func countDomains(u Users, domain string) (DomainStat, error) { - result := make(DomainStat) - i:=0 - for _, user := range u { - if strings.Contains(user.Email,"."+domain) { - result[strings.ToLower(strings.SplitN(user.Email, "@", 2)[1])]++ - i++ - } - } - return result, nil -} From a33f3a5924544ea1c1fd3f3f3da2bc6354e4f929 Mon Sep 17 00:00:00 2001 From: Andrey Ivanov Date: Sun, 23 Aug 2020 09:02:39 +0300 Subject: [PATCH 4/6] HW10 is completed --- hw10_program_optimization/stats.go | 2 +- hw10_program_optimization/stats_test.go | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/hw10_program_optimization/stats.go b/hw10_program_optimization/stats.go index f4111fc..2fd55d3 100644 --- a/hw10_program_optimization/stats.go +++ b/hw10_program_optimization/stats.go @@ -33,7 +33,7 @@ func GetDomainStat(r io.Reader, domain string) (DomainStat, error) { break } if err = json.Unmarshal(line, &user); err != nil { - return nil, err + return DomainStat{}, err } if strings.Contains(user.Email, "."+domain) { result[strings.ToLower(strings.SplitN(user.Email, "@", 2)[1])]++ diff --git a/hw10_program_optimization/stats_test.go b/hw10_program_optimization/stats_test.go index 48d4df8..ec7ecf7 100644 --- a/hw10_program_optimization/stats_test.go +++ b/hw10_program_optimization/stats_test.go @@ -16,6 +16,9 @@ func TestGetDomainStat(t *testing.T) { {"Id":4,"Name":"Gregory Reid","Username":"tButler","Email":"5Moore@Teklist.net","Phone":"520-04-16","Password":"r639qLNu","Address":"Sunfield Park 20"} {"Id":5,"Name":"Janice Rose","Username":"KeithHart","Email":"nulla@Linktype.com","Phone":"146-91-01","Password":"acSBF5","Address":"Russell Trail 61"}` + dataNeg := `{"Id":1,"Name":"Howard Mendoza","Username":"0Oliver","Email":"aliquid_qui_ea@Brow@sedrive.gov","Phone":"6-866-899-36-79","Password":"InAQJvsq","Address":"Blackbird Place 25" +{"Id":2,"Name":"Jesse Vasquez","Username":"qRichardson","Email":"mLynchbroWsecat.com","Phone":"9-373-949-64-00","Password":"SiZLeNSGn","Address":"Fulton Hill 80"}` + t.Run("find 'com'", func(t *testing.T) { result, err := GetDomainStat(bytes.NewBufferString(data), "com") require.NoError(t, err) @@ -36,4 +39,10 @@ func TestGetDomainStat(t *testing.T) { require.NoError(t, err) require.Equal(t, DomainStat{}, result) }) + + t.Run("Error test", func(t *testing.T) { + result, err := GetDomainStat(bytes.NewBufferString(dataNeg), "unknown") + require.Error(t, err) + require.Equal(t, DomainStat{}, result) + }) } From 7a8768923cf885920ebef08d1ea1dbd6945e7d41 Mon Sep 17 00:00:00 2001 From: Andrey Ivanov Date: Mon, 24 Aug 2020 10:45:48 +0300 Subject: [PATCH 5/6] HW10 is completed --- hw10_program_optimization/stats.go | 3 ++- hw10_program_optimization/stats_test.go | 10 ++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/hw10_program_optimization/stats.go b/hw10_program_optimization/stats.go index 2fd55d3..20741c7 100644 --- a/hw10_program_optimization/stats.go +++ b/hw10_program_optimization/stats.go @@ -35,7 +35,8 @@ func GetDomainStat(r io.Reader, domain string) (DomainStat, error) { if err = json.Unmarshal(line, &user); err != nil { return DomainStat{}, err } - if strings.Contains(user.Email, "."+domain) { + + if strings.HasSuffix(user.Email, "."+domain) { result[strings.ToLower(strings.SplitN(user.Email, "@", 2)[1])]++ } } diff --git a/hw10_program_optimization/stats_test.go b/hw10_program_optimization/stats_test.go index ec7ecf7..a358b65 100644 --- a/hw10_program_optimization/stats_test.go +++ b/hw10_program_optimization/stats_test.go @@ -19,6 +19,10 @@ func TestGetDomainStat(t *testing.T) { dataNeg := `{"Id":1,"Name":"Howard Mendoza","Username":"0Oliver","Email":"aliquid_qui_ea@Brow@sedrive.gov","Phone":"6-866-899-36-79","Password":"InAQJvsq","Address":"Blackbird Place 25" {"Id":2,"Name":"Jesse Vasquez","Username":"qRichardson","Email":"mLynchbroWsecat.com","Phone":"9-373-949-64-00","Password":"SiZLeNSGn","Address":"Fulton Hill 80"}` + dataCase1 := `{"Id":1,"Name":"Howard Mendoza","Username":"0Oliver","Email":"aliquid_qui_ea@Browsedrive.run","Phone":"6-866-899-36-79","Password":"InAQJvsq","Address":"Blackbird Place 25"} +{"Id":2,"Name":"Jesse Vasquez","Username":"qRichardson","Email":"mLynch@broWsecat.run","Phone":"9-373-949-64-00","Password":"SiZLeNSGn","Address":"Fulton Hill 80"} +{"Id":4,"Name":"Gregory Reid","Username":"tButler","Email":"5Moore@Teklist.ru","Phone":"520-04-16","Password":"r639qLNu","Address":"Sunfield Park 20"}` + t.Run("find 'com'", func(t *testing.T) { result, err := GetDomainStat(bytes.NewBufferString(data), "com") require.NoError(t, err) @@ -45,4 +49,10 @@ func TestGetDomainStat(t *testing.T) { require.Error(t, err) require.Equal(t, DomainStat{}, result) }) + + t.Run("Case1 test", func(t *testing.T) { + result, err := GetDomainStat(bytes.NewBufferString(dataCase1), "ru") + require.NoError(t, err) + require.Equal(t, DomainStat{"teklist.ru":1}, result) + }) } From 6f0ccb593d2cd03c98694318f6b0867c4879b927 Mon Sep 17 00:00:00 2001 From: Andrey Ivanov Date: Mon, 24 Aug 2020 10:50:32 +0300 Subject: [PATCH 6/6] HW10 is completed (go fmt) --- hw10_program_optimization/stats.go | 2 +- hw10_program_optimization/stats_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hw10_program_optimization/stats.go b/hw10_program_optimization/stats.go index 20741c7..614c28a 100644 --- a/hw10_program_optimization/stats.go +++ b/hw10_program_optimization/stats.go @@ -36,7 +36,7 @@ func GetDomainStat(r io.Reader, domain string) (DomainStat, error) { return DomainStat{}, err } - if strings.HasSuffix(user.Email, "."+domain) { + if strings.HasSuffix(user.Email, "."+domain) { result[strings.ToLower(strings.SplitN(user.Email, "@", 2)[1])]++ } } diff --git a/hw10_program_optimization/stats_test.go b/hw10_program_optimization/stats_test.go index a358b65..8fd1b40 100644 --- a/hw10_program_optimization/stats_test.go +++ b/hw10_program_optimization/stats_test.go @@ -53,6 +53,6 @@ func TestGetDomainStat(t *testing.T) { t.Run("Case1 test", func(t *testing.T) { result, err := GetDomainStat(bytes.NewBufferString(dataCase1), "ru") require.NoError(t, err) - require.Equal(t, DomainStat{"teklist.ru":1}, result) + require.Equal(t, DomainStat{"teklist.ru": 1}, result) }) }