From 7bd72aa110aa851277fd29583856cf88ff0565c2 Mon Sep 17 00:00:00 2001 From: Andrey Ivanov Date: Fri, 6 Nov 2020 16:48:25 +0300 Subject: [PATCH] =?UTF-8?q?=D0=9A=D1=8D=D1=88=20=D1=85=D1=80=D0=B0=D0=BD?= =?UTF-8?q?=D0=B8=D1=82=20=D0=BA=D0=B0=D1=80=D1=82=D0=B8=D0=BD=D0=BA=D0=B8?= =?UTF-8?q?=20=D0=B2=20=D1=84=D0=B0=D0=B9=D0=BB=D0=BE=D0=B2=D0=BE=D0=B9=20?= =?UTF-8?q?=D1=81=D0=B8=D1=81=D1=82=D0=B5=D0=BC=D0=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 4 +- LICENSE | 8 ++ README.md | 2 + assets/cache/nofile | 0 cmd/main.go | 4 + cmd/main_test.go | 4 +- internal/application/application.go | 2 +- internal/application/query.go | 2 +- internal/cache/cache.go | 93 +++++++++++++++++- internal/cache/cache_test.go | 58 +++++++---- internal/config/config.go | 8 +- internal/logger/logger_test.go | 2 +- previewer.conf | 1 + {assets => test/data}/gopher_1024x252.jpg | Bin {assets => test/data}/gopher_2000x1000.jpg | Bin {assets => test/data}/gopher_200x700.jpg | Bin {assets => test/data}/gopher_256x126.jpg | Bin {assets => test/data}/gopher_333x666.jpg | Bin {assets => test/data}/gopher_500x500.jpg | Bin {assets => test/data}/gopher_50x50.jpg | Bin .../data}/gopher_original_1024x504.jpg | Bin 21 files changed, 156 insertions(+), 32 deletions(-) create mode 100644 LICENSE create mode 100644 assets/cache/nofile rename {assets => test/data}/gopher_1024x252.jpg (100%) rename {assets => test/data}/gopher_2000x1000.jpg (100%) rename {assets => test/data}/gopher_200x700.jpg (100%) rename {assets => test/data}/gopher_256x126.jpg (100%) rename {assets => test/data}/gopher_333x666.jpg (100%) rename {assets => test/data}/gopher_500x500.jpg (100%) rename {assets => test/data}/gopher_50x50.jpg (100%) rename {assets => test/data}/gopher_original_1024x504.jpg (100%) diff --git a/.gitignore b/.gitignore index 1dc0f8a..345eacf 100644 --- a/.gitignore +++ b/.gitignore @@ -14,7 +14,7 @@ # Dependency directories (remove the comment below to include it) # vendor/ -examples .idea *.log -bin \ No newline at end of file +bin +assets/cache/*.* \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..1a30626 --- /dev/null +++ b/LICENSE @@ -0,0 +1,8 @@ +Copyright (c) 2020, Andrey "Tiburon" Ivanov All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. +Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. +Neither the names of contributors may be used to endorse or promote products derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/README.md b/README.md index 604bb86..66a27e4 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ +[![Build Status](https://travis-ci.com/tiburon-777/OTUS_Project.svg)](https://travis-ci.org/tiburon-777/OTUS_Project) +[![Go Report Card](https://goreportcard.com/badge/github.com/tiburon-777/OTUS_Project)](https://goreportcard.com/report/github.com/tiburon-777/OTUS_Project) # Проектная работа Image previewer service ## Общее описание diff --git a/assets/cache/nofile b/assets/cache/nofile new file mode 100644 index 0000000..e69de29 diff --git a/cmd/main.go b/cmd/main.go index 07ae475..99ec26a 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -21,6 +21,10 @@ func main() { } app := application.New(conf) + err = app.Cache.Clear() + if err != nil { + log.Fatalf("can't clean cache: %s", err.Error()) + } go func() { signals := make(chan os.Signal, 1) signal.Notify(signals) diff --git a/cmd/main_test.go b/cmd/main_test.go index d86de8e..02b9516 100644 --- a/cmd/main_test.go +++ b/cmd/main_test.go @@ -17,7 +17,7 @@ const testPortBase = 3000 func TestIntegrationPositive(t *testing.T) { testPort := strconv.Itoa(testPortBase + 1) wg := sync.WaitGroup{} - server := &http.Server{Addr: "localhost:" + testPort, Handler: http.FileServer(http.Dir("../assets"))} + server := &http.Server{Addr: "localhost:" + testPort, Handler: http.FileServer(http.Dir("../test/data"))} go func() { err := server.ListenAndServe() if err != nil { @@ -53,7 +53,7 @@ func TestIntegrationPositive(t *testing.T) { func TestIntegrationNegative(t *testing.T) { testPort := strconv.Itoa(testPortBase + 2) wg := sync.WaitGroup{} - server := &http.Server{Addr: "localhost:" + testPort, Handler: http.FileServer(http.Dir("../assets"))} + server := &http.Server{Addr: "localhost:" + testPort, Handler: http.FileServer(http.Dir("../test/data"))} go func() { err := server.ListenAndServe() if err != nil { diff --git a/internal/application/application.go b/internal/application/application.go index 82f1ad7..88627d9 100644 --- a/internal/application/application.go +++ b/internal/application/application.go @@ -22,7 +22,7 @@ func New(conf config.Config) *App { if err != nil { oslog.Fatal("не удалось прикрутить логгер: ", err.Error()) } - c := cache.NewCache(conf.Cache.Capacity) + c := cache.NewCache(conf.Cache.Capacity, conf.Cache.StoragePath) return &App{Server: &http.Server{Addr: net.JoinHostPort(conf.Server.Address, conf.Server.Port)}, Log: loger, Cache: c, Conf: conf} } diff --git a/internal/application/query.go b/internal/application/query.go index 9cd02a9..2b075b4 100644 --- a/internal/application/query.go +++ b/internal/application/query.go @@ -40,7 +40,7 @@ func buildQuery(u *url.URL) (q Query, err error) { } func (q Query) id() string { - return strconv.Itoa(q.Width) + "/" + strconv.Itoa(q.Height) + "/" + q.URL.Path + return strings.Replace(strconv.Itoa(q.Width)+"/"+strconv.Itoa(q.Height)+q.URL.Path, "/", "_", -1) } func (q Query) fromOrigin(headers http.Header, timeout time.Duration) ([]byte, *http.Response, error) { diff --git a/internal/cache/cache.go b/internal/cache/cache.go index 09d9ea1..3d840c6 100644 --- a/internal/cache/cache.go +++ b/internal/cache/cache.go @@ -2,6 +2,10 @@ package cache import ( "fmt" + "io/ioutil" + "log" + "os" + "path" "sync" ) @@ -10,11 +14,12 @@ type Key string type Cache interface { Set(key Key, value interface{}) (bool, error) // Добавить значение в кэш по ключу Get(key Key) (interface{}, bool, error) // Получить значение из кэша по ключу - Clear() // Очистить кэш + Clear() error // Очистить кэш } type lruCache struct { capacity int + path string queue *List items map[Key]*ListItem mx sync.RWMutex @@ -25,9 +30,13 @@ type Item struct { Value interface{} } -func NewCache(capacity int) Cache { +func NewCache(capacity int, path string) Cache { + if _, err := ioutil.ReadDir(path); err != nil { + log.Fatalf("cache directory %s not exists: %s", path, err.Error()) + } return &lruCache{ capacity: capacity, + path: path, queue: NewList(), items: make(map[Key]*ListItem), } @@ -37,6 +46,10 @@ func (l *lruCache) Set(key Key, value interface{}) (bool, error) { l.mx.Lock() defer l.mx.Unlock() if _, exists := l.items[key]; exists { + err := l.loadOut(key, value) + if err != nil { + return false, fmt.Errorf("can't replace file %s: %w", path.Join([]string{l.path, string(key)}...), err) + } l.items[key].Value = Item{Value: value, Key: key} l.queue.MoveToFront(l.items[key]) return exists, nil @@ -46,10 +59,21 @@ func (l *lruCache) Set(key Key, value interface{}) (bool, error) { if !ok { return false, fmt.Errorf("can't cast type") } + err := l.remove(k.Value.(Key)) + if err != nil { + return false, fmt.Errorf("can't delete file %s: %w", path.Join([]string{l.path, k.Value.(string)}...), err) + } delete(l.items, k.Key) l.queue.Remove(l.queue.Back()) } - l.items[key] = l.queue.PushFront(Item{Value: value, Key: key}) + err := l.loadOut(key, value) + if err != nil { + return false, fmt.Errorf("can't save file %s: %w", path.Join([]string{l.path, string(key)}...), err) + } + if l.items == nil { + l.items = make(map[Key]*ListItem) + } + l.items[key] = l.queue.PushFront(Item{Value: key, Key: key}) return false, nil } @@ -64,13 +88,72 @@ func (l *lruCache) Get(key Key) (interface{}, bool, error) { if !ok { return nil, false, fmt.Errorf("can't cast type") } - return s.Value, true, nil + pic, err := l.loadIn(s.Key) + if err != nil { + return nil, false, fmt.Errorf("can't load file %s: %w", path.Join([]string{l.path, string(s.Key)}...), err) + } + return pic, true, nil } -func (l *lruCache) Clear() { +func (l *lruCache) Clear() error { l.mx.Lock() defer l.mx.Unlock() + err := l.drop() + if err != nil { + return fmt.Errorf("can't remove files from %s: %w", l.path, err) + } l.items = nil l.queue.len = 0 l.queue.Info = ListItem{} + return nil +} + +func (l *lruCache) loadOut(name Key, pic interface{}) error { + filename := path.Join([]string{l.path, string(name)}...) + err := ioutil.WriteFile(filename, pic.([]byte), 0600) + if err != nil { + return fmt.Errorf("can't create or write file %s: %w", filename, err) + } + return nil +} + +func (l *lruCache) loadIn(name Key) ([]byte, error) { + filename := path.Join([]string{l.path, string(name)}...) + f, err := os.Open(filename) + defer func() { + _ = f.Close() + }() + if err != nil { + return nil, fmt.Errorf("can't open file %s: %w", filename, err) + } + res, err := ioutil.ReadAll(f) + if err != nil { + return nil, fmt.Errorf("can't read file %s: %w", filename, err) + } + return res, nil +} + +func (l *lruCache) remove(name Key) error { + filename := path.Join([]string{l.path, string(name)}...) + err := os.RemoveAll(filename) + if err != nil { + return fmt.Errorf("can't remove file %s: %w", filename, err) + } + return nil +} + +func (l *lruCache) drop() error { + dir, err := ioutil.ReadDir(l.path) + if err != nil { + return fmt.Errorf("can't read directory %s: %w", l.path, err) + } + for _, d := range dir { + if d.Name() != "nofile" { + err := os.Remove(path.Join([]string{l.path, d.Name()}...)) + if err != nil { + return fmt.Errorf("can't remove file %s/%s: %w", l.path, d.Name(), err) + } + } + } + return nil } diff --git a/internal/cache/cache_test.go b/internal/cache/cache_test.go index 1eb1011..5a17225 100644 --- a/internal/cache/cache_test.go +++ b/internal/cache/cache_test.go @@ -11,7 +11,9 @@ import ( func TestCache(t *testing.T) { t.Run("empty cache", func(t *testing.T) { - c := NewCache(10) + c := NewCache(10, "../../assets/cache") + err := c.Clear() + require.NoError(t, err, err) _, ok, err := c.Get("aaa") require.NoError(t, err) @@ -20,56 +22,66 @@ func TestCache(t *testing.T) { _, ok, err = c.Get("bbb") require.NoError(t, err) require.False(t, ok) + + err = c.Clear() + require.NoError(t, err, err) }) t.Run("simple", func(t *testing.T) { - c := NewCache(5) + c := NewCache(5, "../../assets/cache") + err := c.Clear() + require.NoError(t, err, err) - wasInCache, err := c.Set("aaa", 100) + wasInCache, err := c.Set("aaa", []byte("pic #1111")) require.NoError(t, err) require.False(t, wasInCache) - wasInCache, err = c.Set("bbb", 200) + wasInCache, err = c.Set("bbb", []byte("pic #2222")) require.NoError(t, err) require.False(t, wasInCache) val, ok, err := c.Get("aaa") require.NoError(t, err) require.True(t, ok) - require.Equal(t, 100, val) + require.Equal(t, []byte("pic #1111"), val) val, ok, err = c.Get("bbb") require.NoError(t, err) require.True(t, ok) - require.Equal(t, 200, val) + require.Equal(t, []byte("pic #2222"), val) - wasInCache, err = c.Set("aaa", 300) + wasInCache, err = c.Set("aaa", []byte("pic #3333")) require.NoError(t, err) require.True(t, wasInCache) val, ok, err = c.Get("aaa") require.NoError(t, err) require.True(t, ok) - require.Equal(t, 300, val) + require.Equal(t, []byte("pic #3333"), val) val, ok, err = c.Get("ccc") require.NoError(t, err) require.False(t, ok) require.Nil(t, val) + + err = c.Clear() + require.NoError(t, err, err) }) t.Run("purge logic", func(t *testing.T) { - c := NewCache(3) + c := NewCache(3, "../../assets/cache") + err := c.Clear() + require.NoError(t, err, err) - wasInCache, err := c.Set("aaa", 100) + wasInCache, err := c.Set("aaa", []byte("pic #1111")) require.NoError(t, err) require.False(t, wasInCache) - wasInCache, err = c.Set("bbb", 200) + wasInCache, err = c.Set("bbb", []byte("pic #2222")) require.NoError(t, err) require.False(t, wasInCache) - wasInCache, err = c.Set("ccc", 300) + wasInCache, err = c.Set("ccc", []byte("pic #3333")) require.NoError(t, err) require.False(t, wasInCache) @@ -81,7 +93,7 @@ func TestCache(t *testing.T) { require.NoError(t, err) require.True(t, ok) - wasInCache, err = c.Set("ddd", 400) + wasInCache, err = c.Set("ddd", []byte("pic #4444")) require.NoError(t, err) require.False(t, wasInCache) @@ -92,28 +104,38 @@ func TestCache(t *testing.T) { _, ok, err = c.Get("ccc") require.NoError(t, err) require.False(t, ok) + + err = c.Clear() + require.NoError(t, err, err) }) } func TestCacheMultithreading(t *testing.T) { + c := NewCache(10, "../../assets/cache") + err := c.Clear() + require.NoError(t, err, err) - c := NewCache(10) wg := &sync.WaitGroup{} wg.Add(2) go func() { defer wg.Done() - for i := 0; i < 1_000_000; i++ { - c.Set(Key(strconv.Itoa(i)), i) + for i := 0; i < 10_000; i++ { + itm := strconv.Itoa(i) + c.Set(Key(itm), []byte(itm)) } }() go func() { defer wg.Done() - for i := 0; i < 1_000_000; i++ { - c.Get(Key(strconv.Itoa(rand.Intn(1_000_000)))) + for i := 0; i < 10_000; i++ { + itm := strconv.Itoa(rand.Intn(10_000)) + c.Get(Key(itm)) } }() wg.Wait() + + err = c.Clear() + require.NoError(t, err, err) } diff --git a/internal/config/config.go b/internal/config/config.go index de5c2f0..b5791e1 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -13,7 +13,8 @@ type Config struct { Port string } Cache struct { - Capacity int + Capacity int + StoragePath string } Query struct { Timeout int @@ -45,7 +46,10 @@ func (c *Config) SetDefault() { Address string Port string }{Address: "localhost", Port: "8080"} - c.Cache = struct{ Capacity int }{Capacity: 20} + c.Cache = struct { + Capacity int + StoragePath string + }{Capacity: 20, StoragePath: "../assets/cache"} c.Query = struct{ Timeout int }{Timeout: 15} c.Log = struct { File string diff --git a/internal/logger/logger_test.go b/internal/logger/logger_test.go index 103374d..24bfeeb 100644 --- a/internal/logger/logger_test.go +++ b/internal/logger/logger_test.go @@ -23,7 +23,7 @@ func TestLoggerLogic(t *testing.T) { oslog.Fatal(err) } - t.Run("Messages arround the level", func(t *testing.T) { + t.Run("Messages around the level", func(t *testing.T) { log.Debugf("debug message") log.Errorf("error message") diff --git a/previewer.conf b/previewer.conf index 95b4767..50d13f8 100644 --- a/previewer.conf +++ b/previewer.conf @@ -4,6 +4,7 @@ Port = "80" [Cache] Capacity = 20 +StoragePath = "./assets/cache" [Query] Timeout = 15 diff --git a/assets/gopher_1024x252.jpg b/test/data/gopher_1024x252.jpg similarity index 100% rename from assets/gopher_1024x252.jpg rename to test/data/gopher_1024x252.jpg diff --git a/assets/gopher_2000x1000.jpg b/test/data/gopher_2000x1000.jpg similarity index 100% rename from assets/gopher_2000x1000.jpg rename to test/data/gopher_2000x1000.jpg diff --git a/assets/gopher_200x700.jpg b/test/data/gopher_200x700.jpg similarity index 100% rename from assets/gopher_200x700.jpg rename to test/data/gopher_200x700.jpg diff --git a/assets/gopher_256x126.jpg b/test/data/gopher_256x126.jpg similarity index 100% rename from assets/gopher_256x126.jpg rename to test/data/gopher_256x126.jpg diff --git a/assets/gopher_333x666.jpg b/test/data/gopher_333x666.jpg similarity index 100% rename from assets/gopher_333x666.jpg rename to test/data/gopher_333x666.jpg diff --git a/assets/gopher_500x500.jpg b/test/data/gopher_500x500.jpg similarity index 100% rename from assets/gopher_500x500.jpg rename to test/data/gopher_500x500.jpg diff --git a/assets/gopher_50x50.jpg b/test/data/gopher_50x50.jpg similarity index 100% rename from assets/gopher_50x50.jpg rename to test/data/gopher_50x50.jpg diff --git a/assets/gopher_original_1024x504.jpg b/test/data/gopher_original_1024x504.jpg similarity index 100% rename from assets/gopher_original_1024x504.jpg rename to test/data/gopher_original_1024x504.jpg