OTUS
/
GoLang_Project
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Добавил кэш. Решил переделать структуру приложения.

master
Andrey Ivanov 2020-10-22 17:54:00 +03:00 committed by Andrey Ivanov
parent 8432b5a165
commit 50ba0b0cca
11 changed files with 499 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
.golangci.yml
View File

@ -1,19 +1,16 @@
run:
tests: false
tests: true
linters:
disable-all: false
enable-all: true
disable:
- goerr113
- gochecknoglobals
- gochecknoinits
- godox
- goerr113
- gomnd
- lll
- nakedret
- wsl
- gofumpt
- gosec
- nlreturn
- gocritic
- testpackage
- wsl

41
previewer/application/application.go
View File

@ -1,18 +1,18 @@
package application
import (
"github.com/tiburon-777/OTUS_Project/previewer/cache"
"github.com/tiburon-777/OTUS_Project/previewer/config"
"github.com/tiburon-777/OTUS_Project/previewer/logger"
oslog "log"
"net"
"net/http"
"time"
"github.com/tiburon-777/OTUS_Project/previewer/config"
"github.com/tiburon-777/OTUS_Project/previewer/logger"
)
type App struct {
*http.Server
Log logger.Interface
Cache cache.Cache
}
func New(conf config.Config) *App {
@ -20,37 +20,14 @@ func New(conf config.Config) *App {
if err != nil {
oslog.Fatal("не удалось прикрутить логгер: ", err.Error())
}
return &App{Server: &http.Server{Addr: net.JoinHostPort(conf.Server.Address, conf.Server.Port), Handler: LoggingMiddleware(http.HandlerFunc(Handler), loger)}, Log: loger}
}
func Handler(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(200)
_, _ = w.Write([]byte("Hello! Хрен вам, а не картинка!!!"))
}
func LoggingMiddleware(next http.Handler, l logger.Interface) http.HandlerFunc {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
start := time.Now()
defer func() {
var path, useragent string
if r.URL != nil {
path = r.URL.Path
}
if len(r.UserAgent()) > 0 {
useragent = r.UserAgent()
}
latency := time.Since(start)
l.Infof("receive %s request from IP: %s on path: %s, duration: %s useragent: %s ", r.Method, r.RemoteAddr, path, latency, useragent)
}()
next.ServeHTTP(w, r)
})
c := cache.NewCache(conf.Cache.Capasity)
return &App{Server: &http.Server{Addr: net.JoinHostPort(conf.Server.Address, conf.Server.Port), Handler: LoggingMiddleware(http.HandlerFunc(Handler), loger)}, Log: loger, Cache: c}
}
func (s *App) Start() error {
if err := s.ListenAndServe(); err != nil {
return err
}
s.Log.Infof("Server starting")
_ = s.ListenAndServe()
s.Log.Infof("Server stoped")
return nil
}
@ -58,6 +35,6 @@ func (s *App) Stop() error {
if err := s.Close(); err != nil {
return err
}
s.Log.Infof("Server stoped")
return nil
}

51
previewer/application/functions.go Normal file
View File

@ -0,0 +1,51 @@
package application
import (
"errors"
"github.com/tiburon-777/OTUS_Project/previewer/models"
"io/ioutil"
"net/http"
"net/url"
"strconv"
"strings"
)
func buildQuery(u *url.URL) (q models.Query, err error) {
t := strings.Split(u.Path, "/")
q.Width,err=strconv.Atoi(t[1])
if err!=nil {
return models.Query{}, errors.New("width must be an integer")
}
q.Height,err=strconv.Atoi(t[2])
if err!=nil {
return models.Query{}, errors.New("height must be an integer")
}
tn := "http://"+strings.Join(t[3:],"/")
q.URL,err=q.URL.Parse(tn)
if err!=nil {
return models.Query{}, errors.New("not valid url")
}
return q,nil
}
func getPic(q models.Query) ([]byte,http.Header,error) {
client := &http.Client{}
req, err := http.NewRequest("GET", "https://" + q.URL.Host + "/" + q.URL.Path, nil)
if err != nil {
return nil, nil, err
}
req.Close = true
res, err := client.Do(req)
if err != nil {
return nil, nil, err
}
body, err := ioutil.ReadAll(res.Body)
if err != nil {
return nil, nil, err
}
if err = res.Body.Close(); err != nil {
return nil, nil, err
}
return body, res.Header, nil
}

47
previewer/application/handlers.go Normal file
View File

@ -0,0 +1,47 @@
package application
import (
"github.com/tiburon-777/OTUS_Project/previewer/logger"
"net/http"
"time"
)
func Handler(w http.ResponseWriter, r *http.Request) {
q,err := buildQuery(r.URL)
if err!=nil {
writeResponce(w, r.Header,501,[]byte("Can't parse query"))
}
pic, h, err := getPic(q)
if err!=nil {
writeResponce(w, h,501,[]byte("Have problem with cache"))
}
writeResponce(w, r.Header,200,pic)
}
func LoggingMiddleware(next http.Handler, l logger.Interface) http.HandlerFunc {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
start := time.Now()
defer func() {
var path, useragent string
if r.URL != nil {
path = r.URL.Path
}
if len(r.UserAgent()) > 0 {
useragent = r.UserAgent()
}
latency := time.Since(start)
l.Infof("receive %s request from IP: %s on path: %s, duration: %s useragent: %s ", r.Method, r.RemoteAddr, path, latency, useragent)
}()
next.ServeHTTP(w, r)
})
}
func writeResponce(w http.ResponseWriter, h http.Header, code int, body []byte) {
for name, values := range h {
for _, value := range values {
w.Header().Add(name,value)
}
}
_, _ = w.Write(body)
}

80
previewer/cache/cache.go vendored Normal file
View File

@ -0,0 +1,80 @@
package cache
import (
"log"
"sync"
)
type Key string
type Cache interface {
Set(key Key, value interface{}) bool // Добавить значение в кэш по ключу
Get(key Key) (interface{}, bool) // Получить значение из кэша по ключу
Clear() // Очистить кэш
}
type lruCache struct {
capacity int
queue *List
items map[Key]*ListItem
mx sync.RWMutex
}
type Item struct {
Key Key
Value interface{}
}
func NewCache(capacity int) Cache {
return &lruCache{
capacity: capacity,
queue: NewList(),
items: make(map[Key]*ListItem),
}
}
func (l *lruCache) Set(key Key, value interface{}) bool {
if _, exists := l.items[key]; exists {
l.mx.RLock()
l.items[key].Value = Item{Value: value, Key: key}
l.queue.MoveToFront(l.items[key])
l.mx.RUnlock()
return exists
}
if l.queue.Len() == l.capacity {
l.mx.RLock()
k, ok := l.queue.Back().Value.(Item)
if !ok {
log.Fatal("Ошибка приведения типов")
}
delete(l.items, k.Key)
l.queue.Remove(l.queue.Back())
l.mx.RUnlock()
}
l.mx.RLock()
l.items[key] = l.queue.PushFront(Item{Value: value, Key: key})
l.mx.RUnlock()
return false
}
func (l *lruCache) Get(key Key) (interface{}, bool) {
l.mx.Lock()
defer l.mx.Unlock()
if l.items[key] == nil {
return nil, false
}
l.queue.MoveToFront(l.items[key])
s, ok := l.items[key].Value.(Item)
if !ok {
log.Fatal("Ошибка приведения типов")
}
return s.Value, true
}
func (l *lruCache) Clear() {
l.mx.Lock()
l.items = nil
l.queue.len = 0
l.queue.Info = ListItem{}
l.mx.Unlock()
}

102
previewer/cache/cache_test.go vendored Normal file
View File

@ -0,0 +1,102 @@
package cache
import (
"math/rand"
"strconv"
"sync"
"testing"
"github.com/stretchr/testify/require"
)
func TestCache(t *testing.T) {
t.Run("empty cache", func(t *testing.T) {
c := NewCache(10)
_, ok := c.Get("aaa")
require.False(t, ok)
_, ok = c.Get("bbb")
require.False(t, ok)
})
t.Run("simple", func(t *testing.T) {
c := NewCache(5)
wasInCache := c.Set("aaa", 100)
require.False(t, wasInCache)
wasInCache = c.Set("bbb", 200)
require.False(t, wasInCache)
val, ok := c.Get("aaa")
require.True(t, ok)
require.Equal(t, 100, val)
val, ok = c.Get("bbb")
require.True(t, ok)
require.Equal(t, 200, val)
wasInCache = c.Set("aaa", 300)
require.True(t, wasInCache)
val, ok = c.Get("aaa")
require.True(t, ok)
require.Equal(t, 300, val)
val, ok = c.Get("ccc")
require.False(t, ok)
require.Nil(t, val)
})
t.Run("purge logic", func(t *testing.T) {
c := NewCache(3)
wasInCache := c.Set("aaa", 100)
require.False(t, wasInCache)
wasInCache = c.Set("bbb", 200)
require.False(t, wasInCache)
wasInCache = c.Set("ccc", 300)
require.False(t, wasInCache)
_, ok := c.Get("bbb")
require.True(t, ok)
_, ok = c.Get("aaa")
require.True(t, ok)
wasInCache = c.Set("ddd", 400)
require.False(t, wasInCache)
_, ok = c.Get("ddd")
require.True(t, ok)
_, ok = c.Get("ccc")
require.False(t, ok)
})
}
func TestCacheMultithreading(t *testing.T) {
c := NewCache(10)
wg := &sync.WaitGroup{}
wg.Add(2)
go func() {
defer wg.Done()
for i := 0; i < 1_000_000; i++ {
c.Set(Key(strconv.Itoa(i)), i)
}
}()
go func() {
defer wg.Done()
for i := 0; i < 1_000_000; i++ {
c.Get(Key(strconv.Itoa(rand.Intn(1_000_000))))
}
}()
wg.Wait()
}

101
previewer/cache/list.go vendored Normal file
View File

@ -0,0 +1,101 @@
package cache
type ListInterface interface {
Len() int // длина списка
Front() *ListItem // первый Item
Back() *ListItem // последний Item
PushFront(v interface{}) *ListItem // добавить значение в начало
PushBack(v interface{}) *ListItem // добавить значение в конец
Remove(i *ListItem) // удалить элемент
MoveToFront(i *ListItem) // переместить элемент в начало
}
type ListItem struct {
Value interface{} // значение
Next *ListItem // следующий элемент
Prev *ListItem // предыдущий элемент
}
type List struct {
Info ListItem
len int
}
func NewList() *List {
return &List{len: 0}
}
func (l *List) Len() int {
return l.len
}
func (l *List) Front() *ListItem {
return l.Info.Next
}
func (l *List) Back() *ListItem {
return l.Info.Prev
}
func (l *List) PushFront(v interface{}) *ListItem {
e := &ListItem{Value: v}
if l.len != 0 {
e.Prev = l.Info.Next
l.Info.Next.Next = e
} else {
l.Info.Prev = e
}
l.Info.Next = e
l.len++
return e
}
func (l *List) PushBack(v interface{}) *ListItem {
e := &ListItem{Value: v}
if l.len != 0 {
e.Next = l.Info.Prev
l.Info.Prev.Prev = e
} else {
l.Info.Next = e
}
l.Info.Prev = e
l.len++
return e
}
func (l *List) Remove(i *ListItem) {
if i.Prev == nil {
i.Prev = &ListItem{}
l.Info.Prev = i.Next
}
if i.Next == nil {
i.Next = &ListItem{}
l.Info.Next = i.Prev
}
i.Prev.Next = i.Next
i.Next.Prev = i.Prev
if l.Len() > 1 {
l.Info.Next.Next = nil
l.Info.Prev.Prev = nil
} else {
l.Info = ListItem{}
}
l.len--
}
func (l *List) MoveToFront(i *ListItem) {
if l.Info.Next == i {
return
}
if i.Prev != nil {
i.Prev.Next = i.Next
i.Next.Prev = i.Prev
} else {
i.Next.Prev = i.Prev
l.Info.Prev = i.Next
}
i.Prev = l.Front()
l.Front().Next = i
i.Next = nil
l.Info.Next = i
}

96
previewer/cache/list_test.go vendored Normal file
View File

@ -0,0 +1,96 @@
package cache
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestList(t *testing.T) {
t.Run("empty list", func(t *testing.T) {
l := NewList()
require.Equal(t, l.Len(), 0)
require.Nil(t, l.Front())
require.Nil(t, l.Back())
})
t.Run("complex", func(t *testing.T) {
l := NewList()
l.PushFront(10)
require.Equal(t, []int{10}, transpond(l))
require.Equal(t, 1, l.Len())
l.Remove(l.Front())
require.Equal(t, []int{}, transpond(l))
require.Equal(t, 0, l.Len())
l.PushFront(10)
require.Equal(t, []int{10}, transpond(l))
require.Equal(t, 1, l.Len())
l.PushBack(20)
require.Equal(t, []int{20, 10}, transpond(l))
require.Equal(t, 2, l.Len())
l.PushBack(30)
require.Equal(t, []int{30, 20, 10}, transpond(l))
require.Equal(t, 3, l.Len())
middle := l.Back().Next // 20
l.Remove(middle)
require.Equal(t, []int{30, 10}, transpond(l))
require.Equal(t, 2, l.Len())
l.PushFront(20)
require.Equal(t, []int{30, 10, 20}, transpond(l))
require.Equal(t, 3, l.Len())
l.Remove(l.Back())
require.Equal(t, []int{10, 20}, transpond(l))
require.Equal(t, 2, l.Len())
l.PushBack(30)
require.Equal(t, []int{30, 10, 20}, transpond(l))
require.Equal(t, 3, l.Len())
l.Remove(l.Front())
require.Equal(t, []int{30, 10}, transpond(l))
require.Equal(t, 2, l.Len())
l.PushFront(20)
require.Equal(t, []int{30, 10, 20}, transpond(l))
require.Equal(t, 3, l.Len())
for i, v := range [...]int{40, 50, 60, 70, 80} {
if i%2 == 0 {
l.PushFront(v)
} else {
l.PushBack(v)
}
}
require.Equal(t, []int{70, 50, 30, 10, 20, 40, 60, 80}, transpond(l))
require.Equal(t, 8, l.Len())
l.MoveToFront(l.Front())
require.Equal(t, []int{70, 50, 30, 10, 20, 40, 60, 80}, transpond(l))
require.Equal(t, 8, l.Len())
l.MoveToFront(l.Back().Next.Next) // 30
require.Equal(t, []int{70, 50, 10, 20, 40, 60, 80, 30}, transpond(l))
require.Equal(t, 8, l.Len())
l.MoveToFront(l.Back())
require.Equal(t, []int{50, 10, 20, 40, 60, 80, 30, 70}, transpond(l))
require.Equal(t, 8, l.Len())
})
}
func transpond(l *List) []int {
elems := make([]int, 0, l.Len())
for i := l.Back(); i != nil; i = i.Next {
elems = append(elems, i.Value.(int))
}
return elems
}

1
previewer/main.go
View File

@ -20,7 +20,6 @@ func main() {
}
app := application.New(conf)
go func() {
signals := make(chan os.Signal, 1)
signal.Notify(signals)

8
previewer/models/models.go
View File

@ -1,5 +1,13 @@
package models
import "net/url"
type Config struct {
LruCapasity int
}
type Query struct {
Height int
Width int
URL *url.URL
}

1
previewer/webserver/webserver.go Normal file
View File

@ -0,0 +1 @@
package webserver