step-4
Vyacheslav Bakhmutov 2014-03-06 16:09:29 +04:00
parent 2d2849b161
commit 72e592cafc
7 changed files with 637 additions and 5 deletions

View File

@ -85,7 +85,7 @@ func GetApi() *martini.ClassicMartini {
})
api.Any("/bins/:name", func(r render.Render, storage Storage, params martini.Params,
req *http.Request, res http.ResponseWriter){
req *http.Request){
if bin, error := storage.LookupBin(params["name"]); error == nil {
request := NewRequest(req, REQUEST_BODY_SIZE)
if err := storage.CreateRequest(bin, request); err == nil {

331
src/skimmer/api_test.go Normal file
View File

@ -0,0 +1,331 @@
package skimmer
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"net/http"
"net/http/httptest"
"encoding/json"
"errors"
"fmt"
"bytes"
)
type MockedStorage struct{
mock.Mock
}
func (s *MockedStorage) CreateBin(_ *Bin) error {
args := s.Mock.Called()
return args.Error(0)
}
func (s *MockedStorage) UpdateBin(bin *Bin) error {
args := s.Mock.Called(bin)
return args.Error(0)
}
func (s *MockedStorage) LookupBin(name string) (*Bin, error) {
args := s.Mock.Called(name)
return args.Get(0).(*Bin), args.Error(1)
}
func (s *MockedStorage) LookupBins(names []string) ([]*Bin, error) {
args := s.Mock.Called(names)
return args.Get(0).([]*Bin), args.Error(1)
}
func (s *MockedStorage) LookupRequest(binName, id string) (*Request, error) {
args := s.Mock.Called(binName, id)
return args.Get(0).(*Request), args.Error(1)
}
func (s *MockedStorage) CreateRequest(bin *Bin, req *Request) error {
args := s.Mock.Called(bin)
return args.Error(0)
}
func (s *MockedStorage) LookupRequests(binName string, from, to int) ([]*Request, error) {
args := s.Mock.Called(binName, from, to)
return args.Get(0).([]*Request), args.Error(1)
}
func TestBinsPost(t *testing.T) {
api := GetApi()
req, err := http.NewRequest("POST", "/api/v1/bins/", nil)
if assert.Nil(t, err) {
res := httptest.NewRecorder()
api.ServeHTTP(res, req)
assert.Equal(t, res.Code, 201)
data := map[string] interface {}{}
err = json.Unmarshal([]byte(res.Body.String()), &data)
if assert.Nil(t, err) {
assert.Equal(t, len(data["name"].(string)), 6)
}
}
mockedStorage := &MockedStorage{}
api.MapTo(mockedStorage, (*Storage)(nil))
if assert.Nil(t, err) {
res := httptest.NewRecorder()
mockedStorage.On("CreateBin").Return(errors.New("Storage error"))
api.ServeHTTP(res, req)
mockedStorage.AssertExpectations(t)
assert.Equal(t, res.Code, 500)
assert.Contains(t, res.Body.String(), "Storage error")
}
}
func TestBinsGet(t *testing.T) {
req, err := http.NewRequest("GET", "/api/v1/bins/", nil)
if assert.Nil(t, err) {
api := GetApi()
res := httptest.NewRecorder()
api.ServeHTTP(res, req)
assert.Equal(t, res.Code, 200)
data := []map[string] interface {}{}
err = json.Unmarshal([]byte(res.Body.String()), &data)
if assert.Nil(t, err) {
assert.Equal(t, len(data), 0)
}
bin1, err := createBin(api)
if assert.Nil(t, err) {
bin2, err := createBin(api)
if assert.Nil(t, err) {
assert.NotEqual(t, bin1, bin2)
res := httptest.NewRecorder()
api.ServeHTTP(res, req)
assert.Equal(t, res.Code, 200)
data := []map[string] interface {}{}
err = json.Unmarshal([]byte(res.Body.String()), &data)
if assert.Nil(t, err) {
if assert.Equal(t, len(data), 2) {
assert.Equal(t, data[0], bin1)
assert.Equal(t, data[1], bin2)
}
}
}
}
req, _ := http.NewRequest("GET", "/api/v1/bins/", nil)
api = GetApi()
mockedStorage := &MockedStorage{}
api.MapTo(mockedStorage, (*Storage)(nil))
res = httptest.NewRecorder()
mockedStorage.On("LookupBins", []string{}).Return([]*Bin(nil), errors.New("Storage error"))
api.ServeHTTP(res, req)
mockedStorage.AssertExpectations(t)
if assert.Equal(t, res.Code, 500) {
assert.Contains(t, res.Body.String(), "Storage error")
}
}
}
func TestBinGet(t *testing.T) {
api := GetApi()
req, err := http.NewRequest("GET", "/api/v1/bins/name22", nil)
if assert.Nil(t, err) {
res := httptest.NewRecorder()
api.ServeHTTP(res, req)
assert.Equal(t, res.Code, 404)
}
bin, err := createBin(api)
if assert.Nil(t, err) {
req, err = http.NewRequest("GET", fmt.Sprintf("/api/v1/bins/%s", bin["name"]), nil)
if assert.Nil(t, err) {
res := httptest.NewRecorder()
api.ServeHTTP(res, req)
if assert.Equal(t, res.Code, 200) {
tBin := map[string]interface {}{}
err = json.Unmarshal([]byte(res.Body.String()), &tBin)
if assert.Nil(t, err) {
assert.Equal(t, tBin, bin)
}
}
}
}
req, err = http.NewRequest("GET", "/api/v1/bins/name22", nil)
if assert.Nil(t, err) {
api = GetApi()
mockedStorage := &MockedStorage{}
api.MapTo(mockedStorage, (*Storage)(nil))
res := httptest.NewRecorder()
mockedStorage.On("LookupBin", "name22").Return(new(Bin), errors.New("Storage error"))
api.ServeHTTP(res, req)
mockedStorage.AssertExpectations(t)
assert.Equal(t, res.Code, 404)
}
}
func TestRequestCreate(t *testing.T) {
api := GetApi()
bin, err := createBin(api)
if assert.Nil(t, err) {
req, err := http.NewRequest("GET", fmt.Sprintf("/bins/%s", bin["name"]), bytes.NewBuffer(nil))
if assert.Nil(t, err) {
res := httptest.NewRecorder()
api.ServeHTTP(res, req)
if assert.Equal(t, res.Code, 200) {
request := map[string]interface {}{}
err = json.Unmarshal([]byte(res.Body.String()), &request)
if assert.Nil(t, err) {
assert.Equal(t, request["method"].(string), "GET")
}
}
}
req, err = http.NewRequest("GET", fmt.Sprintf("/bins/%s", "name22"), bytes.NewBuffer(nil))
if assert.Nil(t, err) {
res := httptest.NewRecorder()
api.ServeHTTP(res, req)
assert.Equal(t, res.Code, 404)
}
}
api = GetApi()
mockedStorage := &MockedStorage{}
api.MapTo(mockedStorage, (*Storage)(nil))
res := httptest.NewRecorder()
realBin := NewBin()
req, err := http.NewRequest("GET", fmt.Sprintf("/bins/%s", realBin.Name), bytes.NewBuffer(nil))
if assert.Nil(t, err){
mockedStorage.On("LookupBin", realBin.Name).Return(realBin, nil)
mockedStorage.On("CreateRequest", realBin).Return(errors.New("Storage error"))
api.ServeHTTP(res, req)
mockedStorage.AssertExpectations(t)
if assert.Equal(t, res.Code, 500) {
assert.Contains(t, res.Body.String(), "Storage error")
}
}
}
func TestRequestsGet(t *testing.T) {
api := GetApi()
if bin, err := createBin(api); assert.Nil(t, err) {
url := fmt.Sprintf("/api/v1/bins/%s/requests/", bin["name"])
if req, err := http.NewRequest("GET", url, nil); assert.Nil(t, err){
res := httptest.NewRecorder()
api.ServeHTTP(res, req)
if assert.Equal(t, res.Code, 200) {
requests := []map[string]interface {}{}
if err = json.Unmarshal([]byte(res.Body.String()), &requests); assert.Nil(t, err) {
assert.Equal(t, len(requests), 0)
}
}
}
if request1, err := createRequest(api, bin["name"].(string)); assert.Nil(t, err) {
if request2, err := createRequest(api, bin["name"].(string)); assert.Nil(t, err) {
url = fmt.Sprintf("/api/v1/bins/%s/requests/", bin["name"])
if req, err := http.NewRequest("GET", url, nil); assert.Nil(t, err) {
res := httptest.NewRecorder()
api.ServeHTTP(res, req)
if assert.Equal(t, res.Code, 200) {
requests := []map[string]interface {}{}
if err = json.Unmarshal([]byte(res.Body.String()), &requests); assert.Nil(t, err) {
assert.Equal(t, len(requests), 2)
assert.Equal(t, requests, []map[string]interface {}{request2, request1})
}
}
}
url = fmt.Sprintf("/api/v1/bins/%s/requests/?from=1&to=2", bin["name"])
if req, err := http.NewRequest("GET", url, nil); assert.Nil(t, err) {
res := httptest.NewRecorder()
api.ServeHTTP(res, req)
if assert.Equal(t, res.Code, 200) {
requests := []map[string]interface {}{}
if err = json.Unmarshal([]byte(res.Body.String()), &requests); assert.Nil(t, err) {
assert.Equal(t, len(requests), 1)
assert.Equal(t, requests, []map[string]interface {}{request1})
}
}
}
}
}
}
api = GetApi()
mockedStorage := &MockedStorage{}
api.MapTo(mockedStorage, (*Storage)(nil))
realBin := NewBin()
req, err := http.NewRequest("GET", fmt.Sprintf("/api/v1/bins/%s/requests/", realBin.Name), bytes.NewBuffer(nil))
if assert.Nil(t, err){
res := httptest.NewRecorder()
mockedStorage.On("LookupBin", realBin.Name).Return(realBin, errors.New("Storage error")).Once()
api.ServeHTTP(res, req)
mockedStorage.AssertExpectations(t)
assert.Equal(t, res.Code, 404)
res = httptest.NewRecorder()
mockedStorage.On("LookupBin", realBin.Name).Return(realBin, nil)
mockedStorage.On("LookupRequests", realBin.Name, 0, 20).Return([]*Request{}, errors.New("Storage error"))
api.ServeHTTP(res, req)
mockedStorage.AssertExpectations(t)
if assert.Equal(t, res.Code, 500) {
assert.Contains(t, res.Body.String(), "Storage error")
}
}
}
func TestRequestGet(t *testing.T){
api := GetApi()
if bin, err := createBin(api); assert.Nil(t, err) {
url := fmt.Sprintf("/api/v1/bins/%s/requests/name", bin["name"])
if req, err := http.NewRequest("GET", url, nil); assert.Nil(t, err){
res := httptest.NewRecorder()
api.ServeHTTP(res, req)
assert.Equal(t, res.Code, 404)
}
if request, err := createRequest(api, bin["name"].(string)); assert.Nil(t, err) {
url = fmt.Sprintf("/api/v1/bins/%s/requests/%s", bin["name"], request["id"])
if req, err := http.NewRequest("GET", url, nil); assert.Nil(t, err) {
res := httptest.NewRecorder()
api.ServeHTTP(res, req)
if assert.Equal(t, res.Code, 200) {
tRequest := map[string]interface {}{}
if err = json.Unmarshal([]byte(res.Body.String()), &tRequest); assert.Nil(t, err) {
assert.Equal(t, tRequest, request)
}
}
}
}
}
api = GetApi()
mockedStorage := &MockedStorage{}
api.MapTo(mockedStorage, (*Storage)(nil))
res := httptest.NewRecorder()
realBin := NewBin()
req, err := http.NewRequest("GET", fmt.Sprintf("/api/v1/bins/%s/requests/id", realBin.Name), bytes.NewBuffer(nil))
if assert.Nil(t, err){
mockedStorage.On("LookupRequest", realBin.Name, "id").Return(new(Request), errors.New("Storage error"))
api.ServeHTTP(res, req)
mockedStorage.AssertExpectations(t)
assert.Equal(t, res.Code, 404)
}
}
func createBin(handler http.Handler) (bin map[string]interface {}, err error){
if req, err := http.NewRequest("POST", "/api/v1/bins/", nil); err == nil {
res := httptest.NewRecorder()
handler.ServeHTTP(res, req)
err = json.Unmarshal([]byte(res.Body.String()), &bin)
}
return
}
func createRequest(handler http.Handler, binName string) (request map[string]interface {}, err error){
if req, err := http.NewRequest("POST", fmt.Sprintf("/bins/%s", binName), bytes.NewBuffer(nil)); err == nil {
res := httptest.NewRecorder()
handler.ServeHTTP(res, req)
err = json.Unmarshal([]byte(res.Body.String()), &request)
}
return
}

View File

@ -106,12 +106,12 @@ func (storage *MemoryStorage) LookupRequests(binName string, from int, to int) (
if from > to {
from = to
}
reversedLen := to - from
reversedLen := len(binRecord.requests)
reversed := make([]*Request, reversedLen)
for i, request := range binRecord.requests[from:to] {
for i, request := range binRecord.requests {
reversed[reversedLen-i-1] = request
}
return reversed, nil
return reversed[from:to], nil
} else {
return nil, err
}

205
src/skimmer/memory_test.go Normal file
View File

@ -0,0 +1,205 @@
package skimmer
import (
"testing"
"github.com/stretchr/testify/assert"
"net/http"
"bytes"
)
func getMemoryStorage() *MemoryStorage {
return NewMemoryStorage(REQUEST_BODY_SIZE)
}
func TestNewMemoryStorage(t *testing.T) {
maxRequests := 20
storage := NewMemoryStorage(maxRequests)
assert.Equal(t, storage.maxRequests, maxRequests)
assert.NotNil(t, storage.binRecords)
}
func TestCreateBin(t *testing.T){
storage := getMemoryStorage()
bin := NewBin()
err := storage.CreateBin(bin)
if assert.Nil(t, err){
assert.Equal(t, storage.binRecords[bin.Name].bin, bin)
}
}
func TestUpdateBin(t *testing.T){
storage := getMemoryStorage()
bin := NewBin()
if err := storage.CreateBin(bin); assert.Nil(t, err) {
if err := storage.UpdateBin(bin); assert.Nil(t, err) {
assert.Equal(t, storage.binRecords[bin.Name].bin, bin)
}
}
}
func TestGetBinRecord(t *testing.T) {
storage := getMemoryStorage()
binRecord, err := storage.getBinRecord("test")
assert.NotNil(t, err)
assert.Nil(t, binRecord)
testBin := NewBin()
testBin.Name = "test"
err = storage.CreateBin(testBin)
if assert.Nil(t, err){
binRecord, err = storage.getBinRecord("test")
if assert.Nil(t, err) {
assert.Equal(t, binRecord.bin, testBin)
}
}
}
func TestLookupBin(t *testing.T) {
storage := getMemoryStorage()
_, err := storage.LookupBin("test")
assert.NotNil(t, err)
testBin := NewBin()
testBin.Name = "test"
err = storage.CreateBin(testBin)
if assert.Nil(t, err){
bin, err := storage.LookupBin("test")
if assert.Nil(t, err) {
assert.Equal(t, bin, testBin)
}
}
}
func TestLookupBins(t *testing.T) {
storage := getMemoryStorage()
bins, err := storage.LookupBins([]string{"test1", "test2"})
if assert.Nil(t, err) {
assert.Empty(t, bins)
}
testBin1 := NewBin()
testBin1.Name = "test1"
err = storage.CreateBin(testBin1)
if assert.Nil(t, err){
testBin2 := NewBin()
testBin2.Name = "test2"
err = storage.CreateBin(testBin2)
if assert.Nil(t, err){
bins, err = storage.LookupBins([]string{"test2", "test1"})
if assert.Nil(t, err) {
assert.Equal(t, bins, []*Bin{testBin2, testBin1})
}
bins, err = storage.LookupBins([]string{"test1"})
if assert.Nil(t, err) {
assert.Equal(t, bins, []*Bin{testBin1})
}
}
}
}
func TestCreateRequest(t *testing.T) {
storage := NewMemoryStorage(2)
bin := NewBin()
storage.CreateBin(bin)
httpRequest, _ := http.NewRequest("GET", "/", bytes.NewBuffer([]byte("body")))
request1 := NewRequest(httpRequest, REQUEST_BODY_SIZE)
err := storage.CreateRequest(bin, request1)
if assert.Nil(t, err) {
binRecord, _ := storage.getBinRecord(bin.Name)
assert.Equal(t, len(binRecord.requests), 1)
assert.Equal(t, binRecord.requests[0], request1)
assert.Equal(t, len(binRecord.requestMap), 1)
assert.Equal(t, binRecord.requestMap[request1.Id], request1)
assert.Equal(t, binRecord.bin.RequestCount, 1)
request2 := NewRequest(httpRequest, REQUEST_BODY_SIZE)
err = storage.CreateRequest(bin, request2)
if assert.Nil(t, err) {
assert.NotEqual(t, request1, request2)
assert.Equal(t, len(binRecord.requests), 2)
assert.Equal(t, binRecord.requests[1], request2)
assert.Equal(t, len(binRecord.requestMap), 2)
assert.Equal(t, binRecord.requestMap[request2.Id], request2)
assert.Equal(t, binRecord.bin.RequestCount, 2)
// shrinking
request3 := NewRequest(httpRequest, REQUEST_BODY_SIZE)
err = storage.CreateRequest(bin, request3)
if assert.Nil(t, err) {
assert.Equal(t, len(binRecord.requests), 2)
assert.Equal(t, binRecord.requests[1], request3)
assert.Equal(t, len(binRecord.requestMap), 2)
assert.Equal(t, binRecord.requestMap[request3.Id], request3)
assert.Equal(t, binRecord.bin.RequestCount, 2)
_, err = storage.LookupRequest(bin.Name, request1.Id)
assert.NotNil(t, err)
}
}
}
err = storage.CreateRequest(&Bin{Name:"wrong_name"}, request1)
assert.NotNil(t, err)
}
func TestLookupRequest(t *testing.T) {
storage := getMemoryStorage()
bin := NewBin()
storage.CreateBin(bin)
httpRequest, _ := http.NewRequest("GET", "/", bytes.NewBuffer([]byte("body")))
request := NewRequest(httpRequest, REQUEST_BODY_SIZE)
err := storage.CreateRequest(bin, request)
if assert.Nil(t, err) {
tmpRequest, err := storage.LookupRequest(bin.Name, request.Id)
if assert.Nil(t, err) {
assert.Equal(t, request, tmpRequest)
}
}
_, err = storage.LookupRequest(bin.Name, "bad id")
assert.NotNil(t, err)
_, err = storage.LookupRequest("bad name", request.Id)
assert.NotNil(t, err)
}
func TestLookupRequests(t *testing.T) {
storage := getMemoryStorage()
_, err := storage.LookupRequests("bad name", 0, 2)
assert.NotNil(t, err)
bin := NewBin()
storage.CreateBin(bin)
requests, err := storage.LookupRequests(bin.Name, 0, 3)
if assert.Nil(t, err) {
assert.Empty(t, requests)
}
httpRequest, _ := http.NewRequest("GET", "/", bytes.NewBuffer([]byte("body")))
request1 := NewRequest(httpRequest, REQUEST_BODY_SIZE)
request2 := NewRequest(httpRequest, REQUEST_BODY_SIZE)
request3 := NewRequest(httpRequest, REQUEST_BODY_SIZE)
storage.CreateRequest(bin, request1)
storage.CreateRequest(bin, request2)
storage.CreateRequest(bin, request3)
requests, err = storage.LookupRequests(bin.Name, 1, 3)
if assert.Nil(t, err) {
assert.Equal(t, requests[0], request2)
assert.Equal(t, requests[1], request1)
}
requests, err = storage.LookupRequests(bin.Name, 0, 1)
if assert.Nil(t, err) {
assert.Equal(t, requests[0], request3)
}
requests, err = storage.LookupRequests(bin.Name, -1, 100)
if assert.Nil(t, err) {
assert.Equal(t, len(requests), 3)
}
requests, err = storage.LookupRequests(bin.Name, 1, -2)
if assert.Nil(t, err) {
assert.Equal(t, len(requests), 0)
}
}

View File

@ -79,7 +79,7 @@ func NewRequest(httpRequest *http.Request, maxBodySize int) *Request {
Header: httpRequest.Header,
ContentLength: httpRequest.ContentLength,
RemoteAddr: httpRequest.RemoteAddr,
RequestURI: httpRequest.RequestURI,
RequestURI: httpRequest.URL.RequestURI(),
FormValue: formValue,
FormFile: formFile,
Body: bodyValue,

View File

@ -0,0 +1,78 @@
package skimmer
import (
"github.com/stretchr/testify/assert"
"testing"
"time"
"bytes"
"strings"
"net/http"
"fmt"
)
func TestRs(t *testing.T) {
pool := "0123456789abcdefghijklmnopqrstuvwxyz"
val := rs.Generate(1000)
for _, alpha := range val {
assert.Contains(t, pool, string(alpha))
}
}
func TestNewBin(t *testing.T) {
now := time.Now().Unix()
bin := NewBin()
if assert.NotNil(t, bin) {
assert.Equal(t, len(bin.Name), 6)
assert.Equal(t, bin.RequestCount, 0)
assert.Equal(t, bin.Created, bin.Updated)
assert.True(t, bin.Created < (now+1))
assert.True(t, bin.Created > (now-1))
}
}
func TestNewRequest(t *testing.T) {
body := []byte("arg1=value1&arg2=value2")
httpRequest, err := http.NewRequest("POST", "http://www.google.com/path/", bytes.NewBuffer(body))
if assert.Nil(t, err){
httpRequest.Header.Add("Content-Type", "application/x-www-form-urlencoded")
req := NewRequest(httpRequest, 100)
if assert.NotNil(t, req) {
assert.Equal(t, len(req.Id), 12)
assert.Equal(t, req.Method, "POST")
assert.Equal(t, req.Proto, "HTTP/1.1")
assert.Equal(t, req.Host, "www.google.com")
assert.Equal(t, req.ContentLength, len(body))
assert.Equal(t, req.RequestURI, "/path/")
assert.Equal(t, req.FormValue, map[string][]string{"arg1": {"value1"}, "arg2": {"value2"}})
assert.Equal(t, req.FormFile, []string(nil))
assert.Equal(t, req.Body, string(body))
}
}
httpRequest, err = http.NewRequest("POST", "http://www.google.com/path/", bytes.NewBuffer(body))
if assert.Nil(t, err){
httpRequest.Header.Add("Content-Type", "application/x-www-form-urlencoded")
// test truncating
req := NewRequest(httpRequest, 10)
if assert.NotNil(t, req) {
assert.Equal(t, req.ContentLength, len(body))
assert.Equal(t, req.FormValue, map[string][]string{"arg1": {"value1"}, "arg2": {"value2"}})
assert.NotEqual(t, req.Body, string(body))
assert.True(t, strings.HasSuffix(string(req.Body), fmt.Sprintf("\n<<<TRUNCATED , %d of %d", 10, len(body))))
assert.Equal(t, strings.Index(string(req.Body), "\n<<<TRUNCATED"), 10)
}
}
multipartBody := []byte("\r\n--B1897D34-1FF0-40CD-BDAA-61A5662758C1\r\nContent-Disposition: form-data; name=\"todo.sublime-project\"; filename=\"todo.sublime-project\"\r\nContent-Transfer-Encoding: binary\r\nContent-Type: (null)\r\n\r\n{\n}\n\r\n--B1897D34-1FF0-40CD-BDAA-61A5662758C1--\r\n")
httpRequest, err = http.NewRequest("POST", "http://www.google.com/path/", bytes.NewBuffer(multipartBody))
if assert.Nil(t, err) {
httpRequest.Header.Add("Content-Type", "multipart/form-data; boundary=B1897D34-1FF0-40CD-BDAA-61A5662758C1")
req := NewRequest(httpRequest, 1000)
if assert.NotNil(t, req) {
assert.NotEqual(t, req.ContentLength, len(body))
assert.Equal(t, req.FormValue, map[string][]string{})
assert.Equal(t, req.Body, string(multipartBody))
assert.Equal(t, req.FormFile, []string{"todo.sublime-project"})
}
}
}

18
src/skimmer/utils_test.go Normal file
View File

@ -0,0 +1,18 @@
package skimmer
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestRandomString(t *testing.T) {
pool := "abcdefg12345"
rs := NewRandomString(pool)
val := rs.Generate(1000)
for _, alpha := range(val) {
assert.Contains(t, pool, string(alpha))
}
assert.Empty(t, rs.Generate(-1))
}