diff --git a/src/skimmer/api.go b/src/skimmer/api.go index 91ee574..a3dafd3 100644 --- a/src/skimmer/api.go +++ b/src/skimmer/api.go @@ -85,7 +85,7 @@ func GetApi() *martini.ClassicMartini { }) api.Any("/bins/:name", func(r render.Render, storage Storage, params martini.Params, - req *http.Request, res http.ResponseWriter){ + req *http.Request){ if bin, error := storage.LookupBin(params["name"]); error == nil { request := NewRequest(req, REQUEST_BODY_SIZE) if err := storage.CreateRequest(bin, request); err == nil { diff --git a/src/skimmer/api_test.go b/src/skimmer/api_test.go new file mode 100644 index 0000000..ff1c2f2 --- /dev/null +++ b/src/skimmer/api_test.go @@ -0,0 +1,331 @@ +package skimmer + + +import ( + "testing" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "net/http" + "net/http/httptest" + "encoding/json" + "errors" + "fmt" + "bytes" +) + +type MockedStorage struct{ + mock.Mock +} + +func (s *MockedStorage) CreateBin(_ *Bin) error { + args := s.Mock.Called() + return args.Error(0) +} + +func (s *MockedStorage) UpdateBin(bin *Bin) error { + args := s.Mock.Called(bin) + return args.Error(0) +} + +func (s *MockedStorage) LookupBin(name string) (*Bin, error) { + args := s.Mock.Called(name) + return args.Get(0).(*Bin), args.Error(1) +} + +func (s *MockedStorage) LookupBins(names []string) ([]*Bin, error) { + args := s.Mock.Called(names) + return args.Get(0).([]*Bin), args.Error(1) +} + +func (s *MockedStorage) LookupRequest(binName, id string) (*Request, error) { + args := s.Mock.Called(binName, id) + return args.Get(0).(*Request), args.Error(1) +} + +func (s *MockedStorage) CreateRequest(bin *Bin, req *Request) error { + args := s.Mock.Called(bin) + return args.Error(0) +} + +func (s *MockedStorage) LookupRequests(binName string, from, to int) ([]*Request, error) { + args := s.Mock.Called(binName, from, to) + return args.Get(0).([]*Request), args.Error(1) +} + + +func TestBinsPost(t *testing.T) { + api := GetApi() + req, err := http.NewRequest("POST", "/api/v1/bins/", nil) + if assert.Nil(t, err) { + res := httptest.NewRecorder() + api.ServeHTTP(res, req) + assert.Equal(t, res.Code, 201) + data := map[string] interface {}{} + err = json.Unmarshal([]byte(res.Body.String()), &data) + if assert.Nil(t, err) { + assert.Equal(t, len(data["name"].(string)), 6) + } + } + mockedStorage := &MockedStorage{} + api.MapTo(mockedStorage, (*Storage)(nil)) + if assert.Nil(t, err) { + res := httptest.NewRecorder() + mockedStorage.On("CreateBin").Return(errors.New("Storage error")) + api.ServeHTTP(res, req) + mockedStorage.AssertExpectations(t) + assert.Equal(t, res.Code, 500) + assert.Contains(t, res.Body.String(), "Storage error") + } +} + +func TestBinsGet(t *testing.T) { + req, err := http.NewRequest("GET", "/api/v1/bins/", nil) + if assert.Nil(t, err) { + api := GetApi() + res := httptest.NewRecorder() + api.ServeHTTP(res, req) + assert.Equal(t, res.Code, 200) + data := []map[string] interface {}{} + err = json.Unmarshal([]byte(res.Body.String()), &data) + if assert.Nil(t, err) { + assert.Equal(t, len(data), 0) + } + + bin1, err := createBin(api) + if assert.Nil(t, err) { + bin2, err := createBin(api) + if assert.Nil(t, err) { + assert.NotEqual(t, bin1, bin2) + res := httptest.NewRecorder() + api.ServeHTTP(res, req) + assert.Equal(t, res.Code, 200) + data := []map[string] interface {}{} + err = json.Unmarshal([]byte(res.Body.String()), &data) + if assert.Nil(t, err) { + if assert.Equal(t, len(data), 2) { + assert.Equal(t, data[0], bin1) + assert.Equal(t, data[1], bin2) + } + } + } + } + req, _ := http.NewRequest("GET", "/api/v1/bins/", nil) + api = GetApi() + mockedStorage := &MockedStorage{} + api.MapTo(mockedStorage, (*Storage)(nil)) + res = httptest.NewRecorder() + mockedStorage.On("LookupBins", []string{}).Return([]*Bin(nil), errors.New("Storage error")) + api.ServeHTTP(res, req) + mockedStorage.AssertExpectations(t) + if assert.Equal(t, res.Code, 500) { + assert.Contains(t, res.Body.String(), "Storage error") + } + } + +} + +func TestBinGet(t *testing.T) { + api := GetApi() + req, err := http.NewRequest("GET", "/api/v1/bins/name22", nil) + if assert.Nil(t, err) { + res := httptest.NewRecorder() + api.ServeHTTP(res, req) + assert.Equal(t, res.Code, 404) + } + bin, err := createBin(api) + if assert.Nil(t, err) { + req, err = http.NewRequest("GET", fmt.Sprintf("/api/v1/bins/%s", bin["name"]), nil) + if assert.Nil(t, err) { + res := httptest.NewRecorder() + api.ServeHTTP(res, req) + if assert.Equal(t, res.Code, 200) { + tBin := map[string]interface {}{} + err = json.Unmarshal([]byte(res.Body.String()), &tBin) + if assert.Nil(t, err) { + assert.Equal(t, tBin, bin) + } + } + } + } + req, err = http.NewRequest("GET", "/api/v1/bins/name22", nil) + if assert.Nil(t, err) { + api = GetApi() + mockedStorage := &MockedStorage{} + api.MapTo(mockedStorage, (*Storage)(nil)) + res := httptest.NewRecorder() + mockedStorage.On("LookupBin", "name22").Return(new(Bin), errors.New("Storage error")) + api.ServeHTTP(res, req) + mockedStorage.AssertExpectations(t) + assert.Equal(t, res.Code, 404) + } + +} + +func TestRequestCreate(t *testing.T) { + api := GetApi() + bin, err := createBin(api) + if assert.Nil(t, err) { + req, err := http.NewRequest("GET", fmt.Sprintf("/bins/%s", bin["name"]), bytes.NewBuffer(nil)) + if assert.Nil(t, err) { + res := httptest.NewRecorder() + api.ServeHTTP(res, req) + if assert.Equal(t, res.Code, 200) { + request := map[string]interface {}{} + err = json.Unmarshal([]byte(res.Body.String()), &request) + if assert.Nil(t, err) { + assert.Equal(t, request["method"].(string), "GET") + } + } + } + req, err = http.NewRequest("GET", fmt.Sprintf("/bins/%s", "name22"), bytes.NewBuffer(nil)) + if assert.Nil(t, err) { + res := httptest.NewRecorder() + api.ServeHTTP(res, req) + assert.Equal(t, res.Code, 404) + } + } + + api = GetApi() + mockedStorage := &MockedStorage{} + api.MapTo(mockedStorage, (*Storage)(nil)) + res := httptest.NewRecorder() + realBin := NewBin() + req, err := http.NewRequest("GET", fmt.Sprintf("/bins/%s", realBin.Name), bytes.NewBuffer(nil)) + if assert.Nil(t, err){ + mockedStorage.On("LookupBin", realBin.Name).Return(realBin, nil) + mockedStorage.On("CreateRequest", realBin).Return(errors.New("Storage error")) + api.ServeHTTP(res, req) + mockedStorage.AssertExpectations(t) + if assert.Equal(t, res.Code, 500) { + assert.Contains(t, res.Body.String(), "Storage error") + } + } + +} + +func TestRequestsGet(t *testing.T) { + api := GetApi() + + if bin, err := createBin(api); assert.Nil(t, err) { + url := fmt.Sprintf("/api/v1/bins/%s/requests/", bin["name"]) + if req, err := http.NewRequest("GET", url, nil); assert.Nil(t, err){ + res := httptest.NewRecorder() + api.ServeHTTP(res, req) + if assert.Equal(t, res.Code, 200) { + requests := []map[string]interface {}{} + if err = json.Unmarshal([]byte(res.Body.String()), &requests); assert.Nil(t, err) { + assert.Equal(t, len(requests), 0) + } + } + } + if request1, err := createRequest(api, bin["name"].(string)); assert.Nil(t, err) { + if request2, err := createRequest(api, bin["name"].(string)); assert.Nil(t, err) { + url = fmt.Sprintf("/api/v1/bins/%s/requests/", bin["name"]) + if req, err := http.NewRequest("GET", url, nil); assert.Nil(t, err) { + res := httptest.NewRecorder() + api.ServeHTTP(res, req) + if assert.Equal(t, res.Code, 200) { + requests := []map[string]interface {}{} + if err = json.Unmarshal([]byte(res.Body.String()), &requests); assert.Nil(t, err) { + assert.Equal(t, len(requests), 2) + assert.Equal(t, requests, []map[string]interface {}{request2, request1}) + } + } + } + url = fmt.Sprintf("/api/v1/bins/%s/requests/?from=1&to=2", bin["name"]) + + if req, err := http.NewRequest("GET", url, nil); assert.Nil(t, err) { + res := httptest.NewRecorder() + api.ServeHTTP(res, req) + if assert.Equal(t, res.Code, 200) { + requests := []map[string]interface {}{} + if err = json.Unmarshal([]byte(res.Body.String()), &requests); assert.Nil(t, err) { + assert.Equal(t, len(requests), 1) + assert.Equal(t, requests, []map[string]interface {}{request1}) + } + } + } + } + } + } + + api = GetApi() + mockedStorage := &MockedStorage{} + api.MapTo(mockedStorage, (*Storage)(nil)) + realBin := NewBin() + req, err := http.NewRequest("GET", fmt.Sprintf("/api/v1/bins/%s/requests/", realBin.Name), bytes.NewBuffer(nil)) + if assert.Nil(t, err){ + res := httptest.NewRecorder() + mockedStorage.On("LookupBin", realBin.Name).Return(realBin, errors.New("Storage error")).Once() + api.ServeHTTP(res, req) + mockedStorage.AssertExpectations(t) + assert.Equal(t, res.Code, 404) + + res = httptest.NewRecorder() + mockedStorage.On("LookupBin", realBin.Name).Return(realBin, nil) + mockedStorage.On("LookupRequests", realBin.Name, 0, 20).Return([]*Request{}, errors.New("Storage error")) + api.ServeHTTP(res, req) + mockedStorage.AssertExpectations(t) + if assert.Equal(t, res.Code, 500) { + assert.Contains(t, res.Body.String(), "Storage error") + } + } +} + +func TestRequestGet(t *testing.T){ + api := GetApi() + + if bin, err := createBin(api); assert.Nil(t, err) { + url := fmt.Sprintf("/api/v1/bins/%s/requests/name", bin["name"]) + if req, err := http.NewRequest("GET", url, nil); assert.Nil(t, err){ + res := httptest.NewRecorder() + api.ServeHTTP(res, req) + assert.Equal(t, res.Code, 404) + } + if request, err := createRequest(api, bin["name"].(string)); assert.Nil(t, err) { + url = fmt.Sprintf("/api/v1/bins/%s/requests/%s", bin["name"], request["id"]) + if req, err := http.NewRequest("GET", url, nil); assert.Nil(t, err) { + res := httptest.NewRecorder() + api.ServeHTTP(res, req) + if assert.Equal(t, res.Code, 200) { + tRequest := map[string]interface {}{} + if err = json.Unmarshal([]byte(res.Body.String()), &tRequest); assert.Nil(t, err) { + assert.Equal(t, tRequest, request) + } + } + } + } + } + + api = GetApi() + mockedStorage := &MockedStorage{} + api.MapTo(mockedStorage, (*Storage)(nil)) + res := httptest.NewRecorder() + realBin := NewBin() + req, err := http.NewRequest("GET", fmt.Sprintf("/api/v1/bins/%s/requests/id", realBin.Name), bytes.NewBuffer(nil)) + if assert.Nil(t, err){ + mockedStorage.On("LookupRequest", realBin.Name, "id").Return(new(Request), errors.New("Storage error")) + api.ServeHTTP(res, req) + mockedStorage.AssertExpectations(t) + assert.Equal(t, res.Code, 404) + } +} + +func createBin(handler http.Handler) (bin map[string]interface {}, err error){ + if req, err := http.NewRequest("POST", "/api/v1/bins/", nil); err == nil { + res := httptest.NewRecorder() + handler.ServeHTTP(res, req) + err = json.Unmarshal([]byte(res.Body.String()), &bin) + } + return +} + +func createRequest(handler http.Handler, binName string) (request map[string]interface {}, err error){ + if req, err := http.NewRequest("POST", fmt.Sprintf("/bins/%s", binName), bytes.NewBuffer(nil)); err == nil { + res := httptest.NewRecorder() + handler.ServeHTTP(res, req) + err = json.Unmarshal([]byte(res.Body.String()), &request) + } + return +} + diff --git a/src/skimmer/memory.go b/src/skimmer/memory.go index 2575dc4..3b35dd8 100644 --- a/src/skimmer/memory.go +++ b/src/skimmer/memory.go @@ -106,12 +106,12 @@ func (storage *MemoryStorage) LookupRequests(binName string, from int, to int) ( if from > to { from = to } - reversedLen := to - from + reversedLen := len(binRecord.requests) reversed := make([]*Request, reversedLen) - for i, request := range binRecord.requests[from:to] { + for i, request := range binRecord.requests { reversed[reversedLen-i-1] = request } - return reversed, nil + return reversed[from:to], nil } else { return nil, err } diff --git a/src/skimmer/memory_test.go b/src/skimmer/memory_test.go new file mode 100644 index 0000000..ef046e1 --- /dev/null +++ b/src/skimmer/memory_test.go @@ -0,0 +1,205 @@ +package skimmer + +import ( + "testing" + "github.com/stretchr/testify/assert" + "net/http" + "bytes" +) + + +func getMemoryStorage() *MemoryStorage { + return NewMemoryStorage(REQUEST_BODY_SIZE) +} + +func TestNewMemoryStorage(t *testing.T) { + maxRequests := 20 + storage := NewMemoryStorage(maxRequests) + + assert.Equal(t, storage.maxRequests, maxRequests) + assert.NotNil(t, storage.binRecords) +} + +func TestCreateBin(t *testing.T){ + storage := getMemoryStorage() + bin := NewBin() + err := storage.CreateBin(bin) + if assert.Nil(t, err){ + assert.Equal(t, storage.binRecords[bin.Name].bin, bin) + } +} + +func TestUpdateBin(t *testing.T){ + storage := getMemoryStorage() + bin := NewBin() + if err := storage.CreateBin(bin); assert.Nil(t, err) { + if err := storage.UpdateBin(bin); assert.Nil(t, err) { + assert.Equal(t, storage.binRecords[bin.Name].bin, bin) + } + } +} + +func TestGetBinRecord(t *testing.T) { + storage := getMemoryStorage() + + binRecord, err := storage.getBinRecord("test") + assert.NotNil(t, err) + assert.Nil(t, binRecord) + + testBin := NewBin() + testBin.Name = "test" + err = storage.CreateBin(testBin) + if assert.Nil(t, err){ + binRecord, err = storage.getBinRecord("test") + if assert.Nil(t, err) { + assert.Equal(t, binRecord.bin, testBin) + } + } +} + +func TestLookupBin(t *testing.T) { + storage := getMemoryStorage() + + _, err := storage.LookupBin("test") + assert.NotNil(t, err) + + testBin := NewBin() + testBin.Name = "test" + err = storage.CreateBin(testBin) + if assert.Nil(t, err){ + bin, err := storage.LookupBin("test") + if assert.Nil(t, err) { + assert.Equal(t, bin, testBin) + } + } +} + +func TestLookupBins(t *testing.T) { + storage := getMemoryStorage() + + bins, err := storage.LookupBins([]string{"test1", "test2"}) + if assert.Nil(t, err) { + assert.Empty(t, bins) + } + + testBin1 := NewBin() + testBin1.Name = "test1" + err = storage.CreateBin(testBin1) + if assert.Nil(t, err){ + testBin2 := NewBin() + testBin2.Name = "test2" + err = storage.CreateBin(testBin2) + if assert.Nil(t, err){ + bins, err = storage.LookupBins([]string{"test2", "test1"}) + if assert.Nil(t, err) { + assert.Equal(t, bins, []*Bin{testBin2, testBin1}) + } + bins, err = storage.LookupBins([]string{"test1"}) + if assert.Nil(t, err) { + assert.Equal(t, bins, []*Bin{testBin1}) + } + } + } +} + +func TestCreateRequest(t *testing.T) { + storage := NewMemoryStorage(2) + bin := NewBin() + storage.CreateBin(bin) + httpRequest, _ := http.NewRequest("GET", "/", bytes.NewBuffer([]byte("body"))) + request1 := NewRequest(httpRequest, REQUEST_BODY_SIZE) + err := storage.CreateRequest(bin, request1) + if assert.Nil(t, err) { + + binRecord, _ := storage.getBinRecord(bin.Name) + assert.Equal(t, len(binRecord.requests), 1) + assert.Equal(t, binRecord.requests[0], request1) + assert.Equal(t, len(binRecord.requestMap), 1) + assert.Equal(t, binRecord.requestMap[request1.Id], request1) + assert.Equal(t, binRecord.bin.RequestCount, 1) + + request2 := NewRequest(httpRequest, REQUEST_BODY_SIZE) + err = storage.CreateRequest(bin, request2) + if assert.Nil(t, err) { + assert.NotEqual(t, request1, request2) + assert.Equal(t, len(binRecord.requests), 2) + assert.Equal(t, binRecord.requests[1], request2) + assert.Equal(t, len(binRecord.requestMap), 2) + assert.Equal(t, binRecord.requestMap[request2.Id], request2) + assert.Equal(t, binRecord.bin.RequestCount, 2) + + // shrinking + request3 := NewRequest(httpRequest, REQUEST_BODY_SIZE) + err = storage.CreateRequest(bin, request3) + if assert.Nil(t, err) { + assert.Equal(t, len(binRecord.requests), 2) + assert.Equal(t, binRecord.requests[1], request3) + assert.Equal(t, len(binRecord.requestMap), 2) + assert.Equal(t, binRecord.requestMap[request3.Id], request3) + assert.Equal(t, binRecord.bin.RequestCount, 2) + _, err = storage.LookupRequest(bin.Name, request1.Id) + assert.NotNil(t, err) + } + } + } + err = storage.CreateRequest(&Bin{Name:"wrong_name"}, request1) + assert.NotNil(t, err) +} + +func TestLookupRequest(t *testing.T) { + storage := getMemoryStorage() + bin := NewBin() + storage.CreateBin(bin) + httpRequest, _ := http.NewRequest("GET", "/", bytes.NewBuffer([]byte("body"))) + request := NewRequest(httpRequest, REQUEST_BODY_SIZE) + err := storage.CreateRequest(bin, request) + if assert.Nil(t, err) { + tmpRequest, err := storage.LookupRequest(bin.Name, request.Id) + if assert.Nil(t, err) { + assert.Equal(t, request, tmpRequest) + } + } + _, err = storage.LookupRequest(bin.Name, "bad id") + assert.NotNil(t, err) + + _, err = storage.LookupRequest("bad name", request.Id) + assert.NotNil(t, err) +} + +func TestLookupRequests(t *testing.T) { + storage := getMemoryStorage() + _, err := storage.LookupRequests("bad name", 0, 2) + assert.NotNil(t, err) + bin := NewBin() + storage.CreateBin(bin) + requests, err := storage.LookupRequests(bin.Name, 0, 3) + if assert.Nil(t, err) { + assert.Empty(t, requests) + } + + httpRequest, _ := http.NewRequest("GET", "/", bytes.NewBuffer([]byte("body"))) + request1 := NewRequest(httpRequest, REQUEST_BODY_SIZE) + request2 := NewRequest(httpRequest, REQUEST_BODY_SIZE) + request3 := NewRequest(httpRequest, REQUEST_BODY_SIZE) + storage.CreateRequest(bin, request1) + storage.CreateRequest(bin, request2) + storage.CreateRequest(bin, request3) + requests, err = storage.LookupRequests(bin.Name, 1, 3) + if assert.Nil(t, err) { + assert.Equal(t, requests[0], request2) + assert.Equal(t, requests[1], request1) + } + requests, err = storage.LookupRequests(bin.Name, 0, 1) + if assert.Nil(t, err) { + assert.Equal(t, requests[0], request3) + } + requests, err = storage.LookupRequests(bin.Name, -1, 100) + if assert.Nil(t, err) { + assert.Equal(t, len(requests), 3) + } + requests, err = storage.LookupRequests(bin.Name, 1, -2) + if assert.Nil(t, err) { + assert.Equal(t, len(requests), 0) + } + +} diff --git a/src/skimmer/models.go b/src/skimmer/models.go index 3ba21c6..33f55e4 100644 --- a/src/skimmer/models.go +++ b/src/skimmer/models.go @@ -79,7 +79,7 @@ func NewRequest(httpRequest *http.Request, maxBodySize int) *Request { Header: httpRequest.Header, ContentLength: httpRequest.ContentLength, RemoteAddr: httpRequest.RemoteAddr, - RequestURI: httpRequest.RequestURI, + RequestURI: httpRequest.URL.RequestURI(), FormValue: formValue, FormFile: formFile, Body: bodyValue, diff --git a/src/skimmer/models_test.go b/src/skimmer/models_test.go new file mode 100644 index 0000000..4353608 --- /dev/null +++ b/src/skimmer/models_test.go @@ -0,0 +1,78 @@ +package skimmer + +import ( + "github.com/stretchr/testify/assert" + "testing" + "time" + "bytes" + "strings" + "net/http" + "fmt" +) + +func TestRs(t *testing.T) { + pool := "0123456789abcdefghijklmnopqrstuvwxyz" + val := rs.Generate(1000) + for _, alpha := range val { + assert.Contains(t, pool, string(alpha)) + } +} + +func TestNewBin(t *testing.T) { + now := time.Now().Unix() + bin := NewBin() + if assert.NotNil(t, bin) { + assert.Equal(t, len(bin.Name), 6) + assert.Equal(t, bin.RequestCount, 0) + assert.Equal(t, bin.Created, bin.Updated) + assert.True(t, bin.Created < (now+1)) + assert.True(t, bin.Created > (now-1)) + } +} + +func TestNewRequest(t *testing.T) { + body := []byte("arg1=value1&arg2=value2") + httpRequest, err := http.NewRequest("POST", "http://www.google.com/path/", bytes.NewBuffer(body)) + if assert.Nil(t, err){ + httpRequest.Header.Add("Content-Type", "application/x-www-form-urlencoded") + req := NewRequest(httpRequest, 100) + if assert.NotNil(t, req) { + assert.Equal(t, len(req.Id), 12) + assert.Equal(t, req.Method, "POST") + assert.Equal(t, req.Proto, "HTTP/1.1") + assert.Equal(t, req.Host, "www.google.com") + assert.Equal(t, req.ContentLength, len(body)) + assert.Equal(t, req.RequestURI, "/path/") + assert.Equal(t, req.FormValue, map[string][]string{"arg1": {"value1"}, "arg2": {"value2"}}) + assert.Equal(t, req.FormFile, []string(nil)) + assert.Equal(t, req.Body, string(body)) + } + } + httpRequest, err = http.NewRequest("POST", "http://www.google.com/path/", bytes.NewBuffer(body)) + if assert.Nil(t, err){ + httpRequest.Header.Add("Content-Type", "application/x-www-form-urlencoded") + // test truncating + req := NewRequest(httpRequest, 10) + if assert.NotNil(t, req) { + assert.Equal(t, req.ContentLength, len(body)) + assert.Equal(t, req.FormValue, map[string][]string{"arg1": {"value1"}, "arg2": {"value2"}}) + assert.NotEqual(t, req.Body, string(body)) + assert.True(t, strings.HasSuffix(string(req.Body), fmt.Sprintf("\n<<