diff --git a/src/skimmer/api.go b/src/skimmer/api.go index 63595af..91ee574 100644 --- a/src/skimmer/api.go +++ b/src/skimmer/api.go @@ -2,11 +2,10 @@ package skimmer import ( - "fmt" "github.com/codegangsta/martini" "github.com/codegangsta/martini-contrib/render" "net/http" - "net/http/httputil" + "strconv" ) @@ -15,46 +14,87 @@ type ErrorMsg struct{ Error string `json:"error"` } +const ( + REQUEST_BODY_SIZE = 1024 * 30 + MAX_REQUEST_COUNT = 20 +) + func GetApi() *martini.ClassicMartini { - bins := map[string]*Bin{} history := []string{} + memoryStorage := NewMemoryStorage(MAX_REQUEST_COUNT) + api := martini.Classic() api.Use(render.Renderer()) + api.MapTo(memoryStorage, (*Storage)(nil)) - api.Post("/api/v1/bins/", func(r render.Render){ + api.Post("/api/v1/bins/", func(r render.Render, storage Storage){ bin := NewBin() - bins[bin.Name] = bin - history = append(history, bin.Name) - r.JSON(http.StatusCreated, bin) - }) - - api.Get("/api/v1/bins/", func(r render.Render){ - filteredBins := []*Bin{} - for _, name := range(history) { - if bin, ok := bins[name]; ok { - filteredBins = append(filteredBins, bin) - } + if err := storage.CreateBin(bin); err == nil { + history = append(history, bin.Name) + r.JSON(http.StatusCreated, bin) + } else { + r.JSON(http.StatusInternalServerError, ErrorMsg{err.Error()}) } - r.JSON(http.StatusOK, filteredBins) }) - api.Get("/api/v1/bins/:bin", func(r render.Render, params martini.Params){ - if bin, ok := bins[params["bin"]]; ok{ + api.Get("/api/v1/bins/", func(r render.Render, storage Storage){ + if bins, err := storage.LookupBins(history); err == nil { + r.JSON(http.StatusOK, bins) + } else { + r.JSON(http.StatusInternalServerError, ErrorMsg{err.Error()}) + } + }) + + api.Get("/api/v1/bins/:bin", func(r render.Render, params martini.Params, storage Storage){ + if bin, err := storage.LookupBin(params["bin"]); err == nil{ r.JSON(http.StatusOK, bin) + } else { + r.JSON(http.StatusNotFound, ErrorMsg{err.Error()}) + } + }) + + api.Get("/api/v1/bins/:bin/requests/", func(r render.Render, storage Storage, params martini.Params, + req *http.Request){ + if bin, error := storage.LookupBin(params["bin"]); error == nil { + from := 0 + to := 20 + if fromVal, err := strconv.Atoi(req.FormValue("from")); err == nil { + from = fromVal + } + if toVal, err := strconv.Atoi(req.FormValue("to")); err == nil { + to = toVal + } + if requests, err := storage.LookupRequests(bin.Name, from, to); err == nil { + r.JSON(http.StatusOK, requests) + } else { + r.JSON(http.StatusInternalServerError, ErrorMsg{err.Error()}) + } } else { r.Error(http.StatusNotFound) } }) - api.Any("/", func(res http.ResponseWriter, req *http.Request) { - if dumped, err := httputil.DumpRequest(req, true); err == nil { - res.WriteHeader(http.StatusOK) - res.Write(dumped) + api.Get("/api/v1/bins/:bin/requests/:request", func(r render.Render, storage Storage, params martini.Params){ + if request, err := storage.LookupRequest(params["bin"], params["request"]); err == nil { + r.JSON(http.StatusOK, request) } else { - res.WriteHeader(http.StatusInternalServerError) - fmt.Fprintf(res, "Error: %v", err) + r.JSON(http.StatusNotFound, ErrorMsg{err.Error()}) + } + }) + + api.Any("/bins/:name", func(r render.Render, storage Storage, params martini.Params, + req *http.Request, res http.ResponseWriter){ + if bin, error := storage.LookupBin(params["name"]); error == nil { + request := NewRequest(req, REQUEST_BODY_SIZE) + if err := storage.CreateRequest(bin, request); err == nil { + r.JSON(http.StatusOK, request) + } else { + r.JSON(http.StatusInternalServerError, ErrorMsg{err.Error()}) + } + } else { + r.Error(http.StatusNotFound) } }) return api diff --git a/src/skimmer/memory.go b/src/skimmer/memory.go new file mode 100644 index 0000000..2575dc4 --- /dev/null +++ b/src/skimmer/memory.go @@ -0,0 +1,132 @@ +package skimmer + +import ( + "errors" + "sync" +) + +type MemoryStorage struct { + BaseStorage + sync.RWMutex + binRecords map[string]*BinRecord +} + +type BinRecord struct { + bin *Bin + requests []*Request + requestMap map[string]*Request +} + +func (binRecord *BinRecord) ShrinkRequests(size int) { + if size > 0 && len(binRecord.requests) > size { + requests := binRecord.requests + lenDiff := len(requests) - size + removed := requests[:lenDiff] + for _, removedReq := range removed { + delete(binRecord.requestMap, removedReq.Id) + } + requests = requests[lenDiff:] + binRecord.requests = requests + } +} + +func NewMemoryStorage(maxRequests int) *MemoryStorage { + return &MemoryStorage{ + BaseStorage{ + maxRequests: maxRequests, + }, + sync.RWMutex{}, + map[string]*BinRecord{}, + } +} + +func (storage *MemoryStorage) getBinRecord(name string) (*BinRecord, error) { + storage.RLock() + defer storage.RUnlock() + if binRecord, ok := storage.binRecords[name]; ok { + return binRecord, nil + } + return nil, errors.New("Bin not found") +} + +func (storage *MemoryStorage) LookupBin(name string) (*Bin, error) { + if binRecord, err := storage.getBinRecord(name); err == nil { + return binRecord.bin, nil + } else { + return nil, err + } +} + +func (storage *MemoryStorage) LookupBins(names []string) ([]*Bin, error) { + bins := []*Bin{} + for _, name := range names { + if binRecord, err := storage.getBinRecord(name); err == nil { + bins = append(bins, binRecord.bin) + } + } + return bins, nil +} + +func (storage *MemoryStorage) CreateBin(bin *Bin) error { + storage.Lock() + defer storage.Unlock() + binRec := BinRecord{bin, []*Request{}, map[string]*Request{}} + storage.binRecords[bin.Name] = &binRec + return nil +} + +func (storage *MemoryStorage) UpdateBin(_ *Bin) error { + return nil +} + +func (storage *MemoryStorage) LookupRequest(binName, id string) (*Request, error) { + if binRecord, err := storage.getBinRecord(binName); err == nil { + if request, ok := binRecord.requestMap[id]; ok { + return request, nil + } else { + return nil, errors.New("Request not found") + } + } else { + return nil, err + } +} + +func (storage *MemoryStorage) LookupRequests(binName string, from int, to int) ([]*Request, error) { + if binRecord, err := storage.getBinRecord(binName); err == nil { + requestLen := len(binRecord.requests) + if to >= requestLen { + to = requestLen + } + if to < 0 { + to = 0 + } + if from < 0 { + from = 0 + } + if from > to { + from = to + } + reversedLen := to - from + reversed := make([]*Request, reversedLen) + for i, request := range binRecord.requests[from:to] { + reversed[reversedLen-i-1] = request + } + return reversed, nil + } else { + return nil, err + } +} + +func (storage *MemoryStorage) CreateRequest(bin *Bin, req *Request) error { + if binRecord, err := storage.getBinRecord(bin.Name); err == nil { + storage.Lock() + defer storage.Unlock() + binRecord.requests = append(binRecord.requests, req) + binRecord.requestMap[req.Id] = req + binRecord.ShrinkRequests(storage.maxRequests) + binRecord.bin.RequestCount = len(binRecord.requests) + return nil + } else { + return err + } +} diff --git a/src/skimmer/models.go b/src/skimmer/models.go index 3fae090..3ba21c6 100644 --- a/src/skimmer/models.go +++ b/src/skimmer/models.go @@ -1,6 +1,12 @@ package skimmer -import "time" +import ( + "bytes" + "fmt" + "io/ioutil" + "net/http" + "time" +) var rs = NewRandomString("0123456789abcdefghijklmnopqrstuvwxyz") @@ -20,3 +26,63 @@ func NewBin() *Bin { } return &bin } + +type Request struct { + Id string `json:"id"` + Created int64 `json:"created"` + + Method string `json:"method"` // GET, POST, PUT, etc. + Proto string `json:"proto"` // "HTTP/1.0" + Header http.Header `json:"header"` + ContentLength int64 `json:"contentLength"` + RemoteAddr string `json:"remoteAddr"` + Host string `json:"host"` + RequestURI string `json:"requestURI"` + Body string `json:"body"` + FormValue map[string][]string `json:"formValue"` + FormFile []string `json:"formFile"` +} + +func NewRequest(httpRequest *http.Request, maxBodySize int) *Request { + var ( + bodyValue string + formValue map[string][]string + formFile []string + ) + if body, err := ioutil.ReadAll(httpRequest.Body); err == nil { + if len(body) > 0 && maxBodySize != 0 { + if maxBodySize == -1 || httpRequest.ContentLength < int64(maxBodySize) { + bodyValue = string(body) + } else { + bodyValue = fmt.Sprintf("%s\n<<