diff options
Diffstat (limited to 'Godeps/_workspace/src/gopkg.in')
44 files changed, 2494 insertions, 57 deletions
diff --git a/Godeps/_workspace/src/gopkg.in/fsnotify.v1/.travis.yml b/Godeps/_workspace/src/gopkg.in/fsnotify.v1/.travis.yml index 67467e140..1b5151f12 100644 --- a/Godeps/_workspace/src/gopkg.in/fsnotify.v1/.travis.yml +++ b/Godeps/_workspace/src/gopkg.in/fsnotify.v1/.travis.yml @@ -2,10 +2,15 @@ sudo: false language: go go: - - 1.4.1 + - 1.5.1 before_script: - - FIXED=$(go fmt ./... | wc -l); if [ $FIXED -gt 0 ]; then echo "gofmt - $FIXED file(s) not formatted correctly, please run gofmt to fix this." && exit 1; fi + - go get -u github.com/golang/lint/golint + +after_script: + - test -z "$(gofmt -s -l -w . | tee /dev/stderr)" + - test -z "$(golint ./... | tee /dev/stderr)" + - go vet ./... os: - linux diff --git a/Godeps/_workspace/src/gopkg.in/fsnotify.v1/AUTHORS b/Godeps/_workspace/src/gopkg.in/fsnotify.v1/AUTHORS index 4e0e8284e..763b853c3 100644 --- a/Godeps/_workspace/src/gopkg.in/fsnotify.v1/AUTHORS +++ b/Godeps/_workspace/src/gopkg.in/fsnotify.v1/AUTHORS @@ -11,18 +11,22 @@ Adrien Bustany <adrien@bustany.org> Caleb Spare <cespare@gmail.com> Case Nelson <case@teammating.com> -Chris Howey <howeyc@gmail.com> <chris@howey.me> +Chris Howey <chris@howey.me> <howeyc@gmail.com> Christoffer Buchholz <christoffer.buchholz@gmail.com> +Daniel Wagner-Hall <dawagner@gmail.com> Dave Cheney <dave@cheney.net> +Evan Phoenix <evan@fallingsnow.net> Francisco Souza <f@souza.cc> Hari haran <hariharan.uno@gmail.com> John C Barstow Kelvin Fo <vmirage@gmail.com> +Ken-ichirou MATSUZAWA <chamas@h4.dion.ne.jp> Matt Layher <mdlayher@gmail.com> Nathan Youngman <git@nathany.com> Paul Hammond <paul@paulhammond.org> Pieter Droogendijk <pieter@binky.org.uk> Pursuit92 <JoshChase@techpursuit.net> +Riku Voipio <riku.voipio@linaro.org> Rob Figueiredo <robfig@gmail.com> Soge Zhang <zhssoge@gmail.com> Tilak Sharma <tilaks@google.com> @@ -32,3 +36,4 @@ Yukang <moorekang@gmail.com> bronze1man <bronze1man@gmail.com> debrando <denis.brandolini@gmail.com> henrikedwards <henrik.edwards@gmail.com> +铁哥 <guotie.9@gmail.com> diff --git a/Godeps/_workspace/src/gopkg.in/fsnotify.v1/CHANGELOG.md b/Godeps/_workspace/src/gopkg.in/fsnotify.v1/CHANGELOG.md index ea9428a2a..4e6672702 100644 --- a/Godeps/_workspace/src/gopkg.in/fsnotify.v1/CHANGELOG.md +++ b/Godeps/_workspace/src/gopkg.in/fsnotify.v1/CHANGELOG.md @@ -1,5 +1,16 @@ # Changelog +## v1.2.5 / 2015-10-17 + +* inotify: use epoll_create1 for arm64 support (requires Linux 2.6.27 or later) [#100](https://github.com/go-fsnotify/fsnotify/pull/100) (thanks @suihkulokki) +* inotify: fix path leaks [#73](https://github.com/go-fsnotify/fsnotify/pull/73) (thanks @chamaken) +* kqueue: watch for rename events on subdirectories [#83](https://github.com/go-fsnotify/fsnotify/pull/83) (thanks @guotie) +* kqueue: avoid infinite loops from symlinks cycles [#101](https://github.com/go-fsnotify/fsnotify/pull/101) (thanks @illicitonion) + +## v1.2.1 / 2015-10-14 + +* kqueue: don't watch named pipes [#98](https://github.com/go-fsnotify/fsnotify/pull/98) (thanks @evanphx) + ## v1.2.0 / 2015-02-08 * inotify: use epoll to wake up readEvents [#66](https://github.com/go-fsnotify/fsnotify/pull/66) (thanks @PieterD) diff --git a/Godeps/_workspace/src/gopkg.in/fsnotify.v1/NotUsed.xcworkspace b/Godeps/_workspace/src/gopkg.in/fsnotify.v1/NotUsed.xcworkspace deleted file mode 100644 index e69de29bb..000000000 --- a/Godeps/_workspace/src/gopkg.in/fsnotify.v1/NotUsed.xcworkspace +++ /dev/null diff --git a/Godeps/_workspace/src/gopkg.in/fsnotify.v1/README.md b/Godeps/_workspace/src/gopkg.in/fsnotify.v1/README.md index 7a0b24736..f2b432e96 100644 --- a/Godeps/_workspace/src/gopkg.in/fsnotify.v1/README.md +++ b/Godeps/_workspace/src/gopkg.in/fsnotify.v1/README.md @@ -1,6 +1,6 @@ # File system notifications for Go -[![Coverage](http://gocover.io/_badge/github.com/go-fsnotify/fsnotify)](http://gocover.io/github.com/go-fsnotify/fsnotify) [![GoDoc](https://godoc.org/gopkg.in/fsnotify.v1?status.svg)](https://godoc.org/gopkg.in/fsnotify.v1) +[![GoDoc](https://godoc.org/gopkg.in/fsnotify.v1?status.svg)](https://godoc.org/gopkg.in/fsnotify.v1) [![Coverage](http://gocover.io/_badge/github.com/go-fsnotify/fsnotify)](http://gocover.io/github.com/go-fsnotify/fsnotify) Go 1.3+ required. @@ -8,8 +8,8 @@ Cross platform: Windows, Linux, BSD and OS X. |Adapter |OS |Status | |----------|----------|----------| -|inotify |Linux, Android\*|Supported [![Build Status](https://travis-ci.org/go-fsnotify/fsnotify.svg?branch=master)](https://travis-ci.org/go-fsnotify/fsnotify)| -|kqueue |BSD, OS X, iOS\*|Supported [![Circle CI](https://circleci.com/gh/go-fsnotify/fsnotify.svg?style=svg)](https://circleci.com/gh/go-fsnotify/fsnotify)| +|inotify |Linux 2.6.27 or later, Android\*|Supported [![Build Status](https://travis-ci.org/go-fsnotify/fsnotify.svg?branch=master)](https://travis-ci.org/go-fsnotify/fsnotify)| +|kqueue |BSD, OS X, iOS\*|Supported [![Build Status](https://travis-ci.org/go-fsnotify/fsnotify.svg?branch=master)](https://travis-ci.org/go-fsnotify/fsnotify)| |ReadDirectoryChangesW|Windows|Supported [![Build status](https://ci.appveyor.com/api/projects/status/ivwjubaih4r0udeh/branch/master?svg=true)](https://ci.appveyor.com/project/NathanYoungman/fsnotify/branch/master)| |FSEvents |OS X |[Planned](https://github.com/go-fsnotify/fsnotify/issues/11)| |FEN |Solaris 11 |[Planned](https://github.com/go-fsnotify/fsnotify/issues/12)| @@ -55,5 +55,10 @@ Please refer to [CONTRIBUTING][] before opening an issue or pull request. See [example_test.go](https://github.com/go-fsnotify/fsnotify/blob/master/example_test.go). - [contributing]: https://github.com/go-fsnotify/fsnotify/blob/master/CONTRIBUTING.md + +## Related Projects + +* [notify](https://github.com/rjeczalik/notify) +* [fsevents](https://github.com/go-fsnotify/fsevents) + diff --git a/Godeps/_workspace/src/gopkg.in/fsnotify.v1/circle.yml b/Godeps/_workspace/src/gopkg.in/fsnotify.v1/circle.yml deleted file mode 100644 index 204217fb0..000000000 --- a/Godeps/_workspace/src/gopkg.in/fsnotify.v1/circle.yml +++ /dev/null @@ -1,26 +0,0 @@ -## OS X build (CircleCI iOS beta) - -# Pretend like it's an Xcode project, at least to get it running. -machine: - environment: - XCODE_WORKSPACE: NotUsed.xcworkspace - XCODE_SCHEME: NotUsed - # This is where the go project is actually checked out to: - CIRCLE_BUILD_DIR: $HOME/.go_project/src/github.com/go-fsnotify/fsnotify - -dependencies: - pre: - - brew upgrade go - -test: - override: - - go test ./... - -# Idealized future config, eventually with cross-platform build matrix :-) - -# machine: -# go: -# version: 1.4 -# os: -# - osx -# - linux diff --git a/Godeps/_workspace/src/gopkg.in/fsnotify.v1/inotify.go b/Godeps/_workspace/src/gopkg.in/fsnotify.v1/inotify.go index d7759ec8c..06f4bba88 100644 --- a/Godeps/_workspace/src/gopkg.in/fsnotify.v1/inotify.go +++ b/Godeps/_workspace/src/gopkg.in/fsnotify.v1/inotify.go @@ -23,6 +23,7 @@ type Watcher struct { Events chan Event Errors chan error mu sync.Mutex // Map access + cv *sync.Cond // sync removing on rm_watch with IN_IGNORE fd int poller *fdPoller watches map[string]*watch // Map of inotify watches (key: path) @@ -54,6 +55,7 @@ func NewWatcher() (*Watcher, error) { done: make(chan struct{}), doneResp: make(chan struct{}), } + w.cv = sync.NewCond(&w.mu) go w.readEvents() return w, nil @@ -134,8 +136,10 @@ func (w *Watcher) Remove(name string) error { } // inotify_rm_watch will return EINVAL if the file has been deleted; // the inotify will already have been removed. - // That means we can safely delete it from our watches, whatever inotify_rm_watch does. - delete(w.watches, name) + // watches and pathes are deleted in ignoreLinux() implicitly and asynchronously + // by calling inotify_rm_watch() below. e.g. readEvents() goroutine receives IN_IGNORE + // so that EINVAL means that the wd is being rm_watch()ed or its file removed + // by another thread and we have not received IN_IGNORE event. success, errno := syscall.InotifyRmWatch(w.fd, watch.wd) if success == -1 { // TODO: Perhaps it's not helpful to return an error here in every case. @@ -146,6 +150,14 @@ func (w *Watcher) Remove(name string) error { // explicitly by inotify_rm_watch, implicitly when the file they are watching is deleted. return errno } + + // wait until ignoreLinux() deleting maps + exists := true + for exists { + w.cv.Wait() + _, exists = w.watches[name] + } + return nil } @@ -249,7 +261,7 @@ func (w *Watcher) readEvents() { event := newEvent(name, mask) // Send the events that are not ignored on the events channel - if !event.ignoreLinux(mask) { + if !event.ignoreLinux(w, raw.Wd, mask) { select { case w.Events <- event: case <-w.done: @@ -266,9 +278,15 @@ func (w *Watcher) readEvents() { // Certain types of events can be "ignored" and not sent over the Events // channel. Such as events marked ignore by the kernel, or MODIFY events // against files that do not exist. -func (e *Event) ignoreLinux(mask uint32) bool { +func (e *Event) ignoreLinux(w *Watcher, wd int32, mask uint32) bool { // Ignore anything the inotify API says to ignore if mask&syscall.IN_IGNORED == syscall.IN_IGNORED { + w.mu.Lock() + defer w.mu.Unlock() + name := w.paths[int(wd)] + delete(w.paths, int(wd)) + delete(w.watches, name) + w.cv.Broadcast() return true } diff --git a/Godeps/_workspace/src/gopkg.in/fsnotify.v1/inotify_poller.go b/Godeps/_workspace/src/gopkg.in/fsnotify.v1/inotify_poller.go index 3b4178404..23a5ca146 100644 --- a/Godeps/_workspace/src/gopkg.in/fsnotify.v1/inotify_poller.go +++ b/Godeps/_workspace/src/gopkg.in/fsnotify.v1/inotify_poller.go @@ -39,7 +39,7 @@ func newFdPoller(fd int) (*fdPoller, error) { poller.fd = fd // Create epoll fd - poller.epfd, errno = syscall.EpollCreate(1) + poller.epfd, errno = syscall.EpollCreate1(0) if poller.epfd == -1 { return nil, errno } diff --git a/Godeps/_workspace/src/gopkg.in/fsnotify.v1/inotify_test.go b/Godeps/_workspace/src/gopkg.in/fsnotify.v1/inotify_test.go index 035ee8f95..4b5c4b184 100644 --- a/Godeps/_workspace/src/gopkg.in/fsnotify.v1/inotify_test.go +++ b/Godeps/_workspace/src/gopkg.in/fsnotify.v1/inotify_test.go @@ -7,6 +7,7 @@ package fsnotify import ( + "fmt" "os" "path/filepath" "syscall" @@ -281,12 +282,60 @@ func TestInotifyRemoveTwice(t *testing.T) { } err = w.Remove(testFile) - if err != syscall.EINVAL { - t.Fatalf("Expected EINVAL from Remove, got: %v", err) + if err == nil { + t.Fatalf("no error on removing invalid file") } + s1 := fmt.Sprintf("%s", err) err = w.Remove(testFile) - if err == syscall.EINVAL { - t.Fatalf("Got EINVAL again, watch was not removed") + if err == nil { + t.Fatalf("no error on removing invalid file") + } + s2 := fmt.Sprintf("%s", err) + + if s1 != s2 { + t.Fatalf("receive different error - %s / %s", s1, s2) + } +} + +func TestInotifyInnerMapLength(t *testing.T) { + testDir := tempMkdir(t) + defer os.RemoveAll(testDir) + testFile := filepath.Join(testDir, "testfile") + + handle, err := os.Create(testFile) + if err != nil { + t.Fatalf("Create failed: %v", err) + } + handle.Close() + + w, err := NewWatcher() + if err != nil { + t.Fatalf("Failed to create watcher: %v", err) + } + defer w.Close() + + err = w.Add(testFile) + if err != nil { + t.Fatalf("Failed to add testFile: %v", err) + } + go func() { + for err := range w.Errors { + t.Fatalf("error received: %s", err) + } + }() + + err = os.Remove(testFile) + if err != nil { + t.Fatalf("Failed to remove testFile: %v", err) + } + _ = <-w.Events // consume Remove event + <-time.After(50 * time.Millisecond) // wait IN_IGNORE propagated + + if len(w.watches) != 0 { + t.Fatalf("Expected watches len is 0, but got: %d, %v", len(w.watches), w.watches) + } + if len(w.paths) != 0 { + t.Fatalf("Expected paths len is 0, but got: %d, %v", len(w.paths), w.paths) } } diff --git a/Godeps/_workspace/src/gopkg.in/fsnotify.v1/integration_test.go b/Godeps/_workspace/src/gopkg.in/fsnotify.v1/integration_test.go index 59169c6af..49e48ff1c 100644 --- a/Godeps/_workspace/src/gopkg.in/fsnotify.v1/integration_test.go +++ b/Godeps/_workspace/src/gopkg.in/fsnotify.v1/integration_test.go @@ -10,6 +10,7 @@ import ( "io/ioutil" "os" "os/exec" + "path" "path/filepath" "runtime" "sync/atomic" @@ -1065,6 +1066,53 @@ func TestFsnotifyFakeSymlink(t *testing.T) { watcher.Close() } +func TestCyclicSymlink(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("symlinks don't work on Windows.") + } + + watcher := newWatcher(t) + + testDir := tempMkdir(t) + defer os.RemoveAll(testDir) + + link := path.Join(testDir, "link") + if err := os.Symlink(".", link); err != nil { + t.Fatalf("could not make symlink: %v", err) + } + addWatch(t, watcher, testDir) + + var createEventsReceived counter + go func() { + for ev := range watcher.Events { + if ev.Op&Create == Create { + createEventsReceived.increment() + } + } + }() + + if err := os.Remove(link); err != nil { + t.Fatalf("Error removing link: %v", err) + } + + // It would be nice to be able to expect a delete event here, but kqueue has + // no way for us to get events on symlinks themselves, because opening them + // opens an fd to the file to which they point. + + if err := ioutil.WriteFile(link, []byte("foo"), 0700); err != nil { + t.Fatalf("could not make symlink: %v", err) + } + + // We expect this event to be received almost immediately, but let's wait 500 ms to be sure + time.Sleep(500 * time.Millisecond) + + if got := createEventsReceived.value(); got == 0 { + t.Errorf("want at least 1 create event got %v", got) + } + + watcher.Close() +} + // TestConcurrentRemovalOfWatch tests that concurrent calls to RemoveWatch do not race. // See https://codereview.appspot.com/103300045/ // go test -test.run=TestConcurrentRemovalOfWatch -test.cpu=1,1,1,1,1 -race diff --git a/Godeps/_workspace/src/gopkg.in/fsnotify.v1/kqueue.go b/Godeps/_workspace/src/gopkg.in/fsnotify.v1/kqueue.go index 265622d20..9662a50a8 100644 --- a/Godeps/_workspace/src/gopkg.in/fsnotify.v1/kqueue.go +++ b/Godeps/_workspace/src/gopkg.in/fsnotify.v1/kqueue.go @@ -94,7 +94,8 @@ func (w *Watcher) Add(name string) error { w.mu.Lock() w.externalWatches[name] = true w.mu.Unlock() - return w.addWatch(name, noteAllEvents) + _, err := w.addWatch(name, noteAllEvents) + return err } // Remove stops watching the the named file or directory (non-recursively). @@ -153,7 +154,8 @@ var keventWaitTime = durationToTimespec(100 * time.Millisecond) // addWatch adds name to the watched file set. // The flags are interpreted as described in kevent(2). -func (w *Watcher) addWatch(name string, flags uint32) error { +// Returns the real path to the file which was added, if any, which may be different from the one passed in the case of symlinks. +func (w *Watcher) addWatch(name string, flags uint32) (string, error) { var isDir bool // Make ./name and name equivalent name = filepath.Clean(name) @@ -161,7 +163,7 @@ func (w *Watcher) addWatch(name string, flags uint32) error { w.mu.Lock() if w.isClosed { w.mu.Unlock() - return errors.New("kevent instance already closed") + return "", errors.New("kevent instance already closed") } watchfd, alreadyWatching := w.watches[name] // We already have a watch, but we can still override flags. @@ -173,12 +175,17 @@ func (w *Watcher) addWatch(name string, flags uint32) error { if !alreadyWatching { fi, err := os.Lstat(name) if err != nil { - return err + return "", err } // Don't watch sockets. if fi.Mode()&os.ModeSocket == os.ModeSocket { - return nil + return "", nil + } + + // Don't watch named pipes. + if fi.Mode()&os.ModeNamedPipe == os.ModeNamedPipe { + return "", nil } // Follow Symlinks @@ -190,18 +197,26 @@ func (w *Watcher) addWatch(name string, flags uint32) error { if fi.Mode()&os.ModeSymlink == os.ModeSymlink { name, err = filepath.EvalSymlinks(name) if err != nil { - return nil + return "", nil + } + + w.mu.Lock() + _, alreadyWatching = w.watches[name] + w.mu.Unlock() + + if alreadyWatching { + return name, nil } fi, err = os.Lstat(name) if err != nil { - return nil + return "", nil } } watchfd, err = syscall.Open(name, openMode, 0700) if watchfd == -1 { - return err + return "", err } isDir = fi.IsDir() @@ -210,7 +225,7 @@ func (w *Watcher) addWatch(name string, flags uint32) error { const registerAdd = syscall.EV_ADD | syscall.EV_CLEAR | syscall.EV_ENABLE if err := register(w.kq, []int{watchfd}, registerAdd, flags); err != nil { syscall.Close(watchfd) - return err + return "", err } if !alreadyWatching { @@ -224,6 +239,7 @@ func (w *Watcher) addWatch(name string, flags uint32) error { // Watch the directory if it has not been watched before, // or if it was watched before, but perhaps only a NOTE_DELETE (watchDirectoryFiles) w.mu.Lock() + watchDir := (flags&syscall.NOTE_WRITE) == syscall.NOTE_WRITE && (!alreadyWatching || (w.dirFlags[name]&syscall.NOTE_WRITE) != syscall.NOTE_WRITE) // Store flags so this watch can be updated later @@ -232,11 +248,11 @@ func (w *Watcher) addWatch(name string, flags uint32) error { if watchDir { if err := w.watchDirectoryFiles(name); err != nil { - return err + return "", err } } } - return nil + return name, nil } // readEvents reads from kqueue and converts the received kevents into @@ -359,7 +375,8 @@ func (w *Watcher) watchDirectoryFiles(dirPath string) error { for _, fileInfo := range files { filePath := filepath.Join(dirPath, fileInfo.Name()) - if err := w.internalWatch(filePath, fileInfo); err != nil { + filePath, err = w.internalWatch(filePath, fileInfo) + if err != nil { return err } @@ -394,7 +411,8 @@ func (w *Watcher) sendDirectoryChangeEvents(dirPath string) { } // like watchDirectoryFiles (but without doing another ReadDir) - if err := w.internalWatch(filePath, fileInfo); err != nil { + filePath, err = w.internalWatch(filePath, fileInfo) + if err != nil { return } @@ -404,7 +422,7 @@ func (w *Watcher) sendDirectoryChangeEvents(dirPath string) { } } -func (w *Watcher) internalWatch(name string, fileInfo os.FileInfo) error { +func (w *Watcher) internalWatch(name string, fileInfo os.FileInfo) (string, error) { if fileInfo.IsDir() { // mimic Linux providing delete events for subdirectories // but preserve the flags used if currently watching subdirectory @@ -412,7 +430,7 @@ func (w *Watcher) internalWatch(name string, fileInfo os.FileInfo) error { flags := w.dirFlags[name] w.mu.Unlock() - flags |= syscall.NOTE_DELETE + flags |= syscall.NOTE_DELETE | syscall.NOTE_RENAME return w.addWatch(name, flags) } diff --git a/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/.gitignore b/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/.gitignore new file mode 100644 index 000000000..c2a6499b4 --- /dev/null +++ b/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/.gitignore @@ -0,0 +1,10 @@ +.DS_Store +*.swp +*.swo +*.test +examples/interval/interval +examples/interval-vary/interval-vary +examples/interval-many/interval-many +examples/memstats/memstats +examples/rate-limit/rate-limit +examples/custom/custom diff --git a/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/.travis.yml b/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/.travis.yml new file mode 100644 index 000000000..1b2427202 --- /dev/null +++ b/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/.travis.yml @@ -0,0 +1,10 @@ +sudo: false +language: go + +go: + - 1.2 + - tip + +install: go get -t ./... + +script: go test -v -short ./... diff --git a/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/LICENSE b/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/LICENSE new file mode 100644 index 000000000..f9616483e --- /dev/null +++ b/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/LICENSE @@ -0,0 +1,12 @@ +Copyright (c) 2014, Martin Angers and Contributors. +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + +* Neither the name of the author nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/README.md b/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/README.md new file mode 100644 index 000000000..fbb4a01fc --- /dev/null +++ b/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/README.md @@ -0,0 +1,82 @@ +# Throttled [![build status](https://secure.travis-ci.org/throttled/throttled.png)](http://travis-ci.org/throttled/throttled) [![GoDoc](https://godoc.org/gopkg.in/throttled/throttled.v1?status.png)](http://godoc.org/gopkg.in/throttled/throttled.v1) + +Package throttled implements different throttling strategies for controlling +access to HTTP handlers. + +*As of July 27, 2015, the package is now located under its own GitHub + organization and uses gopkg.in for versioning, please adjust your + imports to `gopkg.in/throttled/throttled.v1`.* + +## Installation + +`go get gopkg.in/throttled/throttled.v1/...` + +## Interval + +The Interval function creates a throttler that allows requests to go through at +a controlled, constant interval. The interval may be applied to all requests +(vary argument == nil) or independently based on vary-by criteria. + +For example: + + th := throttled.Interval(throttled.PerSec(10), 100, &throttled.VaryBy{Path: true}, 50) + h := th.Throttle(myHandler) + http.ListenAndServe(":9000", h) + +Creates a throttler that will allow a request each 100ms (10 requests per second), with +a buffer of 100 exceeding requests before dropping requests with a status code 429 (by +default, configurable using th.DeniedHandler or the package-global DefaultDeniedHandler +variable). Different paths will be throttled independently, so that /path_a and /path_b +both can serve 10 requests per second. The last argument, 50, indicates the maximum number +of keys that the throttler will keep in memory. + +## MemStats + +The MemStats function creates a throttler that allows requests to go through only if +the memory statistics of the current process are below specified thresholds. + +For example: + + th := throttled.MemStats(throttled.MemThresholds(&runtime.MemStats{NumGC: 10}, 10*time.Millisecond) + h := th.Throttle(myHandler) + http.ListenAndServe(":9000", h) + +Creates a throttler that will allow requests to go through until the number of garbage +collections reaches the initial number + 10 (the MemThresholds function creates absolute +memory stats thresholds from offsets). The second argument, 10ms, indicates the refresh +rate of the memory stats. + +## RateLimit + +The RateLimit function creates a throttler that allows a certain number of requests in +a given time window, as is often implemented in public RESTful APIs. + +For example: + + th := throttled.RateLimit(throttled.PerMin(30), &throttled.VaryBy{RemoteAddr: true}, store.NewMemStore(1000)) + h := th.Throttle(myHandler) + http.ListenAndServe(":9000", h) + +Creates a throttler that will limit requests to 30 per minute, based on the remote address +of the client, and will store the counter and remaining time of the current window in the +provided memory store, limiting the number of keys to keep in memory to 1000. The store +sub-package also provides a Redis-based Store implementations. + +The RateLimit throttler sets the expected X-RateLimit-* headers on the response, and +also sets a Retry-After header when the limit is exceeded. + +## Documentation + +The API documentation is available as usual on [godoc.org][doc]. + +There is also a [blog post explaining the package's usage on 0value.com][blog]. + +Finally, many examples are provided in the /examples sub-folder of the repository. + +## License + +The [BSD 3-clause license][bsd]. Copyright (c) 2014 Martin Angers and Contributors. + +[doc]: http://godoc.org/gopkg.in/throttled/throttled.v1 +[blog]: http://0value.com/throttled--guardian-of-the-web-server +[bsd]: http://opensource.org/licenses/BSD-3-Clause diff --git a/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/common_test.go b/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/common_test.go new file mode 100644 index 000000000..ddb57fb1c --- /dev/null +++ b/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/common_test.go @@ -0,0 +1,65 @@ +package throttled + +import ( + "fmt" + "net/http" + "net/http/httptest" + "sync" + "time" + + "github.com/PuerkitoBio/boom/commands" +) + +type stats struct { + sync.Mutex + ok int + dropped int + ts []time.Time + + body func() +} + +func (s *stats) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if s.body != nil { + s.body() + } + s.Lock() + defer s.Unlock() + s.ts = append(s.ts, time.Now()) + s.ok++ + w.WriteHeader(200) +} + +func (s *stats) DeniedHTTP(w http.ResponseWriter, r *http.Request) { + s.Lock() + defer s.Unlock() + s.dropped++ + w.WriteHeader(deniedStatus) +} + +func (s *stats) Stats() (int, int, []time.Time) { + s.Lock() + defer s.Unlock() + return s.ok, s.dropped, s.ts +} + +func runTest(h http.Handler, b ...commands.Boom) []*commands.Report { + srv := httptest.NewServer(h) + defer srv.Close() + + var rpts []*commands.Report + var wg sync.WaitGroup + var mu sync.Mutex + wg.Add(len(b)) + for i, bo := range b { + bo.Req.Url = srv.URL + fmt.Sprintf("/%d", i) + go func(bo commands.Boom) { + mu.Lock() + defer mu.Unlock() + rpts = append(rpts, bo.Run()) + wg.Done() + }(bo) + } + wg.Wait() + return rpts +} diff --git a/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/delayer.go b/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/delayer.go new file mode 100644 index 000000000..e62ec9e86 --- /dev/null +++ b/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/delayer.go @@ -0,0 +1,109 @@ +package throttled + +import "time" + +// The Quota interface defines the method to implement to describe +// a time-window quota, as required by the RateLimit throttler. +type Quota interface { + // Quota returns a number of requests allowed, and a duration. + Quota() (int, time.Duration) +} + +// The Delayer interface defines the method to implement to describe +// a delay as required by the Interval throttler. +type Delayer interface { + // Delay returns a duration. + Delay() time.Duration +} + +// PerSec represents a number of requests per second. +type PerSec int + +// Delay returns the duration to wait before the next request can go through, +// so that PerSec(n) == n requests per second at regular intervals. +func (ps PerSec) Delay() time.Duration { + if ps <= 0 { + return 0 + } + return time.Duration(1.0 / float64(ps) * float64(time.Second)) +} + +// Quota returns the number of requests allowed in a 1 second time window, +// so that PerSec(n) == n requests allowed per second. +func (ps PerSec) Quota() (int, time.Duration) { + return int(ps), time.Second +} + +// PerMin represents a number of requests per minute. +type PerMin int + +// Delay returns the duration to wait before the next request can go through, +// so that PerMin(n) == n requests per minute at regular intervals. +func (pm PerMin) Delay() time.Duration { + if pm <= 0 { + return 0 + } + return time.Duration(1.0 / float64(pm) * float64(time.Minute)) +} + +// Quota returns the number of requests allowed in a 1 minute time window, +// so that PerMin(n) == n requests allowed per minute. +func (pm PerMin) Quota() (int, time.Duration) { + return int(pm), time.Minute +} + +// PerHour represents a number of requests per hour. +type PerHour int + +// Delay returns the duration to wait before the next request can go through, +// so that PerHour(n) == n requests per hour at regular intervals. +func (ph PerHour) Delay() time.Duration { + if ph <= 0 { + return 0 + } + return time.Duration(1.0 / float64(ph) * float64(time.Hour)) +} + +// Quota returns the number of requests allowed in a 1 hour time window, +// so that PerHour(n) == n requests allowed per hour. +func (ph PerHour) Quota() (int, time.Duration) { + return int(ph), time.Hour +} + +// PerDay represents a number of requests per day. +type PerDay int + +// Delay returns the duration to wait before the next request can go through, +// so that PerDay(n) == n requests per day at regular intervals. +func (pd PerDay) Delay() time.Duration { + if pd <= 0 { + return 0 + } + return time.Duration(1.0 / float64(pd) * float64(24*time.Hour)) +} + +// Quota returns the number of requests allowed in a 1 day time window, +// so that PerDay(n) == n requests allowed per day. +func (pd PerDay) Quota() (int, time.Duration) { + return int(pd), 24 * time.Hour +} + +// D represents a custom delay. +type D time.Duration + +// Delay returns the duration to wait before the next request can go through, +// which is the custom duration represented by the D value. +func (d D) Delay() time.Duration { + return time.Duration(d) +} + +// Q represents a custom quota. +type Q struct { + Requests int + Window time.Duration +} + +// Quota returns the number of requests allowed and the custom time window. +func (q Q) Quota() (int, time.Duration) { + return q.Requests, q.Window +} diff --git a/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/delayer_test.go b/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/delayer_test.go new file mode 100644 index 000000000..822978e5d --- /dev/null +++ b/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/delayer_test.go @@ -0,0 +1,65 @@ +package throttled + +import ( + "testing" + "time" +) + +func TestDelayer(t *testing.T) { + cases := []struct { + in Delayer + out time.Duration + }{ + 0: {PerSec(1), time.Second}, + 1: {PerSec(2), 500 * time.Millisecond}, + 2: {PerSec(4), 250 * time.Millisecond}, + 3: {PerSec(5), 200 * time.Millisecond}, + 4: {PerSec(10), 100 * time.Millisecond}, + 5: {PerSec(100), 10 * time.Millisecond}, + 6: {PerSec(3), 333333333 * time.Nanosecond}, + 7: {PerMin(1), time.Minute}, + 8: {PerMin(2), 30 * time.Second}, + 9: {PerMin(4), 15 * time.Second}, + 10: {PerMin(5), 12 * time.Second}, + 11: {PerMin(10), 6 * time.Second}, + 12: {PerMin(60), time.Second}, + 13: {PerHour(1), time.Hour}, + 14: {PerHour(2), 30 * time.Minute}, + 15: {PerHour(4), 15 * time.Minute}, + 16: {PerHour(60), time.Minute}, + 17: {PerHour(120), 30 * time.Second}, + 18: {D(time.Second), time.Second}, + 19: {D(5 * time.Minute), 5 * time.Minute}, + 20: {PerSec(200), 5 * time.Millisecond}, + 21: {PerDay(24), time.Hour}, + } + for i, c := range cases { + got := c.in.Delay() + if got != c.out { + t.Errorf("%d: expected %s, got %s", i, c.out, got) + } + } +} + +func TestQuota(t *testing.T) { + cases := []struct { + q Quota + reqs int + win time.Duration + }{ + 0: {PerSec(10), 10, time.Second}, + 1: {PerMin(30), 30, time.Minute}, + 2: {PerHour(124), 124, time.Hour}, + 3: {PerDay(1), 1, 24 * time.Hour}, + 4: {Q{148, 17 * time.Second}, 148, 17 * time.Second}, + } + for i, c := range cases { + r, w := c.q.Quota() + if r != c.reqs { + t.Errorf("%d: expected %d requests, got %d", i, c.reqs, r) + } + if w != c.win { + t.Errorf("%d: expected %s window, got %s", i, c.win, w) + } + } +} diff --git a/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/doc.go b/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/doc.go new file mode 100644 index 000000000..acf5213b0 --- /dev/null +++ b/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/doc.go @@ -0,0 +1,77 @@ +// Package throttled implements different throttling strategies for controlling +// access to HTTP handlers. +// +// Installation +// +// go get gopkg.in/throttled/throttled.v1/... +// +// Inverval +// +// The Interval function creates a throttler that allows requests to go through at +// a controlled, constant interval. The interval may be applied to all requests +// (vary argument == nil) or independently based on vary-by criteria. +// +// For example: +// +// th := throttled.Interval(throttled.PerSec(10), 100, &throttled.VaryBy{Path: true}, 50) +// h := th.Throttle(myHandler) +// http.ListenAndServe(":9000", h) +// +// Creates a throttler that will allow a request each 100ms (10 requests per second), with +// a buffer of 100 exceeding requests before dropping requests with a status code 429 (by +// default, configurable using th.DeniedHandler or the package-global DefaultDeniedHandler +// variable). Different paths will be throttled independently, so that /path_a and /path_b +// both can serve 10 requests per second. The last argument, 50, indicates the maximum number +// of keys that the throttler will keep in memory. +// +// MemStats +// +// The MemStats function creates a throttler that allows requests to go through only if +// the memory statistics of the current process are below specified thresholds. +// +// For example: +// +// th := throttled.MemStats(throttled.MemThresholds(&runtime.MemStats{NumGC: 10}, 10*time.Millisecond) +// h := th.Throttle(myHandler) +// http.ListenAndServe(":9000", h) +// +// Creates a throttler that will allow requests to go through until the number of garbage +// collections reaches the initial number + 10 (the MemThresholds function creates absolute +// memory stats thresholds from offsets). The second argument, 10ms, indicates the refresh +// rate of the memory stats. +// +// RateLimit +// +// The RateLimit function creates a throttler that allows a certain number of requests in +// a given time window, as is often implemented in public RESTful APIs. +// +// For example: +// +// th := throttled.RateLimit(throttled.PerMin(30), &throttled.VaryBy{RemoteAddr: true}, store.NewMemStore(1000)) +// h := th.Throttle(myHandler) +// http.ListenAndServe(":9000", h) +// +// Creates a throttler that will limit requests to 30 per minute, based on the remote address +// of the client, and will store the counter and remaining time of the current window in the +// provided memory store, limiting the number of keys to keep in memory to 1000. The store +// sub-package also provides a Redis-based Store implementations. +// +// The RateLimit throttler sets the expected X-RateLimit-* headers on the response, and +// also sets a Retry-After header when the limit is exceeded. +// +// Documentation +// +// The API documentation is available as usual on godoc.org: +// http://godoc.org/gopkg.in/throttled/throttled.v1 +// +// There is also a blog post explaining the package's usage on 0value.com: +// http://0value.com/throttled--guardian-of-the-web-server +// +// Finally, many examples are provided in the /examples sub-folder of the repository. +// +// License +// +// The BSD 3-clause license. Copyright (c) 2014 Martin Angers and Contributors. +// http://opensource.org/licenses/BSD-3-Clause +// +package throttled diff --git a/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/examples/README.md b/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/examples/README.md new file mode 100644 index 000000000..6b12dad20 --- /dev/null +++ b/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/examples/README.md @@ -0,0 +1,12 @@ +# Examples + +This directory contains examples for all the throttlers implemented by the throttled package, as well as an example of a custom limiter. + +* custom/ : implements a custom limiter that allows requests to path /a on even seconds, and on path /b on odd seconds. +* interval-many/ : implements a common interval throttler to control two different handlers, one for path /a and another for path /b, so that requests to any one of the handlers go through at the specified interval. +* interval-vary/ : implements an interval throttler that varies by path, so that requests to each different path goes through at the specified interval. +* interval/ : implements an interval throttler so that any request goes through at the specified interval, regardless of path or any other criteria. +* memstats/ : implements a memory-usage throttler that limits access based on current memory statistics. +* rate-limit/ : implements a rate-limiter throttler that varies by path, so that the number of requests allowed are counted based on the requested path. + +Each example app supports a number of command-line flags. Run the example with the -h flag to display usage and defaults. diff --git a/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/examples/custom/main.go b/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/examples/custom/main.go new file mode 100644 index 000000000..128e08b81 --- /dev/null +++ b/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/examples/custom/main.go @@ -0,0 +1,90 @@ +package main + +import ( + "flag" + "fmt" + "log" + "math/rand" + "net/http" + "sync" + "time" + + "github.com/mattermost/platform/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1" +) + +var ( + delayRes = flag.Duration("delay-response", 0, "delay the response by a random duration between 0 and this value") + output = flag.String("output", "v", "type of output, one of `v`erbose, `q`uiet, `ok`-only, `ko`-only") +) + +// Custom limiter: allow requests to the /a path on even seconds only, and +// allow access to the /b path on odd seconds only. +// +// Yes this is absurd. A more realistic case could be to allow requests to some +// contest page only during a limited time window. +type customLimiter struct { +} + +func (c *customLimiter) Start() { + // No-op +} + +func (c *customLimiter) Limit(w http.ResponseWriter, r *http.Request) (<-chan bool, error) { + s := time.Now().Second() + ch := make(chan bool, 1) + ok := (r.URL.Path == "/a" && s%2 == 0) || (r.URL.Path == "/b" && s%2 != 0) + ch <- ok + if *output == "v" { + log.Printf("Custom Limiter: Path=%s, Second=%d; ok? %v", r.URL.Path, s, ok) + } + return ch, nil +} + +func main() { + flag.Parse() + + var h http.Handler + var ok, ko int + var mu sync.Mutex + + // Keep the start time to print since-time + start := time.Now() + // Create the custom throttler using our custom limiter + t := throttled.Custom(&customLimiter{}) + // Set its denied handler + t.DeniedHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if *output == "v" || *output == "ko" { + log.Printf("KO: %s", time.Since(start)) + } + throttled.DefaultDeniedHandler.ServeHTTP(w, r) + mu.Lock() + defer mu.Unlock() + ko++ + }) + // Throttle the OK handler + rand.Seed(time.Now().Unix()) + h = t.Throttle(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if *output == "v" || *output == "ok" { + log.Printf("ok: %s", time.Since(start)) + } + if *delayRes > 0 { + wait := time.Duration(rand.Intn(int(*delayRes))) + time.Sleep(wait) + } + w.WriteHeader(200) + mu.Lock() + defer mu.Unlock() + ok++ + })) + + // Print stats once in a while + go func() { + for _ = range time.Tick(10 * time.Second) { + mu.Lock() + log.Printf("ok: %d, ko: %d", ok, ko) + mu.Unlock() + } + }() + fmt.Println("server listening on port 9000") + http.ListenAndServe(":9000", h) +} diff --git a/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/examples/interval-many/main.go b/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/examples/interval-many/main.go new file mode 100644 index 000000000..03fcfce80 --- /dev/null +++ b/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/examples/interval-many/main.go @@ -0,0 +1,79 @@ +package main + +import ( + "flag" + "fmt" + "log" + "math/rand" + "net/http" + "sync" + "time" + + "github.com/mattermost/platform/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1" +) + +var ( + delay = flag.Duration("delay", 200*time.Millisecond, "delay between calls") + bursts = flag.Int("bursts", 10, "number of bursts allowed") + delayRes = flag.Duration("delay-response", 0, "delay the response by a random duration between 0 and this value") + output = flag.String("output", "v", "type of output, one of `v`erbose, `q`uiet, `ok`-only, `ko`-only") +) + +func main() { + flag.Parse() + + var ok, ko int + var mu sync.Mutex + + // Keep start time to log since-time + start := time.Now() + + // Create the interval throttle + t := throttled.Interval(throttled.D(*delay), *bursts, nil, 0) + // Set its denied handler + t.DeniedHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if *output == "v" || *output == "ko" { + log.Printf("%s: KO: %s", r.URL.Path, time.Since(start)) + } + throttled.DefaultDeniedHandler.ServeHTTP(w, r) + mu.Lock() + defer mu.Unlock() + ko++ + }) + // Create OK handlers + rand.Seed(time.Now().Unix()) + makeHandler := func(ix int) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if *output == "v" || *output == "ok" { + log.Printf("handler %d: %s: ok: %s", ix, r.URL.Path, time.Since(start)) + } + if *delayRes > 0 { + wait := time.Duration(rand.Intn(int(*delayRes))) + time.Sleep(wait) + } + w.WriteHeader(200) + mu.Lock() + defer mu.Unlock() + ok++ + }) + } + // Throttle them using the same interval throttler + h1 := t.Throttle(makeHandler(1)) + h2 := t.Throttle(makeHandler(2)) + + // Handle two paths + mux := http.NewServeMux() + mux.Handle("/a", h1) + mux.Handle("/b", h2) + + // Print stats once in a while + go func() { + for _ = range time.Tick(10 * time.Second) { + mu.Lock() + log.Printf("ok: %d, ko: %d", ok, ko) + mu.Unlock() + } + }() + fmt.Println("server listening on port 9000") + http.ListenAndServe(":9000", mux) +} diff --git a/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/examples/interval-vary/main.go b/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/examples/interval-vary/main.go new file mode 100644 index 000000000..f84e1e41f --- /dev/null +++ b/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/examples/interval-vary/main.go @@ -0,0 +1,74 @@ +package main + +import ( + "flag" + "fmt" + "log" + "math/rand" + "net/http" + "sync" + "time" + + "github.com/mattermost/platform/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1" +) + +var ( + delay = flag.Duration("delay", 200*time.Millisecond, "delay between calls") + bursts = flag.Int("bursts", 10, "number of bursts allowed") + maxkeys = flag.Int("max-keys", 1000, "maximum number of keys") + delayRes = flag.Duration("delay-response", 0, "delay the response by a random duration between 0 and this value") + output = flag.String("output", "v", "type of output, one of `v`erbose, `q`uiet, `ok`-only, `ko`-only") +) + +func main() { + flag.Parse() + + var h http.Handler + var ok, ko int + var mu sync.Mutex + + // Keep the start time to print since-time + start := time.Now() + + // Create the interval throttler + t := throttled.Interval(throttled.D(*delay), *bursts, &throttled.VaryBy{ + Path: true, + }, *maxkeys) + // Set the denied handler + t.DeniedHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if *output == "v" || *output == "ko" { + log.Printf("KO: %s", time.Since(start)) + } + throttled.DefaultDeniedHandler.ServeHTTP(w, r) + mu.Lock() + defer mu.Unlock() + ko++ + }) + + // Throttle the OK handler + rand.Seed(time.Now().Unix()) + h = t.Throttle(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if *output == "v" || *output == "ok" { + log.Printf("%s: ok: %s", r.URL.Path, time.Since(start)) + } + if *delayRes > 0 { + wait := time.Duration(rand.Intn(int(*delayRes))) + time.Sleep(wait) + } + w.WriteHeader(200) + mu.Lock() + defer mu.Unlock() + ok++ + })) + + // Print stats once in a while + go func() { + for _ = range time.Tick(10 * time.Second) { + mu.Lock() + log.Printf("ok: %d, ko: %d", ok, ko) + mu.Unlock() + } + }() + fmt.Println("server listening on port 9000") + http.ListenAndServe(":9000", h) +} diff --git a/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/examples/interval-vary/siege-urls b/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/examples/interval-vary/siege-urls new file mode 100644 index 000000000..9a2d0d312 --- /dev/null +++ b/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/examples/interval-vary/siege-urls @@ -0,0 +1,4 @@ +http://localhost:9000/a +http://localhost:9000/b +http://localhost:9000/c + diff --git a/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/examples/interval/main.go b/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/examples/interval/main.go new file mode 100644 index 000000000..f89067816 --- /dev/null +++ b/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/examples/interval/main.go @@ -0,0 +1,69 @@ +package main + +import ( + "flag" + "fmt" + "log" + "math/rand" + "net/http" + "sync" + "time" + + "github.com/mattermost/platform/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1" +) + +var ( + delay = flag.Duration("delay", 200*time.Millisecond, "delay between calls") + bursts = flag.Int("bursts", 10, "number of bursts allowed") + delayRes = flag.Duration("delay-response", 0, "delay the response by a random duration between 0 and this value") + output = flag.String("output", "v", "type of output, one of `v`erbose, `q`uiet, `ok`-only, `ko`-only") +) + +func main() { + flag.Parse() + + var h http.Handler + var ok, ko int + var mu sync.Mutex + + // Keep the start time to print since-time + start := time.Now() + // Create the interval throttler + t := throttled.Interval(throttled.D(*delay), *bursts, nil, 0) + // Set its denied handler + t.DeniedHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if *output == "v" || *output == "ko" { + log.Printf("KO: %s", time.Since(start)) + } + throttled.DefaultDeniedHandler.ServeHTTP(w, r) + mu.Lock() + defer mu.Unlock() + ko++ + }) + // Throttle the OK handler + rand.Seed(time.Now().Unix()) + h = t.Throttle(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if *output == "v" || *output == "ok" { + log.Printf("ok: %s", time.Since(start)) + } + if *delayRes > 0 { + wait := time.Duration(rand.Intn(int(*delayRes))) + time.Sleep(wait) + } + w.WriteHeader(200) + mu.Lock() + defer mu.Unlock() + ok++ + })) + + // Print stats once in a while + go func() { + for _ = range time.Tick(10 * time.Second) { + mu.Lock() + log.Printf("ok: %d, ko: %d", ok, ko) + mu.Unlock() + } + }() + fmt.Println("server listening on port 9000") + http.ListenAndServe(":9000", h) +} diff --git a/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/examples/memstats/main.go b/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/examples/memstats/main.go new file mode 100644 index 000000000..823de9b25 --- /dev/null +++ b/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/examples/memstats/main.go @@ -0,0 +1,97 @@ +package main + +import ( + "flag" + "fmt" + "io/ioutil" + "log" + "math/rand" + "net/http" + "runtime" + "sync" + "time" + + "github.com/mattermost/platform/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1" +) + +var ( + numgc = flag.Int("gc", 0, "number of GC runs") + mallocs = flag.Int("mallocs", 0, "number of mallocs") + total = flag.Int("total", 0, "total number of bytes allocated") + allocs = flag.Int("allocs", 0, "number of bytes allocated") + refrate = flag.Duration("refresh", 0, "refresh rate of the memory stats") + delayRes = flag.Duration("delay-response", 0, "delay the response by a random duration between 0 and this value") + output = flag.String("output", "v", "type of output, one of `v`erbose, `q`uiet, `ok`-only, `ko`-only") +) + +func main() { + flag.Parse() + + var h http.Handler + var ok, ko int + var mu sync.Mutex + + // Keep the start time to print since-time + start := time.Now() + // Create the thresholds struct + thresh := throttled.MemThresholds(&runtime.MemStats{ + NumGC: uint32(*numgc), + Mallocs: uint64(*mallocs), + TotalAlloc: uint64(*total), + Alloc: uint64(*allocs), + }) + if *output != "q" { + log.Printf("thresholds: NumGC: %d, Mallocs: %d, Alloc: %dKb, Total: %dKb", thresh.NumGC, thresh.Mallocs, thresh.Alloc/1024, thresh.TotalAlloc/1024) + } + // Create the MemStats throttler + t := throttled.MemStats(thresh, *refrate) + // Set its denied handler + t.DeniedHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if *output == "v" || *output == "ko" { + log.Printf("KO: %s", time.Since(start)) + } + throttled.DefaultDeniedHandler.ServeHTTP(w, r) + mu.Lock() + defer mu.Unlock() + ko++ + }) + + // Throttle the OK handler + rand.Seed(time.Now().Unix()) + h = t.Throttle(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if *output == "v" || *output == "ok" { + log.Printf("ok: %s", time.Since(start)) + } + if *delayRes > 0 { + wait := time.Duration(rand.Intn(int(*delayRes))) + time.Sleep(wait) + } + // Read the whole file in memory, to actually use 64Kb (instead of streaming to w) + b, err := ioutil.ReadFile("test-file") + if err != nil { + throttled.Error(w, r, err) + return + } + _, err = w.Write(b) + if err != nil { + throttled.Error(w, r, err) + } + mu.Lock() + defer mu.Unlock() + ok++ + })) + + // Print stats once in a while + go func() { + var mem runtime.MemStats + for _ = range time.Tick(10 * time.Second) { + mu.Lock() + runtime.ReadMemStats(&mem) + log.Printf("ok: %d, ko: %d", ok, ko) + log.Printf("TotalAllocs: %d Kb, Allocs: %d Kb, Mallocs: %d, NumGC: %d", mem.TotalAlloc/1024, mem.Alloc/1024, mem.Mallocs, mem.NumGC) + mu.Unlock() + } + }() + fmt.Println("server listening on port 9000") + http.ListenAndServe(":9000", h) +} diff --git a/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/examples/memstats/test-file b/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/examples/memstats/test-file Binary files differnew file mode 100644 index 000000000..c97c12f9b --- /dev/null +++ b/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/examples/memstats/test-file diff --git a/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/examples/rate-limit/main.go b/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/examples/rate-limit/main.go new file mode 100644 index 000000000..1e4c1cb8f --- /dev/null +++ b/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/examples/rate-limit/main.go @@ -0,0 +1,101 @@ +package main + +import ( + "flag" + "fmt" + "log" + "math/rand" + "net/http" + "sync" + "time" + + "github.com/mattermost/platform/Godeps/_workspace/src/github.com/garyburd/redigo/redis" + "github.com/mattermost/platform/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1" + "github.com/mattermost/platform/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/store" +) + +var ( + requests = flag.Int("requests", 10, "number of requests allowed in the time window") + window = flag.Duration("window", time.Minute, "time window for the limit of requests") + storeType = flag.String("store", "mem", "store to use, one of `mem` or `redis` (on default localhost port)") + delayRes = flag.Duration("delay-response", 0, "delay the response by a random duration between 0 and this value") + output = flag.String("output", "v", "type of output, one of `v`erbose, `q`uiet, `ok`-only, `ko`-only") +) + +func main() { + flag.Parse() + + var h http.Handler + var ok, ko int + var mu sync.Mutex + var st throttled.Store + + // Keep the start time to print since-time + start := time.Now() + // Create the rate-limit store + switch *storeType { + case "mem": + st = store.NewMemStore(0) + case "redis": + st = store.NewRedisStore(setupRedis(), "throttled:", 0) + default: + log.Fatalf("unsupported store: %s", *storeType) + } + // Create the rate-limit throttler, varying on path + t := throttled.RateLimit(throttled.Q{Requests: *requests, Window: *window}, &throttled.VaryBy{ + Path: true, + }, st) + + // Set its denied handler + t.DeniedHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if *output == "v" || *output == "ko" { + log.Printf("KO: %s", time.Since(start)) + } + throttled.DefaultDeniedHandler.ServeHTTP(w, r) + mu.Lock() + defer mu.Unlock() + ko++ + }) + + // Throttle the OK handler + rand.Seed(time.Now().Unix()) + h = t.Throttle(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if *output == "v" || *output == "ok" { + log.Printf("ok: %s", time.Since(start)) + } + if *delayRes > 0 { + wait := time.Duration(rand.Intn(int(*delayRes))) + time.Sleep(wait) + } + w.WriteHeader(200) + mu.Lock() + defer mu.Unlock() + ok++ + })) + + // Print stats once in a while + go func() { + for _ = range time.Tick(10 * time.Second) { + mu.Lock() + log.Printf("ok: %d, ko: %d", ok, ko) + mu.Unlock() + } + }() + fmt.Println("server listening on port 9000") + http.ListenAndServe(":9000", h) +} + +func setupRedis() *redis.Pool { + pool := &redis.Pool{ + MaxIdle: 3, + IdleTimeout: 30 * time.Second, + Dial: func() (redis.Conn, error) { + return redis.Dial("tcp", ":6379") + }, + TestOnBorrow: func(c redis.Conn, t time.Time) error { + _, err := c.Do("PING") + return err + }, + } + return pool +} diff --git a/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/interval.go b/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/interval.go new file mode 100644 index 000000000..ab4d45251 --- /dev/null +++ b/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/interval.go @@ -0,0 +1,164 @@ +package throttled + +import ( + "net/http" + "sync" + "time" + + "github.com/mattermost/platform/Godeps/_workspace/src/github.com/golang/groupcache/lru" +) + +// Static check to ensure that the interval limiters implement the Limiter interface. +var _ Limiter = (*intervalVaryByLimiter)(nil) +var _ Limiter = (*intervalLimiter)(nil) + +// Interval creates a throttler that controls the requests so that they +// go through at a constant interval. The interval is specified by the +// delay argument, and convenience types such as PerSec can be used to +// express the interval in a more expressive way, i.e. PerSec(10) means +// 10 requests per second or one request each 100ms, PerMin(30) means +// 30 requests per minute or on request each 2s, etc. +// +// The bursts argument indicates the number of exceeding requests that may +// be queued up waiting to be processed. Requests that overflow the queue +// are dropped and go through the denied handler, which may be specified +// on the Throttler and that defaults to the package-global variable +// DefaultDeniedHandler. +// +// The vary argument indicates the criteria to use to group the requests, +// so that the interval applies to the requests in the same group (e.g. based on +// the path, or the remote IP address, etc.). If this argument is nil, the +// interval applies to all requests going through this throttler. +// +// The maxKeys indicates the maximum number of keys to keep in memory to apply the interval, +// when a vary argument is specified. A LRU algorithm is used to remove older keys. +// +func Interval(delay Delayer, bursts int, vary *VaryBy, maxKeys int) *Throttler { + var l Limiter + if vary != nil { + if maxKeys < 1 { + maxKeys = 1 + } + l = &intervalVaryByLimiter{ + delay: delay.Delay(), + bursts: bursts, + vary: vary, + maxKeys: maxKeys, + } + } else { + l = &intervalLimiter{ + delay: delay.Delay(), + bursts: bursts, + } + } + return &Throttler{ + limiter: l, + } +} + +// The intervalLimiter struct implements an interval limiter with no vary-by +// criteria. +type intervalLimiter struct { + delay time.Duration + bursts int + + bucket chan chan bool +} + +// Start initializes the limiter for execution. +func (il *intervalLimiter) Start() { + if il.bursts < 0 { + il.bursts = 0 + } + il.bucket = make(chan chan bool, il.bursts) + go process(il.bucket, il.delay) +} + +// Limit is called for each request to the throttled handler. It tries to +// queue the request to allow it to run at the given interval, but if the +// queue is full, the request is denied access. +func (il *intervalLimiter) Limit(w http.ResponseWriter, r *http.Request) (<-chan bool, error) { + ch := make(chan bool, 1) + select { + case il.bucket <- ch: + return ch, nil + default: + ch <- false + return ch, nil + } +} + +// The intervalVaryByLimiter struct implements an interval limiter with a vary-by +// criteria. +type intervalVaryByLimiter struct { + delay time.Duration + bursts int + vary *VaryBy + + lock sync.RWMutex + keys *lru.Cache + maxKeys int +} + +// Start initializes the limiter for execution. +func (il *intervalVaryByLimiter) Start() { + if il.bursts < 0 { + il.bursts = 0 + } + il.keys = lru.New(il.maxKeys) + il.keys.OnEvicted = il.stopProcess +} + +// Limit is called for each request to the throttled handler. It tries to +// queue the request for the vary-by key to allow it to run at the given interval, +// but if the queue is full, the request is denied access. +func (il *intervalVaryByLimiter) Limit(w http.ResponseWriter, r *http.Request) (<-chan bool, error) { + ch := make(chan bool, 1) + key := il.vary.Key(r) + + il.lock.RLock() + item, ok := il.keys.Get(key) + if !ok { + // Create the key, bucket, start goroutine + // First release the read lock and acquire a write lock + il.lock.RUnlock() + il.lock.Lock() + // Create the bucket, add the key + bucket := make(chan chan bool, il.bursts) + il.keys.Add(key, bucket) + // Start the goroutine to process this bucket + go process(bucket, il.delay) + item = bucket + // Release the write lock, acquire the read lock + il.lock.Unlock() + il.lock.RLock() + } + defer il.lock.RUnlock() + bucket := item.(chan chan bool) + select { + case bucket <- ch: + return ch, nil + default: + ch <- false + return ch, nil + } +} + +// process loops through the queued requests for a key's bucket, and sends +// requests through at the given interval. +func process(bucket chan chan bool, delay time.Duration) { + after := time.After(0) + for v := range bucket { + <-after + // Let the request go through + v <- true + // Wait the required duration + after = time.After(delay) + } +} + +// stopProcess is called when a key is removed from the LRU cache so that its +// accompanying goroutine is correctly released. +func (il *intervalVaryByLimiter) stopProcess(key lru.Key, value interface{}) { + close(value.(chan chan bool)) +} diff --git a/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/interval_test.go b/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/interval_test.go new file mode 100644 index 000000000..bc584e134 --- /dev/null +++ b/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/interval_test.go @@ -0,0 +1,114 @@ +package throttled + +import ( + "net/http" + "testing" + + "github.com/PuerkitoBio/boom/commands" +) + +func TestInterval(t *testing.T) { + if testing.Short() { + t.Skip() + } + cases := []struct { + n int + c int + rps int + bursts int + }{ + 0: {60, 10, 20, 100}, + 1: {300, 20, 100, 100}, + 2: {10, 10, 1, 10}, + 3: {1000, 100, 1000, 100}, + } + for i, c := range cases { + // Setup the stats handler + st := &stats{} + // Create the throttler + th := Interval(PerSec(c.rps), c.bursts, nil, 0) + th.DeniedHandler = http.HandlerFunc(st.DeniedHTTP) + b := commands.Boom{ + Req: &commands.ReqOpts{}, + N: c.n, + C: c.c, + Output: "quiet", + } + // Run the test + rpts := runTest(th.Throttle(st), b) + // Assert results + for _, rpt := range rpts { + assertRPS(t, i, c.rps, rpt) + } + assertStats(t, i, st, rpts) + } +} + +func TestIntervalVary(t *testing.T) { + if testing.Short() { + t.Skip() + } + cases := []struct { + n int + c int + urls int + rps int + bursts int + }{ + 0: {60, 10, 3, 20, 100}, + 1: {300, 20, 3, 100, 100}, + 2: {10, 10, 3, 1, 10}, + 3: {500, 10, 2, 1000, 100}, + } + for i, c := range cases { + // Setup the stats handler + st := &stats{} + // Create the throttler + th := Interval(PerSec(c.rps), c.bursts, nil, 0) + th.DeniedHandler = http.HandlerFunc(st.DeniedHTTP) + var booms []commands.Boom + for j := 0; j < c.urls; j++ { + booms = append(booms, commands.Boom{ + Req: &commands.ReqOpts{}, + N: c.n, + C: c.c, + Output: "quiet", + }) + } + // Run the test + rpts := runTest(th.Throttle(st), booms...) + // Assert results + for _, rpt := range rpts { + assertRPS(t, i, c.rps, rpt) + } + assertStats(t, i, st, rpts) + } +} + +func assertRPS(t *testing.T, ix int, exp int, rpt *commands.Report) { + wigglef := 0.2 * float64(exp) + if rpt.SuccessRPS < float64(exp)-wigglef || rpt.SuccessRPS > float64(exp)+wigglef { + t.Errorf("%d: expected RPS to be around %d, got %f", ix, exp, rpt.SuccessRPS) + } +} + +func assertStats(t *testing.T, ix int, st *stats, rpts []*commands.Report) { + ok, ko, _ := st.Stats() + var twos, fives, max int + for _, rpt := range rpts { + twos += rpt.StatusCodeDist[200] + fives += rpt.StatusCodeDist[deniedStatus] + if len(rpt.StatusCodeDist) > max { + max = len(rpt.StatusCodeDist) + } + } + if ok != twos { + t.Errorf("%d: expected %d status 200, got %d", ix, twos, ok) + } + if ko != fives { + t.Errorf("%d: expected %d status 429, got %d", ix, fives, ok) + } + if max > 2 { + t.Errorf("%d: expected at most 2 different status codes, got %d", ix, max) + } +} diff --git a/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/memstats.go b/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/memstats.go new file mode 100644 index 000000000..bd2765630 --- /dev/null +++ b/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/memstats.go @@ -0,0 +1,214 @@ +package throttled + +import ( + "net/http" + "runtime" + "sync" + "time" +) + +// Static check to ensure that memStatsLimiter implements Limiter. +var _ Limiter = (*memStatsLimiter)(nil) + +// The memStatsLimiter struct implements a limiter based on the memory statistics +// of the current process. +type memStatsLimiter struct { + thresholds *runtime.MemStats + refreshRate time.Duration + + lockStats sync.RWMutex + stats runtime.MemStats +} + +// MemStats creates a Throttler based on the memory statistics of the current process. +// Any combination of any (non-array) integer field of Go's runtime.MemStats structure +// can be used as thresholds to deny a request. +// +// As soon as one threshold value is reached, the access is denied. If the value can +// decrease, access will be allowed again once it gets back under the threshold value. +// Denied requests go through the denied handler, which may be specified on the Throttler +// and that defaults to the package-global variable DefaultDeniedHandler. +// +// Thresholds must be specified in absolute numbers (i.e. NumGC = 10 means stop once the +// NumGC reaches 10, not when the current value increments by 10), and zero values are +// ignored. +// +// The refreshRate indicates the frequency at which the process' memory stats are refreshed, +// and 0 means on each request. +// +func MemStats(thresholds *runtime.MemStats, refreshRate time.Duration) *Throttler { + return &Throttler{ + limiter: &memStatsLimiter{ + thresholds: thresholds, + refreshRate: refreshRate, + }, + } +} + +// Start initialized the limiter for execution. +func (m *memStatsLimiter) Start() { + // Make sure there is an initial MemStats reading + runtime.ReadMemStats(&m.stats) + if m.refreshRate > 0 { + go m.refresh() + } +} + +// refresh runs in a separate goroutine and refreshes the memory statistics +// at regular intervals. +func (m *memStatsLimiter) refresh() { + c := time.Tick(m.refreshRate) + for _ = range c { + m.lockStats.Lock() + runtime.ReadMemStats(&m.stats) + m.lockStats.Unlock() + } +} + +// Limit is called for each request to the throttled handler. It checks if +// the request can go through by checking the memory thresholds, and signals it +// via the returned channel. +func (m *memStatsLimiter) Limit(w http.ResponseWriter, r *http.Request) (<-chan bool, error) { + ch := make(chan bool, 1) + // Check if memory thresholds are reached + ch <- m.allow() + return ch, nil +} + +// allow compares the current memory stats with the thresholds, and returns +// false if any threshold is reached. +func (m *memStatsLimiter) allow() bool { + m.lockStats.RLock() + mem := m.stats + m.lockStats.RUnlock() + // If refreshRate == 0, then read on every request. + if m.refreshRate == 0 { + runtime.ReadMemStats(&mem) + } + ok := true + checkStat(m.thresholds.Alloc, mem.Alloc, &ok) + checkStat(m.thresholds.BuckHashSys, mem.BuckHashSys, &ok) + checkStat(m.thresholds.Frees, mem.Frees, &ok) + checkStat(m.thresholds.GCSys, mem.GCSys, &ok) + checkStat(m.thresholds.HeapAlloc, mem.HeapAlloc, &ok) + checkStat(m.thresholds.HeapIdle, mem.HeapIdle, &ok) + checkStat(m.thresholds.HeapInuse, mem.HeapInuse, &ok) + checkStat(m.thresholds.HeapObjects, mem.HeapObjects, &ok) + checkStat(m.thresholds.HeapReleased, mem.HeapReleased, &ok) + checkStat(m.thresholds.HeapSys, mem.HeapSys, &ok) + checkStat(m.thresholds.LastGC, mem.LastGC, &ok) + checkStat(m.thresholds.Lookups, mem.Lookups, &ok) + checkStat(m.thresholds.MCacheInuse, mem.MCacheInuse, &ok) + checkStat(m.thresholds.MCacheSys, mem.MCacheSys, &ok) + checkStat(m.thresholds.MSpanInuse, mem.MSpanInuse, &ok) + checkStat(m.thresholds.MSpanSys, mem.MSpanSys, &ok) + checkStat(m.thresholds.Mallocs, mem.Mallocs, &ok) + checkStat(m.thresholds.NextGC, mem.NextGC, &ok) + checkStat(uint64(m.thresholds.NumGC), uint64(mem.NumGC), &ok) + checkStat(m.thresholds.OtherSys, mem.OtherSys, &ok) + checkStat(m.thresholds.PauseTotalNs, mem.PauseTotalNs, &ok) + checkStat(m.thresholds.StackInuse, mem.StackInuse, &ok) + checkStat(m.thresholds.StackSys, mem.StackSys, &ok) + checkStat(m.thresholds.Sys, mem.Sys, &ok) + checkStat(m.thresholds.TotalAlloc, mem.TotalAlloc, &ok) + return ok +} + +// Checks the threshold value against the actual value, and assigns false +// to the boolean pointer if the threshold is reached. +func checkStat(threshold, actual uint64, ok *bool) { + if !*ok { + return + } + if threshold > 0 { + if actual >= threshold { + *ok = false + } + } +} + +// MemThresholds is a convenience function to create a thresholds memory stats from +// offsets to apply to the current memory stats. Zero values in the offset stats +// are left to 0 in the resulting thresholds memory stats value. +// +// The return value may be used as thresholds argument to the MemStats function. +func MemThresholds(offset *runtime.MemStats) *runtime.MemStats { + var mem, thr runtime.MemStats + runtime.ReadMemStats(&mem) + if offset.Alloc > 0 { + thr.Alloc = mem.Alloc + offset.Alloc + } + if offset.BuckHashSys > 0 { + thr.BuckHashSys = mem.BuckHashSys + offset.BuckHashSys + } + if offset.Frees > 0 { + thr.Frees = mem.Frees + offset.Frees + } + if offset.GCSys > 0 { + thr.GCSys = mem.GCSys + offset.GCSys + } + if offset.HeapAlloc > 0 { + thr.HeapAlloc = mem.HeapAlloc + offset.HeapAlloc + } + if offset.HeapIdle > 0 { + thr.HeapIdle = mem.HeapIdle + offset.HeapIdle + } + if offset.HeapInuse > 0 { + thr.HeapInuse = mem.HeapInuse + offset.HeapInuse + } + if offset.HeapObjects > 0 { + thr.HeapObjects = mem.HeapObjects + offset.HeapObjects + } + if offset.HeapReleased > 0 { + thr.HeapReleased = mem.HeapReleased + offset.HeapReleased + } + if offset.HeapSys > 0 { + thr.HeapSys = mem.HeapSys + offset.HeapSys + } + if offset.LastGC > 0 { + thr.LastGC = mem.LastGC + offset.LastGC + } + if offset.Lookups > 0 { + thr.Lookups = mem.Lookups + offset.Lookups + } + if offset.MCacheInuse > 0 { + thr.MCacheInuse = mem.MCacheInuse + offset.MCacheInuse + } + if offset.MCacheSys > 0 { + thr.MCacheSys = mem.MCacheSys + offset.MCacheSys + } + if offset.MSpanInuse > 0 { + thr.MSpanInuse = mem.MSpanInuse + offset.MSpanInuse + } + if offset.MSpanSys > 0 { + thr.MSpanSys = mem.MSpanSys + offset.MSpanSys + } + if offset.Mallocs > 0 { + thr.Mallocs = mem.Mallocs + offset.Mallocs + } + if offset.NextGC > 0 { + thr.NextGC = mem.NextGC + offset.NextGC + } + if offset.NumGC > 0 { + thr.NumGC = mem.NumGC + offset.NumGC + } + if offset.OtherSys > 0 { + thr.OtherSys = mem.OtherSys + offset.OtherSys + } + if offset.PauseTotalNs > 0 { + thr.PauseTotalNs = mem.PauseTotalNs + offset.PauseTotalNs + } + if offset.StackInuse > 0 { + thr.StackInuse = mem.StackInuse + offset.StackInuse + } + if offset.StackSys > 0 { + thr.StackSys = mem.StackSys + offset.StackSys + } + if offset.Sys > 0 { + thr.Sys = mem.Sys + offset.Sys + } + if offset.TotalAlloc > 0 { + thr.TotalAlloc = mem.TotalAlloc + offset.TotalAlloc + } + return &thr +} diff --git a/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/memstats_test.go b/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/memstats_test.go new file mode 100644 index 000000000..2b8faa721 --- /dev/null +++ b/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/memstats_test.go @@ -0,0 +1,64 @@ +package throttled + +import ( + "net/http" + "runtime" + "testing" + "time" + + "github.com/PuerkitoBio/boom/commands" +) + +func TestMemStats(t *testing.T) { + if testing.Short() { + t.Skip() + } + cases := []struct { + n int + c int + gc uint32 + total uint64 + rate time.Duration + }{ + 0: {1000, 10, 3, 0, 0}, + 1: {200, 10, 0, 600000, 0}, + 2: {500, 10, 2, 555555, 10 * time.Millisecond}, + } + for i, c := range cases { + // Setup the stats handler + st := &stats{} + // Create the throttler + limit := MemThresholds(&runtime.MemStats{NumGC: c.gc, TotalAlloc: c.total}) + th := MemStats(limit, c.rate) + th.DeniedHandler = http.HandlerFunc(st.DeniedHTTP) + // Run the test + b := commands.Boom{ + Req: &commands.ReqOpts{}, + N: c.n, + C: c.c, + Output: "quiet", + } + rpts := runTest(th.Throttle(st), b) + // Assert results + assertStats(t, i, st, rpts) + assertMem(t, i, limit) + } +} + +func assertMem(t *testing.T, ix int, limit *runtime.MemStats) { + var mem runtime.MemStats + runtime.ReadMemStats(&mem) + if mem.NumGC < limit.NumGC { + t.Errorf("%d: expected gc to be at least %d, got %d", ix, limit.NumGC, mem.NumGC) + } + if mem.TotalAlloc < limit.TotalAlloc { + t.Errorf("%d: expected total alloc to be at least %dKb, got %dKb", ix, limit.TotalAlloc/1024, mem.TotalAlloc/1024) + } +} + +func BenchmarkReadMemStats(b *testing.B) { + var mem runtime.MemStats + for i := 0; i < b.N; i++ { + runtime.ReadMemStats(&mem) + } +} diff --git a/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/misc/pre-commit b/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/misc/pre-commit new file mode 100644 index 000000000..88b61bfde --- /dev/null +++ b/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/misc/pre-commit @@ -0,0 +1,38 @@ +#!/bin/sh +# Copyright 2012 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +# git gofmt pre-commit hook +# +# To use, store as .git/hooks/pre-commit inside your repository and make sure +# it has execute permissions. +# +# This script does not handle file names that contain spaces. + +# golint is purely informational, it doesn't fail with exit code != 0 if it finds something, +# because it may find a lot of false positives. Just print out its result for information. +echo "lint result (informational only):" +echo +golint . + +# go vet returns 1 if an error was found. Exit the hook with this exit code. +go vet ./... +vetres=$? + +# Check for gofmt problems and report if any. +gofiles=$(git diff --cached --name-only --diff-filter=ACM | grep '.go$') +[ -z "$gofiles" ] && echo "EXIT $vetres" && exit $vetres + +unformatted=$(gofmt -l $gofiles) +[ -z "$unformatted" ] && echo "EXIT $vetres" && exit $vetres + +# Some files are not gofmt'd. Print message and fail. + +echo >&2 "Go files must be formatted with gofmt. Please run:" +for fn in $unformatted; do + echo >&2 " gofmt -w $PWD/$fn" +done + +echo "EXIT 1" +exit 1 diff --git a/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/rate.go b/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/rate.go new file mode 100644 index 000000000..d7a7de6d7 --- /dev/null +++ b/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/rate.go @@ -0,0 +1,116 @@ +package throttled + +import ( + "math" + "net/http" + "strconv" + "time" +) + +// Static check to ensure that rateLimiter implements Limiter. +var _ Limiter = (*rateLimiter)(nil) + +// RateLimit creates a throttler that limits the number of requests allowed +// in a certain time window defined by the Quota q. The q parameter specifies +// the requests per time window, and it is silently set to at least 1 request +// and at least a 1 second window if it is less than that. The time window +// starts when the first request is made outside an existing window. Fractions +// of seconds are not supported, they are truncated. +// +// The vary parameter indicates what criteria should be used to group requests +// for which the limit must be applied (ex.: rate limit based on the remote address). +// See varyby.go for the various options. +// +// The specified store is used to keep track of the request count and the +// time remaining in the window. The throttled package comes with some stores +// in the throttled/store package. Custom stores can be created too, by implementing +// the Store interface. +// +// Requests that bust the rate limit are denied access and go through the denied handler, +// which may be specified on the Throttler and that defaults to the package-global +// variable DefaultDeniedHandler. +// +// The rate limit throttler sets the following headers on the response: +// +// X-RateLimit-Limit : quota +// X-RateLimit-Remaining : number of requests remaining in the current window +// X-RateLimit-Reset : seconds before a new window +// +// Additionally, if the request was denied access, the following header is added: +// +// Retry-After : seconds before the caller should retry +// +func RateLimit(q Quota, vary *VaryBy, store Store) *Throttler { + // Extract requests and window + reqs, win := q.Quota() + + // Create and return the throttler + return &Throttler{ + limiter: &rateLimiter{ + reqs: reqs, + window: win, + vary: vary, + store: store, + }, + } +} + +// The rate limiter implements limiting the request to a certain quota +// based on the vary-by criteria. State is saved in the store. +type rateLimiter struct { + reqs int + window time.Duration + vary *VaryBy + store Store +} + +// Start initializes the limiter for execution. +func (r *rateLimiter) Start() { + if r.reqs < 1 { + r.reqs = 1 + } + if r.window < time.Second { + r.window = time.Second + } +} + +// Limit is called for each request to the throttled handler. It checks if +// the request can go through and signals it via the returned channel. +// It returns an error if the operation fails. +func (r *rateLimiter) Limit(w http.ResponseWriter, req *http.Request) (<-chan bool, error) { + // Create return channel and initialize + ch := make(chan bool, 1) + ok := true + key := r.vary.Key(req) + + // Get the current count and remaining seconds + cnt, secs, err := r.store.Incr(key, r.window) + // Handle the possible situations: error, begin new window, or increment current window. + switch { + case err != nil && err != ErrNoSuchKey: + // An unexpected error occurred + return nil, err + case err == ErrNoSuchKey || secs <= 0: + // Reset counter + if err := r.store.Reset(key, r.window); err != nil { + return nil, err + } + cnt = 1 + secs = int(r.window.Seconds()) + default: + // If the limit is reached, deny access + if cnt > r.reqs { + ok = false + } + } + // Set rate-limit headers + w.Header().Add("X-RateLimit-Limit", strconv.Itoa(r.reqs)) + w.Header().Add("X-RateLimit-Remaining", strconv.Itoa(int(math.Max(float64(r.reqs-cnt), 0)))) + w.Header().Add("X-RateLimit-Reset", strconv.Itoa(secs)) + if !ok { + w.Header().Add("Retry-After", strconv.Itoa(secs)) + } + // Send response via the return channel + ch <- ok + return ch, nil +} diff --git a/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/rate_test.go b/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/rate_test.go new file mode 100644 index 000000000..67dea74b1 --- /dev/null +++ b/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/rate_test.go @@ -0,0 +1,101 @@ +package throttled + +import ( + "net/http" + "net/http/httptest" + "strconv" + "testing" + "time" +) + +const deniedStatus = 429 + +// Simple memory store for tests, unsafe for concurrent access +type mapStore struct { + cnt map[string]int + ts map[string]time.Time +} + +func newMapStore() *mapStore { + return &mapStore{ + make(map[string]int), + make(map[string]time.Time), + } +} +func (ms *mapStore) Incr(key string, window time.Duration) (int, int, error) { + if _, ok := ms.cnt[key]; !ok { + return 0, 0, ErrNoSuchKey + } + ms.cnt[key]++ + ts := ms.ts[key] + return ms.cnt[key], RemainingSeconds(ts, window), nil +} +func (ms *mapStore) Reset(key string, win time.Duration) error { + ms.cnt[key] = 1 + ms.ts[key] = time.Now().UTC() + return nil +} + +func TestRateLimit(t *testing.T) { + quota := Q{5, 5 * time.Second} + cases := []struct { + limit, remain, reset, status int + }{ + 0: {5, 4, 5, 200}, + 1: {5, 3, 4, 200}, + 2: {5, 2, 4, 200}, + 3: {5, 1, 3, 200}, + 4: {5, 0, 3, 200}, + 5: {5, 0, 2, deniedStatus}, + } + // Limit the requests to 2 per second + th := Interval(PerSec(2), 0, nil, 0) + // Rate limit + rl := RateLimit(quota, nil, newMapStore()) + // Create the stats + st := &stats{} + // Create the handler + h := th.Throttle(rl.Throttle(st)) + + // Start the server + srv := httptest.NewServer(h) + defer srv.Close() + for i, c := range cases { + callRateLimited(t, i, c.limit, c.remain, c.reset, c.status, srv.URL) + } + // Wait 3 seconds and call again, should start a new window + time.Sleep(3 * time.Second) + callRateLimited(t, len(cases), 5, 4, 5, 200, srv.URL) +} + +func callRateLimited(t *testing.T, i, limit, remain, reset, status int, url string) { + res, err := http.Get(url) + if err != nil { + t.Fatal(err) + } + defer res.Body.Close() + // Assert status code + if status != res.StatusCode { + t.Errorf("%d: expected status %d, got %d", i, status, res.StatusCode) + } + // Assert headers + if v := res.Header.Get("X-RateLimit-Limit"); v != strconv.Itoa(limit) { + t.Errorf("%d: expected limit header to be %d, got %s", i, limit, v) + } + if v := res.Header.Get("X-RateLimit-Remaining"); v != strconv.Itoa(remain) { + t.Errorf("%d: expected remain header to be %d, got %s", i, remain, v) + } + // Allow 1 second wiggle room + v := res.Header.Get("X-RateLimit-Reset") + vi, _ := strconv.Atoi(v) + if vi < reset-1 || vi > reset+1 { + t.Errorf("%d: expected reset header to be close to %d, got %d", i, reset, vi) + } + if status == deniedStatus { + v := res.Header.Get("Retry-After") + vi, _ := strconv.Atoi(v) + if vi < reset-1 || vi > reset+1 { + t.Errorf("%d: expected retry after header to be close to %d, got %d", i, reset, vi) + } + } +} diff --git a/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/store.go b/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/store.go new file mode 100644 index 000000000..760fe2b69 --- /dev/null +++ b/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/store.go @@ -0,0 +1,31 @@ +package throttled + +import ( + "errors" + "time" +) + +// The error returned if the key does not exist in the Store. +var ErrNoSuchKey = errors.New("throttled: no such key") + +// Store is the interface to implement to store the RateLimit state (number +// of requests per key, time-to-live or creation timestamp). +type Store interface { + // Incr increments the count for the specified key and returns the new value along + // with the number of seconds remaining. It may return an error + // if the operation fails. + // + // The method may return ErrNoSuchKey if the key to increment does not exist, + // in which case Reset will be called to initialize the value. + Incr(string, time.Duration) (int, int, error) + + // Reset resets the key to 1 with the specified window duration. It must create the + // key if it doesn't exist. It returns an error if it fails. + Reset(string, time.Duration) error +} + +// RemainingSeconds is a helper function that returns the number of seconds +// remaining from an absolute timestamp in UTC. +func RemainingSeconds(ts time.Time, window time.Duration) int { + return int((window - time.Now().UTC().Sub(ts)).Seconds()) +} diff --git a/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/store/doc.go b/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/store/doc.go new file mode 100644 index 000000000..adb4618d3 --- /dev/null +++ b/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/store/doc.go @@ -0,0 +1,2 @@ +// Package store offers a memory-based and a Redis-based throttled.Store implementation. +package store diff --git a/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/store/mem.go b/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/store/mem.go new file mode 100644 index 000000000..74d48a73e --- /dev/null +++ b/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/store/mem.go @@ -0,0 +1,90 @@ +package store + +import ( + "sync" + "time" + + "github.com/mattermost/platform/Godeps/_workspace/src/github.com/golang/groupcache/lru" + "github.com/mattermost/platform/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1" +) + +// memStore implements an in-memory Store. +type memStore struct { + sync.Mutex + keys *lru.Cache + m map[string]*counter +} + +// NewMemStore creates a new MemStore. If maxKeys > 0, the number of different keys +// is restricted to the specified amount. In this case, it uses an LRU algorithm to +// evict older keys to make room for newer ones. If a request is made for a key that +// has been evicted, it will be processed as if its count was 0, possibly allowing requests +// that should be denied. +// +// If maxKeys <= 0, there is no limit on the number of keys, which may use an unbounded amount of +// memory depending on the server's load. +// +// The MemStore is only for single-process rate-limiting. To share the rate limit state +// among multiple instances of the web server, use a database- or key-value-based +// store. +// +func NewMemStore(maxKeys int) throttled.Store { + var m *memStore + if maxKeys > 0 { + m = &memStore{ + keys: lru.New(maxKeys), + } + } else { + m = &memStore{ + m: make(map[string]*counter), + } + } + return m +} + +// A counter represents a single entry in the MemStore. +type counter struct { + n int + ts time.Time +} + +// Incr increments the counter for the specified key. It returns the new +// count value and the remaining number of seconds, or an error. +func (ms *memStore) Incr(key string, window time.Duration) (int, int, error) { + ms.Lock() + defer ms.Unlock() + var c *counter + if ms.keys != nil { + v, _ := ms.keys.Get(key) + if v != nil { + c = v.(*counter) + } + } else { + c = ms.m[key] + } + if c == nil { + c = &counter{0, time.Now().UTC()} + } + c.n++ + if ms.keys != nil { + ms.keys.Add(key, c) + } else { + ms.m[key] = c + } + return c.n, throttled.RemainingSeconds(c.ts, window), nil +} + +// Reset resets the counter for the specified key. It sets the count +// to 1 and initializes the timestamp with the current time, in UTC. +// It returns an error if the operation fails. +func (ms *memStore) Reset(key string, win time.Duration) error { + ms.Lock() + defer ms.Unlock() + c := &counter{1, time.Now().UTC()} + if ms.keys != nil { + ms.keys.Add(key, c) + } else { + ms.m[key] = c + } + return nil +} diff --git a/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/store/mem_test.go b/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/store/mem_test.go new file mode 100644 index 000000000..e8ef8d0da --- /dev/null +++ b/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/store/mem_test.go @@ -0,0 +1,43 @@ +package store + +import ( + "testing" + "time" +) + +func TestMemStore(t *testing.T) { + st := NewMemStore(0) + win := time.Second + + // Reset stores a key with count of 1, current timestamp + err := st.Reset("k", time.Second) + if err != nil { + t.Errorf("expected reset to return nil, got %s", err) + } + cnt, sec1, _ := st.Incr("k", win) + if cnt != 2 { + t.Errorf("expected reset+incr to set count to 2, got %d", cnt) + } + + // Incr increments the key, keeps same timestamp + cnt, sec2, err := st.Incr("k", win) + if err != nil { + t.Errorf("expected 2nd incr to return nil error, got %s", err) + } + if cnt != 3 { + t.Errorf("expected 2nd incr to return 3, got %d", cnt) + } + if sec1 != sec2 { + t.Errorf("expected 2nd incr to return %d secs, got %d", sec1, sec2) + } + + // Reset on existing key brings it back to 1, new timestamp + err = st.Reset("k", win) + if err != nil { + t.Errorf("expected reset on existing key to return nil, got %s", err) + } + cnt, _, _ = st.Incr("k", win) + if cnt != 2 { + t.Errorf("expected last reset+incr to return 2, got %d", cnt) + } +} diff --git a/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/store/redis.go b/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/store/redis.go new file mode 100644 index 000000000..44ef43122 --- /dev/null +++ b/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/store/redis.go @@ -0,0 +1,85 @@ +package store + +import ( + "time" + + "github.com/mattermost/platform/Godeps/_workspace/src/github.com/garyburd/redigo/redis" + "github.com/mattermost/platform/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1" +) + +// redisStore implements a Redis-based store. +type redisStore struct { + pool *redis.Pool + prefix string + db int +} + +// NewRedisStore creates a new Redis-based store, using the provided pool to get its +// connections. The keys will have the specified keyPrefix, which may be an empty string, +// and the database index specified by db will be selected to store the keys. +// +func NewRedisStore(pool *redis.Pool, keyPrefix string, db int) throttled.Store { + return &redisStore{ + pool: pool, + prefix: keyPrefix, + db: db, + } +} + +// Incr increments the specified key. If the key did not exist, it sets it to 1 +// and sets it to expire after the number of seconds specified by window. +// +// It returns the new count value and the number of remaining seconds, or an error +// if the operation fails. +func (r *redisStore) Incr(key string, window time.Duration) (int, int, error) { + conn := r.pool.Get() + defer conn.Close() + if err := selectDB(r.db, conn); err != nil { + return 0, 0, err + } + // Atomically increment and read the TTL. + conn.Send("MULTI") + conn.Send("INCR", r.prefix+key) + conn.Send("TTL", r.prefix+key) + vals, err := redis.Values(conn.Do("EXEC")) + if err != nil { + conn.Do("DISCARD") + return 0, 0, err + } + var cnt, ttl int + if _, err = redis.Scan(vals, &cnt, &ttl); err != nil { + return 0, 0, err + } + // If there was no TTL set, then this is a newly created key (INCR creates the key + // if it didn't exist), so set it to expire. + if ttl == -1 { + ttl = int(window.Seconds()) + _, err = conn.Do("EXPIRE", r.prefix+key, ttl) + if err != nil { + return 0, 0, err + } + } + return cnt, ttl, nil +} + +// Reset sets the value of the key to 1, and resets its time window. +func (r *redisStore) Reset(key string, window time.Duration) error { + conn := r.pool.Get() + defer conn.Close() + if err := selectDB(r.db, conn); err != nil { + return err + } + _, err := redis.String(conn.Do("SET", r.prefix+key, "1", "EX", int(window.Seconds()), "NX")) + return err +} + +// Select the specified database index. +func selectDB(db int, conn redis.Conn) error { + // Select the specified database + if db > 0 { + if _, err := redis.String(conn.Do("SELECT", db)); err != nil { + return err + } + } + return nil +} diff --git a/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/store/redis_test.go b/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/store/redis_test.go new file mode 100644 index 000000000..d7d1d1743 --- /dev/null +++ b/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/store/redis_test.go @@ -0,0 +1,66 @@ +package store + +import ( + "testing" + "time" + + "github.com/mattermost/platform/Godeps/_workspace/src/github.com/garyburd/redigo/redis" +) + +func getPool() *redis.Pool { + pool := &redis.Pool{ + MaxIdle: 3, + IdleTimeout: 30 * time.Second, + Dial: func() (redis.Conn, error) { + return redis.Dial("tcp", ":6379") + }, + TestOnBorrow: func(c redis.Conn, t time.Time) error { + _, err := c.Do("PING") + return err + }, + } + return pool +} + +func TestRedisStore(t *testing.T) { + pool := getPool() + c := pool.Get() + if _, err := redis.String(c.Do("PING")); err != nil { + c.Close() + t.Skip("redis server not available on localhost port 6379") + } + st := NewRedisStore(pool, "throttled:", 1) + win := 2 * time.Second + + // Incr increments the key, even if it does not exist + cnt, secs, err := st.Incr("k", win) + if err != nil { + t.Errorf("expected initial incr to return nil error, got %s", err) + } + if cnt != 1 { + t.Errorf("expected initial incr to return 1, got %d", cnt) + } + if secs != int(win.Seconds()) { + t.Errorf("expected initial incr to return %d secs, got %d", int(win.Seconds()), secs) + } + + // Waiting a second diminishes the remaining seconds + time.Sleep(time.Second) + _, sec2, _ := st.Incr("k", win) + if sec2 != secs-1 { + t.Errorf("expected 2nd incr after a 1s sleep to return %d secs, got %d", secs-1, sec2) + } + + // Waiting a second so the key expires, Incr should set back to 1, initial secs + time.Sleep(1100 * time.Millisecond) + cnt, sec3, err := st.Incr("k", win) + if err != nil { + t.Errorf("expected last incr to return nil error, got %s", err) + } + if cnt != 1 { + t.Errorf("expected last incr to return 1, got %d", cnt) + } + if sec3 != int(win.Seconds()) { + t.Errorf("expected last incr to return %d secs, got %d", int(win.Seconds()), sec3) + } +} diff --git a/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/throttler.go b/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/throttler.go new file mode 100644 index 000000000..06da13051 --- /dev/null +++ b/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/throttler.go @@ -0,0 +1,86 @@ +package throttled + +import ( + "net/http" + "sync" +) + +var ( + // DefaultDeniedHandler handles the requests that were denied access because + // of a throttler. By default, returns a 429 status code with a + // generic message. + DefaultDeniedHandler = http.Handler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "limit exceeded", 429) + })) + + // Error is the function to call when an error occurs on a throttled handler. + // By default, returns a 500 status code with a generic message. + Error = ErrorFunc(func(w http.ResponseWriter, r *http.Request, err error) { + http.Error(w, "internal error", http.StatusInternalServerError) + }) +) + +// ErrorFunc defines the function type for the Error variable. +type ErrorFunc func(w http.ResponseWriter, r *http.Request, err error) + +// The Limiter interface defines the methods required to control access to a +// throttled handler. +type Limiter interface { + Start() + Limit(http.ResponseWriter, *http.Request) (<-chan bool, error) +} + +// Custom creates a Throttler using the provided Limiter implementation. +func Custom(l Limiter) *Throttler { + return &Throttler{ + limiter: l, + } +} + +// A Throttler controls access to HTTP handlers using a Limiter. +type Throttler struct { + // DeniedHandler is called if the request is disallowed. If it is nil, + // the DefaultDeniedHandler variable is used. + DeniedHandler http.Handler + + limiter Limiter + // The mutex protects the started flag + mu sync.Mutex + started bool +} + +// Throttle wraps a HTTP handler so that its access is controlled by +// the Throttler. It returns the Handler with the throttling logic. +func (t *Throttler) Throttle(h http.Handler) http.Handler { + dh := t.start() + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ch, err := t.limiter.Limit(w, r) + if err != nil { + Error(w, r, err) + return + } + ok := <-ch + if ok { + h.ServeHTTP(w, r) + } else { + dh.ServeHTTP(w, r) + } + }) +} + +// start starts the throttling and returns the effective denied handler to +// use for requests that were denied access. +func (t *Throttler) start() http.Handler { + t.mu.Lock() + defer t.mu.Unlock() + // Get the effective denied handler + dh := t.DeniedHandler + if dh == nil { + dh = DefaultDeniedHandler + } + if !t.started { + t.limiter.Start() + t.started = true + } + return dh +} diff --git a/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/varyby.go b/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/varyby.go new file mode 100644 index 000000000..3b2cdb011 --- /dev/null +++ b/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/varyby.go @@ -0,0 +1,78 @@ +package throttled + +import ( + "bytes" + "net/http" + "strings" +) + +// VaryBy defines the criteria to use to group requests. +type VaryBy struct { + // Vary by the RemoteAddr as specified by the net/http.Request field. + RemoteAddr bool + + // Vary by the HTTP Method as specified by the net/http.Request field. + Method bool + + // Vary by the URL's Path as specified by the Path field of the net/http.Request + // URL field. + Path bool + + // Vary by this list of header names, read from the net/http.Request Header field. + Headers []string + + // Vary by this list of parameters, read from the net/http.Request FormValue method. + Params []string + + // Vary by this list of cookie names, read from the net/http.Request Cookie method. + Cookies []string + + // Use this separator string to concatenate the various criteria of the VaryBy struct. + // Defaults to a newline character if empty (\n). + Separator string + + // Custom specifies the custom-generated key to use for this request. + // If not nil, the value returned by this function is used instead of any + // VaryBy criteria. + Custom func(r *http.Request) string +} + +// Key returns the key for this request based on the criteria defined by the VaryBy struct. +func (vb *VaryBy) Key(r *http.Request) string { + var buf bytes.Buffer + + if vb == nil { + return "" // Special case for no vary-by option + } + if vb.Custom != nil { + // A custom key generator is specified + return vb.Custom(r) + } + sep := vb.Separator + if sep == "" { + sep = "\n" // Separator defaults to newline + } + if vb.RemoteAddr { + buf.WriteString(strings.ToLower(r.RemoteAddr) + sep) + } + if vb.Method { + buf.WriteString(strings.ToLower(r.Method) + sep) + } + for _, h := range vb.Headers { + buf.WriteString(strings.ToLower(r.Header.Get(h)) + sep) + } + if vb.Path { + buf.WriteString(r.URL.Path + sep) + } + for _, p := range vb.Params { + buf.WriteString(r.FormValue(p) + sep) + } + for _, c := range vb.Cookies { + ck, err := r.Cookie(c) + if err == nil { + buf.WriteString(ck.Value) + } + buf.WriteString(sep) // Write the separator anyway, whether or not the cookie exists + } + return buf.String() +} diff --git a/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/varyby_test.go b/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/varyby_test.go new file mode 100644 index 000000000..91b7ae0ae --- /dev/null +++ b/Godeps/_workspace/src/gopkg.in/throttled/throttled.v1/varyby_test.go @@ -0,0 +1,56 @@ +package throttled + +import ( + "net/http" + "net/url" + "testing" +) + +func TestVaryBy(t *testing.T) { + u, err := url.Parse("http://localhost/test/path?q=s") + if err != nil { + panic(err) + } + ck := &http.Cookie{Name: "ssn", Value: "test"} + cases := []struct { + vb *VaryBy + r *http.Request + k string + }{ + 0: {nil, &http.Request{}, ""}, + 1: {&VaryBy{RemoteAddr: true}, &http.Request{RemoteAddr: "::"}, "::\n"}, + 2: { + &VaryBy{Method: true, Path: true}, + &http.Request{Method: "POST", URL: u}, + "post\n/test/path\n", + }, + 3: { + &VaryBy{Headers: []string{"Content-length"}}, + &http.Request{Header: http.Header{"Content-Type": []string{"text/plain"}, "Content-Length": []string{"123"}}}, + "123\n", + }, + 4: { + &VaryBy{Separator: ",", Method: true, Headers: []string{"Content-length"}, Params: []string{"q", "user"}}, + &http.Request{Method: "GET", Header: http.Header{"Content-Type": []string{"text/plain"}, "Content-Length": []string{"123"}}, Form: url.Values{"q": []string{"s"}, "pwd": []string{"secret"}, "user": []string{"test"}}}, + "get,123,s,test,", + }, + 5: { + &VaryBy{Cookies: []string{"ssn"}}, + &http.Request{Header: http.Header{"Cookie": []string{ck.String()}}}, + "test\n", + }, + 6: { + &VaryBy{Cookies: []string{"ssn"}, RemoteAddr: true, Custom: func(r *http.Request) string { + return "blah" + }}, + &http.Request{Header: http.Header{"Cookie": []string{ck.String()}}}, + "blah", + }, + } + for i, c := range cases { + got := c.vb.Key(c.r) + if got != c.k { + t.Errorf("%d: expected '%s' (%d), got '%s' (%d)", i, c.k, len(c.k), got, len(got)) + } + } +} |