rewrite checkpoint

2026-05-31 18:28:27 +02:00 · 2017-08-13 18:23:38 -07:00
parent b00021ccbd
commit 36632a52f0
883 changed files with 222856 additions and 2972 deletions
@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) [2016] [Seth Ammons]
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
@@ -0,0 +1,126 @@
+# pester
+
+`pester` wraps Go's standard lib http client to provide several options to increase resiliency in your request. If you experience poor network conditions or requests could experience varied delays, you can now pester the endpoint for data.
+- Send out multiple requests and get the first back (only used for GET calls)
+- Retry on errors
+- Backoff
+
+### Simple Example
+Use `pester` where you would use the http client calls. By default, pester will use a concurrency of 1, and retry the endpoint 3 times with the `DefaultBackoff` strategy of waiting 1 second between retries.
+```go
+/* swap in replacement, just switch
+   http.{Get|Post|PostForm|Head|Do} to
+   pester.{Get|Post|PostForm|Head|Do}
+*/
+resp, err := pester.Get("http://sethammons.com")
+```
+
+### Backoff Strategy
+Provide your own backoff strategy, or use one of the provided built in strategies:
+- `DefaultBackoff`: 1 second
+- `LinearBackoff`: n seconds where n is the retry number
+- `LinearJitterBackoff`: n seconds where n is the retry number, +/- 0-33%
+- `ExponentialBackoff`: n seconds where n is 2^(retry number)
+- `ExponentialJitterBackoff`: n seconds where n is 2^(retry number), +/- 0-33%
+
+```go
+client := pester.New()
+client.Backoff = func(retry int) time.Duration {
+    // set up something dynamic or use a look up table
+    return time.Duration(retry) * time.Minute
+}
+```
+
+### Complete example
+For a complete and working example, see the sample directory.
+`pester` allows you to use a constructor to control:
+- backoff strategy
+- reties
+- concurrency
+- keeping a log for debugging
+```go
+package main
+
+import (
+    "log"
+    "net/http"
+    "strings"
+
+    "github.com/sethgrid/pester"
+)
+
+func main() {
+    log.Println("Starting...")
+
+    { // drop in replacement for http.Get and other client methods
+        resp, err := pester.Get("http://example.com")
+        if err != nil {
+            log.Println("error GETing example.com", err)
+        }
+        defer resp.Body.Close()
+        log.Printf("example.com %s", resp.Status)
+    }
+
+    { // control the resiliency
+        client := pester.New()
+        client.Concurrency = 3
+        client.MaxRetries = 5
+        client.Backoff = pester.ExponentialBackoff
+        client.KeepLog = true
+
+        resp, err := client.Get("http://example.com")
+        if err != nil {
+            log.Println("error GETing example.com", client.LogString())
+        }
+        defer resp.Body.Close()
+        log.Printf("example.com %s", resp.Status)
+    }
+
+    { // use the pester version of http.Client.Do
+        req, err := http.NewRequest("POST", "http://example.com", strings.NewReader("data"))
+        if err != nil {
+            log.Fatal("Unable to create a new http request", err)
+        }
+        resp, err := pester.Do(req)
+        if err != nil {
+            log.Println("error POSTing example.com", err)
+        }
+        defer resp.Body.Close()
+        log.Printf("example.com %s", resp.Status)
+    }
+}
+
+```
+
+### Example Log
+`pester` also allows you to control the resiliency and can optionally log the errors.
+```go
+c := pester.New()
+c.KeepLog = true
+
+nonExistantURL := "http://localhost:9000/foo"
+_, _ = c.Get(nonExistantURL)
+
+fmt.Println(c.LogString())
+/*
+Output:
+
+1432402837 Get [GET] http://localhost:9000/foo request-0 retry-0 error: Get http://localhost:9000/foo: dial tcp 127.0.0.1:9000: connection refused
+1432402838 Get [GET] http://localhost:9000/foo request-0 retry-1 error: Get http://localhost:9000/foo: dial tcp 127.0.0.1:9000: connection refused
+1432402839 Get [GET] http://localhost:9000/foo request-0 retry-2 error: Get http://localhost:9000/foo: dial tcp 127.0.0.1:9000: connection refused
+*/
+```
+
+### Tests
+
+You can run tests in the root directory with `$ go test`. There is a benchmark-like test available with `$ cd benchmarks; go test`.
+You can see `pester` in action with `$ cd sample; go run main.go`.
+
+For watching open file descriptors, you can run `watch "lsof -i -P | grep main"` if you started the app with `go run main.go`.
+I did this for watching for FD leaks. My method was to alter `sample/main.go` to only run one case (`pester.Get with set backoff stategy, concurrency and retries increased`)
+and adding a sleep after the result came back. This let me verify if FDs were getting left open when they should have closed. If you know a better way, let me know!
+I was able to see that FDs are now closing when they should :)
+
+![Are we there yet?](http://butchbellah.com/wp-content/uploads/2012/06/Are-We-There-Yet.jpg)
+
+Are we there yet? Are we there yet? Are we there yet? Are we there yet? ...
@@ -0,0 +1,27 @@
+# Timing Tests
+
+It was noted in [issue #2](github.com/sethgrid/pester/issue/2) that Pester may be slower than the standard library (along with bug that was fixed).
+
+I put together a quick test to see how Pester fairs against the stand library. Here are the results:
+
+```
+$ go test
+  Standard Library Get           675178 ns Avg.
+  Pester, Default                690157 ns Avg.
+  Pester, Retries 1, Conc 1      671322 ns Avg.
+  Pester, Retries 2, Conc 2      764386 ns Avg.
+  Pester, Retries 3, Conc 3      893899 ns Avg.
+  Pester, Retries 0, Conc 1      730407 ns Avg.
+  Pester, Retries 0, Conc 2     1077721 ns Avg.
+  Pester, Retries 0, Conc 3     1889403 ns Avg.
+  Pester, Retries 0, Conc 1     1758464 ns Avg.
+  Pester, Retries 2, Conc 1     1249081 ns Avg.
+  Pester, Retries 3, Conc 1     1824322 ns Avg.
+PASS
+```
+
+Running the test locally multiple times shows some variance, but this is a typical result. In raw time, these average times are not far off from each other (about 1ms from the best to worst case). In comparisons between percents, we see a drift of up to 3x.
+
+The up to 3x drift between the near identical default Pester implementation and the Standard Library http.Get call compared to the last test case of 'Retries 3, Conc 1' makes little sense in that the default Pester uses 'Retries 3, Conc 1' as its settings.
+
+I think that it is safe to say that there is no material difference in speed between the Standard Library and Pester. 
@@ -0,0 +1,164 @@
+package main
+
+/*
+	Can't use testing.B Tests because it eats up file descriptors
+*/
+
+import (
+	"fmt"
+	"log"
+	"net"
+	"net/http"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/sethgrid/pester"
+)
+
+type getter func(string) (*http.Response, error)
+
+func TestWarmup(t *testing.T) {
+	// The first request/test takes more time.
+	// Does not matter if we use http.Get or pester.Get
+	// nor if we use the default client or initialize one.
+	// I don't know why yet.
+	c := pester.New()
+	_ = runner("Warm Up", c.Get)
+}
+
+func TestStdLibGet(t *testing.T) {
+	// base case - get a url with std lib
+	fmt.Println(runner("Standard Library Get ", http.Get))
+}
+
+func TestPesterGetDefaults(t *testing.T) {
+	fmt.Println(runner("Pester, Default", pester.Get))
+}
+
+func TestPesterRetry1Conc1(t *testing.T) {
+	c := pester.New()
+	c.MaxRetries = 1
+	c.Concurrency = 1
+
+	fmt.Println(runner("Pester, Retries 1, Conc 1", c.Get))
+}
+
+func TestPesterRetry2Conc2(t *testing.T) {
+	c := pester.New()
+	c.MaxRetries = 2
+	c.Concurrency = 2
+
+	fmt.Println(runner("Pester, Retries 2, Conc 2", c.Get))
+}
+
+func TestPesterRetry3Conc3(t *testing.T) {
+	c := pester.New()
+	c.MaxRetries = 3
+	c.Concurrency = 3
+
+	fmt.Println(runner("Pester, Retries 3, Conc 3", c.Get))
+}
+
+func TestPesterGetRetry0Conc1(t *testing.T) {
+	c := pester.New()
+	c.MaxRetries = 0
+	c.Concurrency = 1
+
+	fmt.Println(runner("Pester, Retries 0, Conc 1", c.Get))
+}
+
+func TestPesterGetRetry0Conc2(t *testing.T) {
+	c := pester.New()
+	c.MaxRetries = 0
+	c.Concurrency = 2
+
+	fmt.Println(runner("Pester, Retries 0, Conc 2", c.Get))
+}
+
+func TestPesterGetRetry0Conc3(t *testing.T) {
+	c := pester.New()
+	c.MaxRetries = 0
+	c.Concurrency = 3
+
+	fmt.Println(runner("Pester, Retries 0, Conc 3", c.Get))
+}
+
+func TestPesterGetRetry1Conc1(t *testing.T) {
+	c := pester.New()
+	c.MaxRetries = 0
+	c.Concurrency = 1
+
+	fmt.Println(runner("Pester, Retries 0, Conc 1", c.Get))
+}
+
+func TestPesterGetRetries2Conc1(t *testing.T) {
+	c := pester.New()
+	c.Concurrency = 2
+	c.MaxRetries = 1
+
+	fmt.Println(runner("Pester, Retries 2, Conc 1", c.Get))
+}
+
+func TestPesterGetRetries3Conc1(t *testing.T) {
+	c := pester.New()
+	c.Concurrency = 3
+	c.MaxRetries = 1
+
+	fmt.Println(runner("Pester, Retries 3, Conc 1", c.Get))
+}
+
+func reportTimings(name string, timings []int64) string {
+	var sum int64
+	for _, t := range timings {
+		sum += t
+	}
+	average := sum / int64(len(timings))
+	return fmt.Sprintf("  %-29s %7d ns Avg.", name, average)
+}
+
+func runServer() int {
+	mux := http.NewServeMux()
+	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+		w.Write([]byte("OK"))
+	})
+	l, err := net.Listen("tcp", ":0")
+	if err != nil {
+		log.Fatal("unable to secure listener", err)
+	}
+	go func() {
+		if err := http.Serve(l, mux); err != nil {
+			log.Fatal("stable server error", err)
+		}
+	}()
+	port, err := strconv.Atoi(strings.Replace(l.Addr().String(), "[::]:", "", 1))
+	if err != nil {
+		log.Fatal("unable to determine port", err)
+	}
+	return port
+}
+
+func runner(name string, Do getter) string {
+	var timings []int64
+	for n := 0; n < 7; n++ {
+		stableServerPort := runServer()
+
+		start := time.Now().UnixNano()
+		r, err := Do(fmt.Sprintf("http://localhost:%d/%d", stableServerPort, time.Now().UnixNano()))
+		if err != nil {
+			log.Fatal("Error came back and it should not have", err)
+		}
+		if r == nil {
+			log.Fatal("No response!")
+		}
+		if r.Body == nil {
+			log.Fatal("No response body!")
+		}
+		r.Body.Close()
+		end := time.Now().UnixNano()
+
+		timings = append(timings, end-start)
+	}
+	return reportTimings(name, timings)
+}
@@ -0,0 +1,423 @@
+package pester
+
+// pester provides additional resiliency over the standard http client methods by
+// allowing you to control concurrency, retries, and a backoff strategy.
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"math"
+	"math/rand"
+	"net/http"
+	"net/url"
+	"sync"
+	"time"
+)
+
+// Client wraps the http client and exposes all the functionality of the http.Client.
+// Additionally, Client provides pester specific values for handling resiliency.
+type Client struct {
+	// wrap it to provide access to http built ins
+	hc *http.Client
+
+	Transport     http.RoundTripper
+	CheckRedirect func(req *http.Request, via []*http.Request) error
+	Jar           http.CookieJar
+	Timeout       time.Duration
+
+	// pester specific
+	Concurrency int
+	MaxRetries  int
+	Backoff     BackoffStrategy
+	KeepLog     bool
+
+	SuccessReqNum   int
+	SuccessRetryNum int
+
+	wg *sync.WaitGroup
+
+	sync.Mutex
+	ErrLog []ErrEntry
+}
+
+// ErrEntry is used to provide the LogString() data and is populated
+// each time an error happens if KeepLog is set.
+// ErrEntry.Retry is deprecated in favor of ErrEntry.Attempt
+type ErrEntry struct {
+	Time    time.Time
+	Method  string
+	URL     string
+	Verb    string
+	Request int
+	Retry   int
+	Attempt int
+	Err     error
+}
+
+// result simplifies the channel communication for concurrent request handling
+type result struct {
+	resp  *http.Response
+	err   error
+	req   int
+	retry int
+}
+
+// params represents all the params needed to run http client calls and pester errors
+type params struct {
+	method   string
+	verb     string
+	req      *http.Request
+	url      string
+	bodyType string
+	body     io.Reader
+	data     url.Values
+}
+
+// New constructs a new DefaultClient with sensible default values
+func New() *Client {
+	return &Client{
+		Concurrency: DefaultClient.Concurrency,
+		MaxRetries:  DefaultClient.MaxRetries,
+		Backoff:     DefaultClient.Backoff,
+		ErrLog:      DefaultClient.ErrLog,
+		wg:          &sync.WaitGroup{},
+	}
+}
+
+// NewExtendedClient allows you to pass in an http.Client that is previously set up
+// and extends it to have Pester's features of concurrency and retries.
+func NewExtendedClient(hc *http.Client) *Client {
+	c := New()
+	c.hc = hc
+	return c
+}
+
+// BackoffStrategy is used to determine how long a retry request should wait until attempted
+type BackoffStrategy func(retry int) time.Duration
+
+// DefaultClient provides sensible defaults
+var DefaultClient = &Client{Concurrency: 1, MaxRetries: 3, Backoff: DefaultBackoff, ErrLog: []ErrEntry{}}
+
+// DefaultBackoff always returns 1 second
+func DefaultBackoff(_ int) time.Duration {
+	return 1 * time.Second
+}
+
+// ExponentialBackoff returns ever increasing backoffs by a power of 2
+func ExponentialBackoff(i int) time.Duration {
+	return time.Duration(math.Pow(2, float64(i))) * time.Second
+}
+
+// ExponentialJitterBackoff returns ever increasing backoffs by a power of 2
+// with +/- 0-33% to prevent sychronized reuqests.
+func ExponentialJitterBackoff(i int) time.Duration {
+	return jitter(int(math.Pow(2, float64(i))))
+}
+
+// LinearBackoff returns increasing durations, each a second longer than the last
+func LinearBackoff(i int) time.Duration {
+	return time.Duration(i) * time.Second
+}
+
+// LinearJitterBackoff returns increasing durations, each a second longer than the last
+// with +/- 0-33% to prevent sychronized reuqests.
+func LinearJitterBackoff(i int) time.Duration {
+	return jitter(i)
+}
+
+// jitter keeps the +/- 0-33% logic in one place
+func jitter(i int) time.Duration {
+	ms := i * 1000
+
+	maxJitter := ms / 3
+
+	rand.Seed(time.Now().Unix())
+	jitter := rand.Intn(maxJitter + 1)
+
+	if rand.Intn(2) == 1 {
+		ms = ms + jitter
+	} else {
+		ms = ms - jitter
+	}
+
+	// a jitter of 0 messes up the time.Tick chan
+	if ms <= 0 {
+		ms = 1
+	}
+
+	return time.Duration(ms) * time.Millisecond
+}
+
+// Wait blocks until all pester requests have returned
+// Probably not that useful outside of testing.
+func (c *Client) Wait() {
+	c.wg.Wait()
+}
+
+// pester provides all the logic of retries, concurrency, backoff, and logging
+func (c *Client) pester(p params) (*http.Response, error) {
+	resultCh := make(chan result)
+	multiplexCh := make(chan result)
+	finishCh := make(chan struct{})
+
+	// track all requests that go out so we can close the late listener routine that closes late incoming response bodies
+	totalSentRequests := &sync.WaitGroup{}
+	totalSentRequests.Add(1)
+	defer totalSentRequests.Done()
+	allRequestsBackCh := make(chan struct{})
+	go func() {
+		totalSentRequests.Wait()
+		close(allRequestsBackCh)
+	}()
+
+	// GET calls should be idempotent and can make use
+	// of concurrency. Other verbs can mutate and should not
+	// make use of the concurrency feature
+	concurrency := c.Concurrency
+	if p.verb != "GET" {
+		concurrency = 1
+	}
+
+	c.Lock()
+	if c.hc == nil {
+		c.hc = &http.Client{}
+		c.hc.Transport = c.Transport
+		c.hc.CheckRedirect = c.CheckRedirect
+		c.hc.Jar = c.Jar
+		c.hc.Timeout = c.Timeout
+	}
+	c.Unlock()
+
+	// re-create the http client so we can leverage the std lib
+	httpClient := http.Client{
+		Transport:     c.hc.Transport,
+		CheckRedirect: c.hc.CheckRedirect,
+		Jar:           c.hc.Jar,
+		Timeout:       c.hc.Timeout,
+	}
+
+	// if we have a request body, we need to save it for later
+	var originalRequestBody []byte
+	var originalBody []byte
+	var err error
+	if p.req != nil && p.req.Body != nil {
+		originalRequestBody, err = ioutil.ReadAll(p.req.Body)
+		if err != nil {
+			return &http.Response{}, errors.New("error reading request body")
+		}
+		p.req.Body.Close()
+	}
+	if p.body != nil {
+		originalBody, err = ioutil.ReadAll(p.body)
+		if err != nil {
+			return &http.Response{}, errors.New("error reading body")
+		}
+	}
+
+	AttemptLimit := c.MaxRetries
+	if AttemptLimit <= 0 {
+		AttemptLimit = 1
+	}
+
+	for req := 0; req < concurrency; req++ {
+		c.wg.Add(1)
+		totalSentRequests.Add(1)
+		go func(n int, p params) {
+			defer c.wg.Done()
+			defer totalSentRequests.Done()
+
+			var err error
+			for i := 1; i <= AttemptLimit; i++ {
+				c.wg.Add(1)
+				defer c.wg.Done()
+				select {
+				case <-finishCh:
+					return
+				default:
+				}
+				resp := &http.Response{}
+
+				// rehydrate the body (it is drained each read)
+				if len(originalRequestBody) > 0 {
+					p.req.Body = ioutil.NopCloser(bytes.NewBuffer(originalRequestBody))
+				}
+				if len(originalBody) > 0 {
+					p.body = bytes.NewBuffer(originalBody)
+				}
+
+				// route the calls
+				switch p.method {
+				case "Do":
+					resp, err = httpClient.Do(p.req)
+				case "Get":
+					resp, err = httpClient.Get(p.url)
+				case "Head":
+					resp, err = httpClient.Head(p.url)
+				case "Post":
+					resp, err = httpClient.Post(p.url, p.bodyType, p.body)
+				case "PostForm":
+					resp, err = httpClient.PostForm(p.url, p.data)
+				}
+
+				// Early return if we have a valid result
+				// Only retry (ie, continue the loop) on 5xx status codes
+				if err == nil && resp.StatusCode < 500 {
+					multiplexCh <- result{resp: resp, err: err, req: n, retry: i}
+					return
+				}
+
+				c.log(ErrEntry{
+					Time:    time.Now(),
+					Method:  p.method,
+					Verb:    p.verb,
+					URL:     p.url,
+					Request: n,
+					Retry:   i + 1, // would remove, but would break backward compatibility
+					Attempt: i,
+					Err:     err,
+				})
+
+				// if it is the last iteration, grab the result (which is an error at this point)
+				if i == AttemptLimit {
+					multiplexCh <- result{resp: resp, err: err}
+					return
+				}
+
+				// if we are retrying, we should close this response body to free the fd
+				if resp != nil {
+					resp.Body.Close()
+				}
+
+				// prevent a 0 from causing the tick to block, pass additional microsecond
+				<-time.Tick(c.Backoff(i) + 1*time.Microsecond)
+			}
+		}(req, p)
+	}
+
+	// spin off the go routine so it can continually listen in on late results and close the response bodies
+	go func() {
+		gotFirstResult := false
+		for {
+			select {
+			case res := <-multiplexCh:
+				if !gotFirstResult {
+					gotFirstResult = true
+					close(finishCh)
+					resultCh <- res
+				} else if res.resp != nil {
+					// we only return one result to the caller; close all other response bodies that come back
+					// drain the body before close as to not prevent keepalive. see https://gist.github.com/mholt/eba0f2cc96658be0f717
+					io.Copy(ioutil.Discard, res.resp.Body)
+					res.resp.Body.Close()
+				}
+			case <-allRequestsBackCh:
+				// don't leave this goroutine running
+				return
+			}
+		}
+	}()
+
+	select {
+	case res := <-resultCh:
+		c.Lock()
+		defer c.Unlock()
+		c.SuccessReqNum = res.req
+		c.SuccessRetryNum = res.retry
+		return res.resp, res.err
+	}
+}
+
+// LogString provides a string representation of the errors the client has seen
+func (c *Client) LogString() string {
+	c.Lock()
+	defer c.Unlock()
+	var res string
+	for _, e := range c.ErrLog {
+		res += fmt.Sprintf("%d %s [%s] %s request-%d retry-%d error: %s\n",
+			e.Time.Unix(), e.Method, e.Verb, e.URL, e.Request, e.Retry, e.Err)
+	}
+	return res
+}
+
+// LogErrCount is a helper method used primarily for test validation
+func (c *Client) LogErrCount() int {
+	c.Lock()
+	defer c.Unlock()
+	return len(c.ErrLog)
+}
+
+// EmbedHTTPClient allows you to extend an existing Pester client with an
+// underlying http.Client, such as https://godoc.org/golang.org/x/oauth2/google#DefaultClient
+func (c *Client) EmbedHTTPClient(hc *http.Client) {
+	c.hc = hc
+}
+
+func (c *Client) log(e ErrEntry) {
+	if c.KeepLog {
+		c.Lock()
+		c.ErrLog = append(c.ErrLog, e)
+		c.Unlock()
+	}
+}
+
+// Do provides the same functionality as http.Client.Do
+func (c *Client) Do(req *http.Request) (resp *http.Response, err error) {
+	return c.pester(params{method: "Do", req: req, verb: req.Method, url: req.URL.String()})
+}
+
+// Get provides the same functionality as http.Client.Get
+func (c *Client) Get(url string) (resp *http.Response, err error) {
+	return c.pester(params{method: "Get", url: url, verb: "GET"})
+}
+
+// Head provides the same functionality as http.Client.Head
+func (c *Client) Head(url string) (resp *http.Response, err error) {
+	return c.pester(params{method: "Head", url: url, verb: "HEAD"})
+}
+
+// Post provides the same functionality as http.Client.Post
+func (c *Client) Post(url string, bodyType string, body io.Reader) (resp *http.Response, err error) {
+	return c.pester(params{method: "Post", url: url, bodyType: bodyType, body: body, verb: "POST"})
+}
+
+// PostForm provides the same functionality as http.Client.PostForm
+func (c *Client) PostForm(url string, data url.Values) (resp *http.Response, err error) {
+	return c.pester(params{method: "PostForm", url: url, data: data, verb: "POST"})
+}
+
+////////////////////////////////////////
+// Provide self-constructing variants //
+////////////////////////////////////////
+
+// Do provides the same functionality as http.Client.Do and creates its own constructor
+func Do(req *http.Request) (resp *http.Response, err error) {
+	c := New()
+	return c.Do(req)
+}
+
+// Get provides the same functionality as http.Client.Get and creates its own constructor
+func Get(url string) (resp *http.Response, err error) {
+	c := New()
+	return c.Get(url)
+}
+
+// Head provides the same functionality as http.Client.Head and creates its own constructor
+func Head(url string) (resp *http.Response, err error) {
+	c := New()
+	return c.Head(url)
+}
+
+// Post provides the same functionality as http.Client.Post and creates its own constructor
+func Post(url string, bodyType string, body io.Reader) (resp *http.Response, err error) {
+	c := New()
+	return c.Post(url, bodyType, body)
+}
+
+// PostForm provides the same functionality as http.Client.PostForm and creates its own constructor
+func PostForm(url string, data url.Values) (resp *http.Response, err error) {
+	c := New()
+	return c.PostForm(url, data)
+}
@@ -0,0 +1,373 @@
+package pester_test
+
+import (
+	"fmt"
+	"log"
+	"net"
+	"runtime"
+	"strconv"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"net/http"
+	"net/http/cookiejar"
+
+	"github.com/sethgrid/pester"
+)
+
+func TestConcurrentRequests(t *testing.T) {
+	t.Parallel()
+
+	c := pester.New()
+	c.Concurrency = 3
+	c.KeepLog = true
+
+	nonExistantURL := "http://localhost:9000/foo"
+
+	_, err := c.Get(nonExistantURL)
+	if err == nil {
+		t.Fatal("expected to get an error")
+	}
+	c.Wait()
+
+	// in the event of an error, let's see what the logs were
+	t.Log("\n", c.LogString())
+
+	if got, want := c.LogErrCount(), c.Concurrency*c.MaxRetries; got != want {
+		t.Errorf("got %d attempts, want %d", got, want)
+	}
+}
+
+func TestConcurrent2Retry0(t *testing.T) {
+	t.Parallel()
+
+	c := pester.New()
+	c.Concurrency = 2
+	c.MaxRetries = 0
+	c.KeepLog = true
+
+	nonExistantURL := "http://localhost:9000/foo"
+
+	_, err := c.Get(nonExistantURL)
+	if err == nil {
+		t.Fatal("expected to get an error")
+	}
+	c.Wait()
+
+	// in the event of an error, let's see what the logs were
+	t.Log("\n", c.LogString())
+
+	if got, want := c.LogErrCount(), c.Concurrency; got != want {
+		t.Errorf("got %d attempts, want %d", got, want)
+	}
+}
+
+func TestDefaultBackoff(t *testing.T) {
+	t.Parallel()
+
+	c := pester.New()
+	c.KeepLog = true
+
+	nonExistantURL := "http://localhost:9000/foo"
+
+	_, err := c.Get(nonExistantURL)
+	if err == nil {
+		t.Fatal("expected to get an error")
+	}
+	c.Wait()
+
+	// in the event of an error, let's see what the logs were
+	t.Log("\n", c.LogString())
+
+	if got, want := c.Concurrency, 1; got != want {
+		t.Errorf("got %d, want %d for concurrency", got, want)
+	}
+
+	if got, want := c.LogErrCount(), c.MaxRetries; got != want {
+		t.Fatalf("got %d errors, want %d", got, want)
+	}
+
+	var startTime int64
+	for i, e := range c.ErrLog {
+		if i == 0 {
+			startTime = e.Time.Unix()
+			continue
+		}
+		if got, want := e.Time.Unix(), startTime+int64(i); got != want {
+			t.Errorf("got time %d, want %d (%d greater than start time %d)", got, want, i, startTime)
+		}
+	}
+
+}
+
+func TestLinearJitterBackoff(t *testing.T) {
+	t.Parallel()
+	c := pester.New()
+	c.Backoff = pester.LinearJitterBackoff
+	c.KeepLog = true
+
+	nonExistantURL := "http://localhost:9000/foo"
+
+	_, err := c.Get(nonExistantURL)
+	if err == nil {
+		t.Fatal("expected to get an error")
+	}
+	c.Wait()
+
+	// in the event of an error, let's see what the logs were
+	t.Log("\n", c.LogString())
+
+	var startTime int64
+	var delta int64
+	for i, e := range c.ErrLog {
+		switch i {
+		case 0:
+			startTime = e.Time.Unix()
+		case 1:
+			delta += 1
+		case 2:
+			delta += 2
+		case 3:
+			delta += 3
+		}
+
+		if got, want := e.Time.Unix(), startTime+delta; withinEpsilon(got, want, 0.0) {
+			t.Errorf("got time %d, want %d (within epsilon of start time %d)", got, want, startTime)
+		}
+	}
+}
+
+func TestExponentialBackoff(t *testing.T) {
+	t.Parallel()
+
+	c := pester.New()
+	c.MaxRetries = 4
+	c.Backoff = pester.ExponentialBackoff
+	c.KeepLog = true
+
+	nonExistantURL := "http://localhost:9000/foo"
+
+	_, err := c.Get(nonExistantURL)
+	if err == nil {
+		t.Fatal("expected to get an error")
+	}
+	c.Wait()
+
+	// in the event of an error, let's see what the logs were
+	t.Log("\n", c.LogString())
+
+	if got, want := c.LogErrCount(), c.MaxRetries; got != want {
+		t.Fatalf("got %d errors, want %d", got, want)
+	}
+
+	var startTime int64
+	var delta int64
+	for i, e := range c.ErrLog {
+		switch i {
+		case 0:
+			startTime = e.Time.Unix()
+		case 1:
+			delta += 2
+		case 2:
+			delta += 4
+		case 3:
+			delta += 8
+		}
+		if got, want := e.Time.Unix(), startTime+delta; got != want {
+			t.Errorf("got time %d, want %d (%d greater than start time %d)", got, want, delta, startTime)
+		}
+	}
+}
+
+func TestCookiesJarPersistence(t *testing.T) {
+	// make sure that client properties like .Jar are held onto through the request
+	port, err := cookieServer()
+	if err != nil {
+		t.Fatal("unable to start cookie server", err)
+	}
+
+	jar, err := cookiejar.New(nil)
+	if err != nil {
+		t.Fatal("Cannot create cookiejar", err)
+	}
+
+	c := pester.New()
+	c.Jar = jar
+
+	url := fmt.Sprintf("http://localhost:%d", port)
+
+	response, err := c.Get(url)
+	if err != nil {
+		t.Fatal("unable to GET", err)
+	}
+	c.Wait()
+
+	response.Body.Close()
+	if !strings.Contains(fmt.Sprintf("%v", jar), "mah-cookie nomnomnom") {
+		t.Error("unable to find expected cookie")
+	}
+}
+
+func TestEmbeddedClientTimeout(t *testing.T) {
+	// set up a server that will timeout
+	clientTimeout := 1000 * time.Millisecond
+	port, err := timeoutServer(2 * clientTimeout)
+	if err != nil {
+		t.Fatal("unable to start timeout server", err)
+	}
+
+	hc := http.DefaultClient
+	hc.Timeout = clientTimeout
+
+	c := pester.NewExtendedClient(hc)
+	_, err = c.Get(fmt.Sprintf("http://localhost:%d/", port))
+	if err == nil {
+		t.Error("expected a timeout error, did not get it")
+	}
+}
+
+func TestConcurrentRequestsNotRacyAndDontLeak_FailedRequest(t *testing.T) {
+	goroStart := runtime.NumGoroutine()
+	c := pester.New()
+	port, err := cookieServer()
+	if err != nil {
+		t.Fatalf("unable to start server %v", err)
+	}
+	goodURL := fmt.Sprintf("http://localhost:%d", port)
+	conc := 5
+	errCh := make(chan error, conc)
+
+	wg := &sync.WaitGroup{}
+	block := make(chan struct{})
+	for i := 0; i < conc; i++ {
+		wg.Add(1)
+		go func() {
+			<-block
+			defer wg.Done()
+			resp, err := c.Get(goodURL)
+			if err != nil {
+				errCh <- fmt.Errorf("got unexpected error getting %s, %v", goodURL, err)
+				return
+			}
+			if resp != nil {
+				resp.Body.Close()
+			}
+		}()
+	}
+	close(block)
+	go func() {
+		select {
+		case err := <-errCh:
+			t.Fatal(err)
+		case <-time.After(250 * time.Millisecond):
+			return
+		}
+	}()
+	wg.Wait()
+
+	// give background goroutines time to clean up
+	<-time.After(1000 * time.Millisecond)
+	goroEnd := runtime.NumGoroutine()
+	if goroStart != goroEnd {
+		t.Errorf("got %d running goroutines, want %d", goroEnd, goroStart)
+	}
+}
+
+func TestConcurrentRequestsNotRacyAndDontLeak_SuccessfulRequest(t *testing.T) {
+	goroStart := runtime.NumGoroutine()
+	c := pester.New()
+	nonExistantURL := "http://localhost:9000/foo"
+	conc := 5
+	errCh := make(chan error, conc)
+
+	wg := &sync.WaitGroup{}
+	block := make(chan struct{})
+	for i := 0; i < conc; i++ {
+		wg.Add(1)
+		go func() {
+			<-block
+			defer wg.Done()
+			resp, err := c.Get(nonExistantURL)
+			if err == nil {
+				errCh <- fmt.Errorf("should have had an error getting %s", nonExistantURL)
+				return
+			}
+			if resp != nil {
+				resp.Body.Close()
+			}
+		}()
+	}
+	close(block)
+	go func() {
+		select {
+		case err := <-errCh:
+			t.Fatal(err)
+		case <-time.After(250 * time.Millisecond):
+			return
+		}
+	}()
+	wg.Wait()
+
+	// give background goroutines time to clean up
+	<-time.After(250 * time.Millisecond)
+	goroEnd := runtime.NumGoroutine()
+	if goroStart != goroEnd {
+		t.Errorf("got %d running goroutines, want %d", goroEnd, goroStart)
+	}
+}
+
+func withinEpsilon(got, want int64, epslion float64) bool {
+	if want <= int64(epslion*float64(got)) || want >= int64(epslion*float64(got)) {
+		return false
+	}
+	return true
+}
+
+func cookieServer() (int, error) {
+	mux := http.NewServeMux()
+	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+		cookie := &http.Cookie{}
+		cookie.Name = "mah-cookie"
+		cookie.Value = "nomnomnom"
+		http.SetCookie(w, cookie)
+		w.Write([]byte("OK"))
+	})
+	l, err := net.Listen("tcp", ":0")
+	if err != nil {
+		return -1, fmt.Errorf("unable to secure listener %v", err)
+	}
+	go func() {
+		if err := http.Serve(l, mux); err != nil {
+			log.Fatalf("slow-server error %v", err)
+		}
+	}()
+	port, err := strconv.Atoi(strings.Replace(l.Addr().String(), "[::]:", "", 1))
+	if err != nil {
+		return -1, fmt.Errorf("unable to determine port %v", err)
+	}
+	return port, nil
+}
+
+func timeoutServer(timeout time.Duration) (int, error) {
+	mux := http.NewServeMux()
+	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+		<-time.After(timeout)
+		w.Write([]byte("OK"))
+	})
+	l, err := net.Listen("tcp", ":0")
+	if err != nil {
+		return -1, fmt.Errorf("unable to secure listener %v", err)
+	}
+	go func() {
+		if err := http.Serve(l, mux); err != nil {
+			log.Fatalf("slow-server error %v", err)
+		}
+	}()
+	port, err := strconv.Atoi(strings.Replace(l.Addr().String(), "[::]:", "", 1))
+	if err != nil {
+		return -1, fmt.Errorf("unable to determine port %v", err)
+	}
+	return port, nil
+}
@@ -0,0 +1,167 @@
+package main
+
+/*
+   We start up a rando response server that will give different response codes
+   and different response times to simulate poor network / service conditions.
+
+   The main function is cut into blocks to perserve variable scope and examples
+   of each pester function can be seen in action.
+
+   The server logs incoming requests while the main blocks log what they intend
+   to do and what they get back.
+*/
+
+import (
+	"flag"
+	"fmt"
+	"log"
+	"math/rand"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/sethgrid/pester"
+)
+
+func init() {
+	rand.Seed(time.Now().Unix())
+}
+
+func main() {
+	// set everything up
+	var port int
+	flag.IntVar(&port, "port", 9000, "set the port for the rando response server")
+	flag.Parse()
+
+	log.Printf("Starting a rando response server on :%d ...\n\n", port)
+
+	go func() {
+		http.HandleFunc("/", randoHandler)
+		log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", port), nil))
+	}()
+
+	//////////////////////////////////////////////////////
+	// begin running through each of the pestor methods //
+	//////////////////////////////////////////////////////
+
+	log.Println("> pester.Get default")
+	{ // drop in replacement for http.Get and other client methods
+		resp, err := pester.Get(fmt.Sprintf("http://localhost:%d", port))
+		if err != nil {
+			log.Fatalf("error GETing default", err)
+		}
+		defer resp.Body.Close()
+
+		log.Printf("GET :%d %s \n\n", port, resp.Status)
+	}
+
+	log.Println("> pester.Get with set backoff stategy, concurrency and retries increased")
+	{ // control the resiliency
+		client := pester.New()
+		client.Concurrency = 3
+		client.MaxRetries = 5
+		client.Backoff = pester.ExponentialJitterBackoff
+		client.KeepLog = true
+
+		resp, err := client.Get(fmt.Sprintf("http://localhost:%d", port))
+		if err != nil {
+			log.Fatalf("error GETing with all options, %s\n\n", client.LogString())
+		}
+		defer resp.Body.Close()
+
+		log.Printf("Exponential Jitter Backoff :%d %s [request %d, retry %d]\n\n", port, resp.Status, client.SuccessReqNum, client.SuccessRetryNum)
+	}
+
+	log.Println("> pester.Get with custom backoff strategy")
+	{ // set a custom backoff strategy
+		client := pester.New()
+		client.Backoff = func(retry int) time.Duration {
+			return time.Duration(retry*200) * time.Millisecond
+		}
+		client.Timeout = 5 * time.Second
+		client.KeepLog = true
+
+		resp, err := client.Get(fmt.Sprintf("http://localhost:%d", port))
+		if err != nil {
+			log.Fatalf("error GETing custom backoff\n\n", client.LogString())
+		}
+		defer resp.Body.Close()
+
+		log.Printf("Custom backoff :%d %s [request %d, retry %d]\n\n", port, resp.Status, client.SuccessReqNum, client.SuccessRetryNum)
+	}
+
+	log.Println("> pester.Post with defaults")
+	{ // use the pester.Post drop in replacement
+		resp, err := pester.Post(fmt.Sprintf("http://localhost:%d", port), "text/plain", strings.NewReader("data"))
+		if err != nil {
+			log.Fatalf("error POSTing with defaults - %v\n\n", err)
+		}
+		defer resp.Body.Close()
+
+		log.Printf("POST :%d %s\n\n", port, resp.Status)
+	}
+
+	log.Println("> pester.Head with defaults")
+	{ // use the pester.Head drop in replacement
+		resp, err := pester.Head(fmt.Sprintf("http://localhost:%d", port))
+		if err != nil {
+			log.Fatalf("error HEADing with defaults - %v\n\n", err)
+		}
+		defer resp.Body.Close()
+
+		log.Printf("HEAD :%d %s\n\n", port, resp.Status)
+	}
+
+	log.Println("> pester.PostForm with defaults")
+	{ // use the pester.Head drop in replacement
+		resp, err := pester.PostForm(fmt.Sprintf("http://localhost:%d", port), url.Values{"param1": []string{"val1a", "val1b"}, "param2": []string{"val2"}})
+		if err != nil {
+			log.Fatalf("error POSTing a form with defaults - %v\n\n", err)
+		}
+		defer resp.Body.Close()
+
+		log.Printf("POST (form) :%d %s\n\n", port, resp.Status)
+	}
+
+	log.Println("> pester Do with POST")
+	{ // use the pester version of http.Client.Do
+		req, err := http.NewRequest("POST", fmt.Sprintf("http://localhost:%d", port), strings.NewReader("data"))
+		if err != nil {
+			log.Fatal("Unable to create a new http request", err)
+		}
+		resp, err := pester.Do(req)
+		if err != nil {
+			log.Fatalf("error POSTing with Do() - %v\n\n", err)
+		}
+		defer resp.Body.Close()
+
+		log.Printf("Do() POST :%d %s\n\n", port, resp.Status)
+	}
+
+}
+
+// randoHandler will cause random delays and give random status responses
+func randoHandler(w http.ResponseWriter, r *http.Request) {
+	delay := rand.Intn(5000)
+	var code int
+	switch rand.Intn(10) {
+	case 0:
+		code = 404
+	case 1:
+		code = 400
+	case 2:
+		code = 501
+	case 3:
+		code = 500
+	default:
+		code = 200
+	}
+
+	log.Printf("incoming request on :9000 - will return %d in %d ms", code, delay)
+
+	<-time.Tick(time.Duration(delay) * time.Millisecond)
+
+	w.WriteHeader(code)
+	w.Write([]byte(http.StatusText(code)))
+}