plugin/forward: Increase minimum read timeout to 200ms (#1889)

After several experiments at SoundCloud we found that the current
minimum read timeout of 10ms is too low. A single request against a
slow/unavailable authoritative server can cause all TCP connections to
get closed. We record a 50th percentile forward/proxy latency of <5ms,
and a 99th percentile latency of 60ms. Using a minimum timeout of 200ms
seems to be a fair trade-off between avoiding unnecessary high
connection churn and reacting to upstream failures in a timely manner.

This change also renames hcDuration to hcInterval to reflect its usage,
and removes the duplicated timeout constant to make code comprehension
easier.

plugin/forward/connect.go

@@ -97,7 +97,7 @@ func (p *Proxy) Connect(ctx context.Context, state request.Request, forceTCP, me
 		conn.UDPSize = 512
 	}
 
-	conn.SetWriteDeadline(time.Now().Add(timeout))
+	conn.SetWriteDeadline(time.Now().Add(maxTimeout))
 	reqTime := time.Now()
 	if err := conn.WriteMsg(state.Req); err != nil {
 		conn.Close() // not giving it back
@@ -110,7 +110,7 @@ func (p *Proxy) Connect(ctx context.Context, state request.Request, forceTCP, me
 	conn.SetReadDeadline(time.Now().Add(p.readTimeout()))
 	ret, err := conn.ReadMsg()
 	if err != nil {
-		p.updateRtt(timeout)
+		p.updateRtt(maxTimeout)
 		conn.Close() // not giving it back
 		if err == io.EOF && cached {
 			return nil, ErrCachedClosed