Skip to content

Commit 906cda9

Browse files
tomberganbradfitz
tombergan
authored and
bradfitz
committed
http2: add configurable knobs for the server's receive window
Upload performance is poor when BDP is higher than the flow-control window. Previously, the server's receive window was fixed at 64KB, which resulted in very poor performance for high-BDP links. The receive window now defaults to 1MB and is configurable. The per-connection and per-stream windows are configurable separately (both default to 1MB as suggested in golang/go#16512). Previously, the server created a "fixedBuffer" for each request body. This is no longer a good idea because a fixedBuffer has fixed size, which means individual streams cannot use varying amounts of the available connection window. To overcome this limitation, I replaced fixedBuffer with "dataBuffer", which grows and shrinks based on current usage. The worst-case fragmentation of dataBuffer is 32KB wasted memory per stream, but I expect that worst-case will be rare. A slightly modified version of adg@'s grpcbench program shows a dramatic improvement when increasing from a 64KB window to a 1MB window, especially at higher latencies (i.e., higher BDPs). Network latency was simulated with netem, e.g., `tc qdisc add dev lo root netem delay 16ms`. Duration Latency Proto H2 Window 11ms±4.05ms 0s HTTP/1.1 - 17ms±1.95ms 0s HTTP/2.0 65535 8ms±1.75ms 0s HTTP/2.0 1048576 10ms±1.49ms 1ms HTTP/1.1 - 47ms±2.91ms 1ms HTTP/2.0 65535 10ms±1.77ms 1ms HTTP/2.0 1048576 15ms±1.69ms 2ms HTTP/1.1 - 88ms±11.29ms 2ms HTTP/2.0 65535 15ms±1.18ms 2ms HTTP/2.0 1048576 23ms±1.42ms 4ms HTTP/1.1 - 152ms±0.77ms 4ms HTTP/2.0 65535 23ms±0.94ms 4ms HTTP/2.0 1048576 40ms±1.54ms 8ms HTTP/1.1 - 288ms±1.67ms 8ms HTTP/2.0 65535 39ms±1.29ms 8ms HTTP/2.0 1048576 72ms±1.13ms 16ms HTTP/1.1 - 559ms±0.68ms 16ms HTTP/2.0 65535 71ms±1.12ms 16ms HTTP/2.0 1048576 136ms±1.15ms 32ms HTTP/1.1 - 1104ms±1.62ms 32ms HTTP/2.0 65535 135ms±0.96ms 32ms HTTP/2.0 1048576 264ms±0.95ms 64ms HTTP/1.1 - 2191ms±2.08ms 64ms HTTP/2.0 65535 263ms±1.57ms 64ms HTTP/2.0 1048576 Fixes golang/go#16512 Updates golang/go#17985 Updates golang/go#18404 Change-Id: Ied385aa94588337e98dad9475cf2ece2f39ba346 Reviewed-on: https://go-review.googlesource.com/37226 Reviewed-by: Brad Fitzpatrick <bradfitz@golang.org> Run-TryBot: Brad Fitzpatrick <bradfitz@golang.org> TryBot-Result: Gobot Gobot <gobot@golang.org>
1 parent bce15e7 commit 906cda9

File tree

2 files changed

+137
-73
lines changed

2 files changed

+137
-73
lines changed

‎http2/server.go

+92-60
Original file line numberDiff line numberDiff line change
@@ -110,11 +110,38 @@ type Server struct {
110110
// activity for the purposes of IdleTimeout.
111111
IdleTimeout time.Duration
112112

113+
// MaxUploadBufferPerConnection is the size of the initial flow
114+
// control window for each connections. The HTTP/2 spec does not
115+
// allow this to be smaller than 65535 or larger than 2^32-1.
116+
// If the value is outside this range, a default value will be
117+
// used instead.
118+
MaxUploadBufferPerConnection int32
119+
120+
// MaxUploadBufferPerStream is the size of the initial flow control
121+
// window for each stream. The HTTP/2 spec does not allow this to
122+
// be larger than 2^32-1. If the value is zero or larger than the
123+
// maximum, a default value will be used instead.
124+
MaxUploadBufferPerStream int32
125+
113126
// NewWriteScheduler constructs a write scheduler for a connection.
114127
// If nil, a default scheduler is chosen.
115128
NewWriteScheduler func() WriteScheduler
116129
}
117130

131+
func (s *Server) initialConnRecvWindowSize() int32 {
132+
if s.MaxUploadBufferPerConnection > initialWindowSize {
133+
return s.MaxUploadBufferPerConnection
134+
}
135+
return 1 << 20
136+
}
137+
138+
func (s *Server) initialStreamRecvWindowSize() int32 {
139+
if s.MaxUploadBufferPerStream > 0 {
140+
return s.MaxUploadBufferPerStream
141+
}
142+
return 1 << 20
143+
}
144+
118145
func (s *Server) maxReadFrameSize() uint32 {
119146
if v := s.MaxReadFrameSize; v >= minMaxFrameSize && v <= maxFrameSize {
120147
return v
@@ -255,27 +282,27 @@ func (s *Server) ServeConn(c net.Conn, opts *ServeConnOpts) {
255282
defer cancel()
256283

257284
sc := &serverConn{
258-
srv: s,
259-
hs: opts.baseConfig(),
260-
conn: c,
261-
baseCtx: baseCtx,
262-
remoteAddrStr: c.RemoteAddr().String(),
263-
bw: newBufferedWriter(c),
264-
handler: opts.handler(),
265-
streams: make(map[uint32]*stream),
266-
readFrameCh: make(chan readFrameResult),
267-
wantWriteFrameCh: make(chan FrameWriteRequest, 8),
268-
wantStartPushCh: make(chan startPushRequest, 8),
269-
wroteFrameCh: make(chan frameWriteResult, 1), // buffered; one send in writeFrameAsync
270-
bodyReadCh: make(chan bodyReadMsg), // buffering doesn't matter either way
271-
doneServing: make(chan struct{}),
272-
clientMaxStreams: math.MaxUint32, // Section 6.5.2: "Initially, there is no limit to this value"
273-
advMaxStreams: s.maxConcurrentStreams(),
274-
initialWindowSize: initialWindowSize,
275-
maxFrameSize: initialMaxFrameSize,
276-
headerTableSize: initialHeaderTableSize,
277-
serveG: newGoroutineLock(),
278-
pushEnabled: true,
285+
srv: s,
286+
hs: opts.baseConfig(),
287+
conn: c,
288+
baseCtx: baseCtx,
289+
remoteAddrStr: c.RemoteAddr().String(),
290+
bw: newBufferedWriter(c),
291+
handler: opts.handler(),
292+
streams: make(map[uint32]*stream),
293+
readFrameCh: make(chan readFrameResult),
294+
wantWriteFrameCh: make(chan FrameWriteRequest, 8),
295+
wantStartPushCh: make(chan startPushRequest, 8),
296+
wroteFrameCh: make(chan frameWriteResult, 1), // buffered; one send in writeFrameAsync
297+
bodyReadCh: make(chan bodyReadMsg), // buffering doesn't matter either way
298+
doneServing: make(chan struct{}),
299+
clientMaxStreams: math.MaxUint32, // Section 6.5.2: "Initially, there is no limit to this value"
300+
advMaxStreams: s.maxConcurrentStreams(),
301+
initialStreamSendWindowSize: initialWindowSize,
302+
maxFrameSize: initialMaxFrameSize,
303+
headerTableSize: initialHeaderTableSize,
304+
serveG: newGoroutineLock(),
305+
pushEnabled: true,
279306
}
280307

281308
// The net/http package sets the write deadline from the
@@ -294,6 +321,9 @@ func (s *Server) ServeConn(c net.Conn, opts *ServeConnOpts) {
294321
sc.writeSched = NewRandomWriteScheduler()
295322
}
296323

324+
// These start at the RFC-specified defaults. If there is a higher
325+
// configured value for inflow, that will be updated when we send a
326+
// WINDOW_UPDATE shortly after sending SETTINGS.
297327
sc.flow.add(initialWindowSize)
298328
sc.inflow.add(initialWindowSize)
299329
sc.hpackEncoder = hpack.NewEncoder(&sc.headerWriteBuf)
@@ -387,34 +417,34 @@ type serverConn struct {
387417
writeSched WriteScheduler
388418

389419
// Everything following is owned by the serve loop; use serveG.check():
390-
serveG goroutineLock // used to verify funcs are on serve()
391-
pushEnabled bool
392-
sawFirstSettings bool // got the initial SETTINGS frame after the preface
393-
needToSendSettingsAck bool
394-
unackedSettings int // how many SETTINGS have we sent without ACKs?
395-
clientMaxStreams uint32 // SETTINGS_MAX_CONCURRENT_STREAMS from client (our PUSH_PROMISE limit)
396-
advMaxStreams uint32 // our SETTINGS_MAX_CONCURRENT_STREAMS advertised the client
397-
curClientStreams uint32 // number of open streams initiated by the client
398-
curPushedStreams uint32 // number of open streams initiated by server push
399-
maxClientStreamID uint32 // max ever seen from client (odd), or 0 if there have been no client requests
400-
maxPushPromiseID uint32 // ID of the last push promise (even), or 0 if there have been no pushes
401-
streams map[uint32]*stream
402-
initialWindowSize int32
403-
maxFrameSize int32
404-
headerTableSize uint32
405-
peerMaxHeaderListSize uint32 // zero means unknown (default)
406-
canonHeader map[string]string // http2-lower-case -> Go-Canonical-Case
407-
writingFrame bool // started writing a frame (on serve goroutine or separate)
408-
writingFrameAsync bool // started a frame on its own goroutine but haven't heard back on wroteFrameCh
409-
needsFrameFlush bool // last frame write wasn't a flush
410-
inGoAway bool // we've started to or sent GOAWAY
411-
inFrameScheduleLoop bool // whether we're in the scheduleFrameWrite loop
412-
needToSendGoAway bool // we need to schedule a GOAWAY frame write
413-
goAwayCode ErrCode
414-
shutdownTimerCh <-chan time.Time // nil until used
415-
shutdownTimer *time.Timer // nil until used
416-
idleTimer *time.Timer // nil if unused
417-
idleTimerCh <-chan time.Time // nil if unused
420+
serveG goroutineLock // used to verify funcs are on serve()
421+
pushEnabled bool
422+
sawFirstSettings bool // got the initial SETTINGS frame after the preface
423+
needToSendSettingsAck bool
424+
unackedSettings int // how many SETTINGS have we sent without ACKs?
425+
clientMaxStreams uint32 // SETTINGS_MAX_CONCURRENT_STREAMS from client (our PUSH_PROMISE limit)
426+
advMaxStreams uint32 // our SETTINGS_MAX_CONCURRENT_STREAMS advertised the client
427+
curClientStreams uint32 // number of open streams initiated by the client
428+
curPushedStreams uint32 // number of open streams initiated by server push
429+
maxClientStreamID uint32 // max ever seen from client (odd), or 0 if there have been no client requests
430+
maxPushPromiseID uint32 // ID of the last push promise (even), or 0 if there have been no pushes
431+
streams map[uint32]*stream
432+
initialStreamSendWindowSize int32
433+
maxFrameSize int32
434+
headerTableSize uint32
435+
peerMaxHeaderListSize uint32 // zero means unknown (default)
436+
canonHeader map[string]string // http2-lower-case -> Go-Canonical-Case
437+
writingFrame bool // started writing a frame (on serve goroutine or separate)
438+
writingFrameAsync bool // started a frame on its own goroutine but haven't heard back on wroteFrameCh
439+
needsFrameFlush bool // last frame write wasn't a flush
440+
inGoAway bool // we've started to or sent GOAWAY
441+
inFrameScheduleLoop bool // whether we're in the scheduleFrameWrite loop
442+
needToSendGoAway bool // we need to schedule a GOAWAY frame write
443+
goAwayCode ErrCode
444+
shutdownTimerCh <-chan time.Time // nil until used
445+
shutdownTimer *time.Timer // nil until used
446+
idleTimer *time.Timer // nil if unused
447+
idleTimerCh <-chan time.Time // nil if unused
418448

419449
// Owned by the writeFrameAsync goroutine:
420450
headerWriteBuf bytes.Buffer
@@ -695,15 +725,17 @@ func (sc *serverConn) serve() {
695725
{SettingMaxFrameSize, sc.srv.maxReadFrameSize()},
696726
{SettingMaxConcurrentStreams, sc.advMaxStreams},
697727
{SettingMaxHeaderListSize, sc.maxHeaderListSize()},
698-
699-
// TODO: more actual settings, notably
700-
// SettingInitialWindowSize, but then we also
701-
// want to bump up the conn window size the
702-
// same amount here right after the settings
728+
{SettingInitialWindowSize, uint32(sc.srv.initialStreamRecvWindowSize())},
703729
},
704730
})
705731
sc.unackedSettings++
706732

733+
// Each connection starts with intialWindowSize inflow tokens.
734+
// If a higher value is configured, we add more tokens.
735+
if diff := sc.srv.initialConnRecvWindowSize() - initialWindowSize; diff > 0 {
736+
sc.sendWindowUpdate(nil, int(diff))
737+
}
738+
707739
if err := sc.readPreface(); err != nil {
708740
sc.condlogf(err, "http2: server: error reading preface from client %v: %v", sc.conn.RemoteAddr(), err)
709741
return
@@ -1394,9 +1426,9 @@ func (sc *serverConn) processSettingInitialWindowSize(val uint32) error {
13941426
// adjust the size of all stream flow control windows that it
13951427
// maintains by the difference between the new value and the
13961428
// old value."
1397-
old := sc.initialWindowSize
1398-
sc.initialWindowSize = int32(val)
1399-
growth := sc.initialWindowSize - old // may be negative
1429+
old := sc.initialStreamSendWindowSize
1430+
sc.initialStreamSendWindowSize = int32(val)
1431+
growth := int32(val) - old // may be negative
14001432
for _, st := range sc.streams {
14011433
if !st.flow.add(growth) {
14021434
// 6.9.2 Initial Flow Control Window Size
@@ -1718,9 +1750,9 @@ func (sc *serverConn) newStream(id, pusherID uint32, state streamState) *stream
17181750
}
17191751
st.cw.Init()
17201752
st.flow.conn = &sc.flow // link to conn-level counter
1721-
st.flow.add(sc.initialWindowSize)
1722-
st.inflow.conn = &sc.inflow // link to conn-level counter
1723-
st.inflow.add(initialWindowSize) // TODO: update this when we send a higher initial window size in the initial settings
1753+
st.flow.add(sc.initialStreamSendWindowSize)
1754+
st.inflow.conn = &sc.inflow // link to conn-level counter
1755+
st.inflow.add(sc.srv.initialStreamRecvWindowSize())
17241756

17251757
sc.streams[id] = st
17261758
sc.writeSched.OpenStream(st.id, OpenStreamOptions{PusherID: pusherID})

‎http2/server_test.go

+45-13
Original file line numberDiff line numberDiff line change
@@ -260,11 +260,52 @@ func (st *serverTester) Close() {
260260
// greet initiates the client's HTTP/2 connection into a state where
261261
// frames may be sent.
262262
func (st *serverTester) greet() {
263+
st.greetAndCheckSettings(func(Setting) error { return nil })
264+
}
265+
266+
func (st *serverTester) greetAndCheckSettings(checkSetting func(s Setting) error) {
263267
st.writePreface()
264268
st.writeInitialSettings()
265-
st.wantSettings()
269+
st.wantSettings().ForeachSetting(checkSetting)
266270
st.writeSettingsAck()
267-
st.wantSettingsAck()
271+
272+
// The initial WINDOW_UPDATE and SETTINGS ACK can come in any order.
273+
var gotSettingsAck bool
274+
var gotWindowUpdate bool
275+
276+
for i := 0; i < 2; i++ {
277+
f, err := st.readFrame()
278+
if err != nil {
279+
st.t.Fatal(err)
280+
}
281+
switch f := f.(type) {
282+
case *SettingsFrame:
283+
if !f.Header().Flags.Has(FlagSettingsAck) {
284+
st.t.Fatal("Settings Frame didn't have ACK set")
285+
}
286+
gotSettingsAck = true
287+
288+
case *WindowUpdateFrame:
289+
if f.FrameHeader.StreamID != 0 {
290+
st.t.Fatalf("WindowUpdate StreamID = %d; want 0", f.FrameHeader.StreamID, 0)
291+
}
292+
incr := uint32((&Server{}).initialConnRecvWindowSize() - initialWindowSize)
293+
if f.Increment != incr {
294+
st.t.Fatalf("WindowUpdate increment = %d; want %d", f.Increment, incr)
295+
}
296+
gotWindowUpdate = true
297+
298+
default:
299+
st.t.Fatalf("Wanting a settings ACK or window update, received a %T", f)
300+
}
301+
}
302+
303+
if !gotSettingsAck {
304+
st.t.Fatalf("Didn't get a settings ACK")
305+
}
306+
if !gotWindowUpdate {
307+
st.t.Fatalf("Didn't get a window update")
308+
}
268309
}
269310

270311
func (st *serverTester) writePreface() {
@@ -584,12 +625,7 @@ func TestServer(t *testing.T) {
584625
server sends in the HTTP/2 connection.
585626
`)
586627

587-
st.writePreface()
588-
st.writeInitialSettings()
589-
st.wantSettings()
590-
st.writeSettingsAck()
591-
st.wantSettingsAck()
592-
628+
st.greet()
593629
st.writeHeaders(HeadersFrameParam{
594630
StreamID: 1, // clients send odd numbers
595631
BlockFragment: st.encodeHeader(),
@@ -2601,11 +2637,9 @@ func TestServerDoS_MaxHeaderListSize(t *testing.T) {
26012637
defer st.Close()
26022638

26032639
// shake hands
2604-
st.writePreface()
2605-
st.writeInitialSettings()
26062640
frameSize := defaultMaxReadFrameSize
26072641
var advHeaderListSize *uint32
2608-
st.wantSettings().ForeachSetting(func(s Setting) error {
2642+
st.greetAndCheckSettings(func(s Setting) error {
26092643
switch s.ID {
26102644
case SettingMaxFrameSize:
26112645
if s.Val < minMaxFrameSize {
@@ -2620,8 +2654,6 @@ func TestServerDoS_MaxHeaderListSize(t *testing.T) {
26202654
}
26212655
return nil
26222656
})
2623-
st.writeSettingsAck()
2624-
st.wantSettingsAck()
26252657

26262658
if advHeaderListSize == nil {
26272659
t.Errorf("server didn't advertise a max header list size")

0 commit comments

Comments
 (0)