`, - 5: `. . . "2"`, - 6: `. . . `, - 7: `. . . . "3"`, - 8: `. . . `, - 9: `. . . . "&4"`, + 0: ``, + 1: `. `, + 2: `. `, + 3: `. . "0<1"`, + 4: `. .
`,
+ 5: `. . . "2"`,
+ 6: `. . . `,
+ 7: `. . . . "3"`,
+ 8: `. . . `,
+ 9: `. . . . "&4"`,
10: `. . "5"`,
11: `. . Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages. Alternative Proxies:`,
12: `. .
`,
@@ -169,3 +171,37 @@ func TestRenderer(t *testing.T) {
t.Errorf("got vs want:\n%s\n%s\n", got, want)
}
}
+
+func TestRenderTextNodes(t *testing.T) {
+ elements := []string{"style", "script", "xmp", "iframe", "noembed", "noframes", "plaintext", "noscript"}
+ for _, namespace := range []string{
+ "", // html
+ "svg",
+ "math",
+ } {
+ for _, e := range elements {
+ var namespaceOpen, namespaceClose string
+ if namespace != "" {
+ namespaceOpen, namespaceClose = fmt.Sprintf("<%s>", namespace), fmt.Sprintf("", namespace)
+ }
+ doc := fmt.Sprintf(`%s<%s>&%s`, namespaceOpen, e, e, namespaceClose)
+ n, err := Parse(strings.NewReader(doc))
+ if err != nil {
+ t.Fatal(err)
+ }
+ b := bytes.NewBuffer(nil)
+ if err := Render(b, n); err != nil {
+ t.Fatal(err)
+ }
+
+ expected := doc
+ if namespace != "" {
+ expected = strings.Replace(expected, "&", "&", 1)
+ }
+
+ if b.String() != expected {
+ t.Errorf("unexpected output: got %q, want %q", b.String(), expected)
+ }
+ }
+ }
+}
diff --git a/http2/Dockerfile b/http2/Dockerfile
deleted file mode 100644
index 8512245952..0000000000
--- a/http2/Dockerfile
+++ /dev/null
@@ -1,51 +0,0 @@
-#
-# This Dockerfile builds a recent curl with HTTP/2 client support, using
-# a recent nghttp2 build.
-#
-# See the Makefile for how to tag it. If Docker and that image is found, the
-# Go tests use this curl binary for integration tests.
-#
-
-FROM ubuntu:trusty
-
-RUN apt-get update && \
- apt-get upgrade -y && \
- apt-get install -y git-core build-essential wget
-
-RUN apt-get install -y --no-install-recommends \
- autotools-dev libtool pkg-config zlib1g-dev \
- libcunit1-dev libssl-dev libxml2-dev libevent-dev \
- automake autoconf
-
-# The list of packages nghttp2 recommends for h2load:
-RUN apt-get install -y --no-install-recommends make binutils \
- autoconf automake autotools-dev \
- libtool pkg-config zlib1g-dev libcunit1-dev libssl-dev libxml2-dev \
- libev-dev libevent-dev libjansson-dev libjemalloc-dev \
- cython python3.4-dev python-setuptools
-
-# Note: setting NGHTTP2_VER before the git clone, so an old git clone isn't cached:
-ENV NGHTTP2_VER 895da9a
-RUN cd /root && git clone https://github.com/tatsuhiro-t/nghttp2.git
-
-WORKDIR /root/nghttp2
-RUN git reset --hard $NGHTTP2_VER
-RUN autoreconf -i
-RUN automake
-RUN autoconf
-RUN ./configure
-RUN make
-RUN make install
-
-WORKDIR /root
-RUN wget https://curl.se/download/curl-7.45.0.tar.gz
-RUN tar -zxvf curl-7.45.0.tar.gz
-WORKDIR /root/curl-7.45.0
-RUN ./configure --with-ssl --with-nghttp2=/usr/local
-RUN make
-RUN make install
-RUN ldconfig
-
-CMD ["-h"]
-ENTRYPOINT ["/usr/local/bin/curl"]
-
diff --git a/http2/Makefile b/http2/Makefile
deleted file mode 100644
index 55fd826f77..0000000000
--- a/http2/Makefile
+++ /dev/null
@@ -1,3 +0,0 @@
-curlimage:
- docker build -t gohttp2/curl .
-
diff --git a/http2/http2_test.go b/http2/http2_test.go
index f77c08a107..a16774b7ff 100644
--- a/http2/http2_test.go
+++ b/http2/http2_test.go
@@ -6,16 +6,13 @@ package http2
import (
"bytes"
- "errors"
"flag"
"fmt"
"io/ioutil"
"net/http"
"os"
- "os/exec"
"path/filepath"
"regexp"
- "strconv"
"strings"
"testing"
"time"
@@ -85,44 +82,6 @@ func encodeHeaderNoImplicit(t *testing.T, headers ...string) []byte {
return buf.Bytes()
}
-// Verify that curl has http2.
-func requireCurl(t *testing.T) {
- out, err := dockerLogs(curl(t, "--version"))
- if err != nil {
- t.Skipf("failed to determine curl features; skipping test")
- }
- if !strings.Contains(string(out), "HTTP2") {
- t.Skip("curl doesn't support HTTP2; skipping test")
- }
-}
-
-func curl(t *testing.T, args ...string) (container string) {
- out, err := exec.Command("docker", append([]string{"run", "-d", "--net=host", "gohttp2/curl"}, args...)...).Output()
- if err != nil {
- t.Skipf("Failed to run curl in docker: %v, %s", err, out)
- }
- return strings.TrimSpace(string(out))
-}
-
-// Verify that h2load exists.
-func requireH2load(t *testing.T) {
- out, err := dockerLogs(h2load(t, "--version"))
- if err != nil {
- t.Skipf("failed to probe h2load; skipping test: %s", out)
- }
- if !strings.Contains(string(out), "h2load nghttp2/") {
- t.Skipf("h2load not present; skipping test. (Output=%q)", out)
- }
-}
-
-func h2load(t *testing.T, args ...string) (container string) {
- out, err := exec.Command("docker", append([]string{"run", "-d", "--net=host", "--entrypoint=/usr/local/bin/h2load", "gohttp2/curl"}, args...)...).Output()
- if err != nil {
- t.Skipf("Failed to run h2load in docker: %v, %s", err, out)
- }
- return strings.TrimSpace(string(out))
-}
-
type puppetCommand struct {
fn func(w http.ResponseWriter, r *http.Request)
done chan<- bool
@@ -151,27 +110,6 @@ func (p *handlerPuppet) do(fn func(http.ResponseWriter, *http.Request)) {
p.ch <- puppetCommand{fn, done}
<-done
}
-func dockerLogs(container string) ([]byte, error) {
- out, err := exec.Command("docker", "wait", container).CombinedOutput()
- if err != nil {
- return out, err
- }
- exitStatus, err := strconv.Atoi(strings.TrimSpace(string(out)))
- if err != nil {
- return out, errors.New("unexpected exit status from docker wait")
- }
- out, err = exec.Command("docker", "logs", container).CombinedOutput()
- exec.Command("docker", "rm", container).Run()
- if err == nil && exitStatus != 0 {
- err = fmt.Errorf("exit status %d: %s", exitStatus, out)
- }
- return out, err
-}
-
-func kill(container string) {
- exec.Command("docker", "kill", container).Run()
- exec.Command("docker", "rm", container).Run()
-}
func cleanDate(res *http.Response) {
if d := res.Header["Date"]; len(d) == 1 {
diff --git a/http2/server.go b/http2/server.go
index 033b6e6db6..02c88b6b3e 100644
--- a/http2/server.go
+++ b/http2/server.go
@@ -581,9 +581,11 @@ type serverConn struct {
advMaxStreams uint32 // our SETTINGS_MAX_CONCURRENT_STREAMS advertised the client
curClientStreams uint32 // number of open streams initiated by the client
curPushedStreams uint32 // number of open streams initiated by server push
+ curHandlers uint32 // number of running handler goroutines
maxClientStreamID uint32 // max ever seen from client (odd), or 0 if there have been no client requests
maxPushPromiseID uint32 // ID of the last push promise (even), or 0 if there have been no pushes
streams map[uint32]*stream
+ unstartedHandlers []unstartedHandler
initialStreamSendWindowSize int32
maxFrameSize int32
peerMaxHeaderListSize uint32 // zero means unknown (default)
@@ -981,6 +983,8 @@ func (sc *serverConn) serve() {
return
case gracefulShutdownMsg:
sc.startGracefulShutdownInternal()
+ case handlerDoneMsg:
+ sc.handlerDone()
default:
panic("unknown timer")
}
@@ -1012,14 +1016,6 @@ func (sc *serverConn) serve() {
}
}
-func (sc *serverConn) awaitGracefulShutdown(sharedCh <-chan struct{}, privateCh chan struct{}) {
- select {
- case <-sc.doneServing:
- case <-sharedCh:
- close(privateCh)
- }
-}
-
type serverMessage int
// Message values sent to serveMsgCh.
@@ -1028,6 +1024,7 @@ var (
idleTimerMsg = new(serverMessage)
shutdownTimerMsg = new(serverMessage)
gracefulShutdownMsg = new(serverMessage)
+ handlerDoneMsg = new(serverMessage)
)
func (sc *serverConn) onSettingsTimer() { sc.sendServeMsg(settingsTimerMsg) }
@@ -1900,9 +1897,11 @@ func (st *stream) copyTrailersToHandlerRequest() {
// onReadTimeout is run on its own goroutine (from time.AfterFunc)
// when the stream's ReadTimeout has fired.
func (st *stream) onReadTimeout() {
- // Wrap the ErrDeadlineExceeded to avoid callers depending on us
- // returning the bare error.
- st.body.CloseWithError(fmt.Errorf("%w", os.ErrDeadlineExceeded))
+ if st.body != nil {
+ // Wrap the ErrDeadlineExceeded to avoid callers depending on us
+ // returning the bare error.
+ st.body.CloseWithError(fmt.Errorf("%w", os.ErrDeadlineExceeded))
+ }
}
// onWriteTimeout is run on its own goroutine (from time.AfterFunc)
@@ -2020,13 +2019,10 @@ func (sc *serverConn) processHeaders(f *MetaHeadersFrame) error {
// (in Go 1.8), though. That's a more sane option anyway.
if sc.hs.ReadTimeout != 0 {
sc.conn.SetReadDeadline(time.Time{})
- if st.body != nil {
- st.readDeadline = time.AfterFunc(sc.hs.ReadTimeout, st.onReadTimeout)
- }
+ st.readDeadline = time.AfterFunc(sc.hs.ReadTimeout, st.onReadTimeout)
}
- go sc.runHandler(rw, req, handler)
- return nil
+ return sc.scheduleHandler(id, rw, req, handler)
}
func (sc *serverConn) upgradeRequest(req *http.Request) {
@@ -2046,6 +2042,10 @@ func (sc *serverConn) upgradeRequest(req *http.Request) {
sc.conn.SetReadDeadline(time.Time{})
}
+ // This is the first request on the connection,
+ // so start the handler directly rather than going
+ // through scheduleHandler.
+ sc.curHandlers++
go sc.runHandler(rw, req, sc.handler.ServeHTTP)
}
@@ -2286,8 +2286,62 @@ func (sc *serverConn) newResponseWriter(st *stream, req *http.Request) *response
return &responseWriter{rws: rws}
}
+type unstartedHandler struct {
+ streamID uint32
+ rw *responseWriter
+ req *http.Request
+ handler func(http.ResponseWriter, *http.Request)
+}
+
+// scheduleHandler starts a handler goroutine,
+// or schedules one to start as soon as an existing handler finishes.
+func (sc *serverConn) scheduleHandler(streamID uint32, rw *responseWriter, req *http.Request, handler func(http.ResponseWriter, *http.Request)) error {
+ sc.serveG.check()
+ maxHandlers := sc.advMaxStreams
+ if sc.curHandlers < maxHandlers {
+ sc.curHandlers++
+ go sc.runHandler(rw, req, handler)
+ return nil
+ }
+ if len(sc.unstartedHandlers) > int(4*sc.advMaxStreams) {
+ return sc.countError("too_many_early_resets", ConnectionError(ErrCodeEnhanceYourCalm))
+ }
+ sc.unstartedHandlers = append(sc.unstartedHandlers, unstartedHandler{
+ streamID: streamID,
+ rw: rw,
+ req: req,
+ handler: handler,
+ })
+ return nil
+}
+
+func (sc *serverConn) handlerDone() {
+ sc.serveG.check()
+ sc.curHandlers--
+ i := 0
+ maxHandlers := sc.advMaxStreams
+ for ; i < len(sc.unstartedHandlers); i++ {
+ u := sc.unstartedHandlers[i]
+ if sc.streams[u.streamID] == nil {
+ // This stream was reset before its goroutine had a chance to start.
+ continue
+ }
+ if sc.curHandlers >= maxHandlers {
+ break
+ }
+ sc.curHandlers++
+ go sc.runHandler(u.rw, u.req, u.handler)
+ sc.unstartedHandlers[i] = unstartedHandler{} // don't retain references
+ }
+ sc.unstartedHandlers = sc.unstartedHandlers[i:]
+ if len(sc.unstartedHandlers) == 0 {
+ sc.unstartedHandlers = nil
+ }
+}
+
// Run on its own goroutine.
func (sc *serverConn) runHandler(rw *responseWriter, req *http.Request, handler func(http.ResponseWriter, *http.Request)) {
+ defer sc.sendServeMsg(handlerDoneMsg)
didPanic := true
defer func() {
rw.rws.stream.cancelCtx()
diff --git a/http2/server_test.go b/http2/server_test.go
index cd73291ea0..22657cbfe4 100644
--- a/http2/server_test.go
+++ b/http2/server_test.go
@@ -20,13 +20,11 @@ import (
"net/http"
"net/http/httptest"
"os"
- "os/exec"
"reflect"
"runtime"
"strconv"
"strings"
"sync"
- "sync/atomic"
"testing"
"time"
@@ -2704,96 +2702,6 @@ func readBodyHandler(t *testing.T, want string) func(w http.ResponseWriter, r *h
}
}
-// TestServerWithCurl currently fails, hence the LenientCipherSuites test. See:
-//
-// https://github.com/tatsuhiro-t/nghttp2/issues/140 &
-// http://sourceforge.net/p/curl/bugs/1472/
-func TestServerWithCurl(t *testing.T) { testServerWithCurl(t, false) }
-func TestServerWithCurl_LenientCipherSuites(t *testing.T) { testServerWithCurl(t, true) }
-
-func testServerWithCurl(t *testing.T, permitProhibitedCipherSuites bool) {
- if runtime.GOOS != "linux" {
- t.Skip("skipping Docker test when not on Linux; requires --net which won't work with boot2docker anyway")
- }
- if testing.Short() {
- t.Skip("skipping curl test in short mode")
- }
- requireCurl(t)
- var gotConn int32
- testHookOnConn = func() { atomic.StoreInt32(&gotConn, 1) }
-
- const msg = "Hello from curl!\n"
- ts := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- w.Header().Set("Foo", "Bar")
- w.Header().Set("Client-Proto", r.Proto)
- io.WriteString(w, msg)
- }))
- ConfigureServer(ts.Config, &Server{
- PermitProhibitedCipherSuites: permitProhibitedCipherSuites,
- })
- ts.TLS = ts.Config.TLSConfig // the httptest.Server has its own copy of this TLS config
- ts.StartTLS()
- defer ts.Close()
-
- t.Logf("Running test server for curl to hit at: %s", ts.URL)
- container := curl(t, "--silent", "--http2", "--insecure", "-v", ts.URL)
- defer kill(container)
- res, err := dockerLogs(container)
- if err != nil {
- t.Fatal(err)
- }
-
- body := string(res)
- // Search for both "key: value" and "key:value", since curl changed their format
- // Our Dockerfile contains the latest version (no space), but just in case people
- // didn't rebuild, check both.
- if !strings.Contains(body, "foo: Bar") && !strings.Contains(body, "foo:Bar") {
- t.Errorf("didn't see foo: Bar header")
- t.Logf("Got: %s", body)
- }
- if !strings.Contains(body, "client-proto: HTTP/2") && !strings.Contains(body, "client-proto:HTTP/2") {
- t.Errorf("didn't see client-proto: HTTP/2 header")
- t.Logf("Got: %s", res)
- }
- if !strings.Contains(string(res), msg) {
- t.Errorf("didn't see %q content", msg)
- t.Logf("Got: %s", res)
- }
-
- if atomic.LoadInt32(&gotConn) == 0 {
- t.Error("never saw an http2 connection")
- }
-}
-
-var doh2load = flag.Bool("h2load", false, "Run h2load test")
-
-func TestServerWithH2Load(t *testing.T) {
- if !*doh2load {
- t.Skip("Skipping without --h2load flag.")
- }
- if runtime.GOOS != "linux" {
- t.Skip("skipping Docker test when not on Linux; requires --net which won't work with boot2docker anyway")
- }
- requireH2load(t)
-
- msg := strings.Repeat("Hello, h2load!\n", 5000)
- ts := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- io.WriteString(w, msg)
- w.(http.Flusher).Flush()
- io.WriteString(w, msg)
- }))
- ts.StartTLS()
- defer ts.Close()
-
- cmd := exec.Command("docker", "run", "--net=host", "--entrypoint=/usr/local/bin/h2load", "gohttp2/curl",
- "-n100000", "-c100", "-m100", ts.URL)
- cmd.Stdout = os.Stdout
- cmd.Stderr = os.Stderr
- if err := cmd.Run(); err != nil {
- t.Fatal(err)
- }
-}
-
func TestServer_MaxDecoderHeaderTableSize(t *testing.T) {
wantHeaderTableSize := uint32(initialHeaderTableSize * 2)
st := newServerTester(t, func(w http.ResponseWriter, r *http.Request) {}, func(s *Server) {
@@ -4756,3 +4664,116 @@ func TestServerWriteDoesNotRetainBufferAfterServerClose(t *testing.T) {
st.ts.Config.Close()
<-donec
}
+
+func TestServerMaxHandlerGoroutines(t *testing.T) {
+ const maxHandlers = 10
+ handlerc := make(chan chan bool)
+ donec := make(chan struct{})
+ defer close(donec)
+ st := newServerTester(t, func(w http.ResponseWriter, r *http.Request) {
+ stopc := make(chan bool, 1)
+ select {
+ case handlerc <- stopc:
+ case <-donec:
+ }
+ select {
+ case shouldPanic := <-stopc:
+ if shouldPanic {
+ panic(http.ErrAbortHandler)
+ }
+ case <-donec:
+ }
+ }, func(s *Server) {
+ s.MaxConcurrentStreams = maxHandlers
+ })
+ defer st.Close()
+
+ st.writePreface()
+ st.writeInitialSettings()
+ st.writeSettingsAck()
+
+ // Make maxHandlers concurrent requests.
+ // Reset them all, but only after the handler goroutines have started.
+ var stops []chan bool
+ streamID := uint32(1)
+ for i := 0; i < maxHandlers; i++ {
+ st.writeHeaders(HeadersFrameParam{
+ StreamID: streamID,
+ BlockFragment: st.encodeHeader(),
+ EndStream: true,
+ EndHeaders: true,
+ })
+ stops = append(stops, <-handlerc)
+ st.fr.WriteRSTStream(streamID, ErrCodeCancel)
+ streamID += 2
+ }
+
+ // Start another request, and immediately reset it.
+ st.writeHeaders(HeadersFrameParam{
+ StreamID: streamID,
+ BlockFragment: st.encodeHeader(),
+ EndStream: true,
+ EndHeaders: true,
+ })
+ st.fr.WriteRSTStream(streamID, ErrCodeCancel)
+ streamID += 2
+
+ // Start another two requests. Don't reset these.
+ for i := 0; i < 2; i++ {
+ st.writeHeaders(HeadersFrameParam{
+ StreamID: streamID,
+ BlockFragment: st.encodeHeader(),
+ EndStream: true,
+ EndHeaders: true,
+ })
+ streamID += 2
+ }
+
+ // The initial maxHandlers handlers are still executing,
+ // so the last two requests don't start any new handlers.
+ select {
+ case <-handlerc:
+ t.Errorf("handler unexpectedly started while maxHandlers are already running")
+ case <-time.After(1 * time.Millisecond):
+ }
+
+ // Tell two handlers to exit.
+ // The pending requests which weren't reset start handlers.
+ stops[0] <- false // normal exit
+ stops[1] <- true // panic
+ stops = stops[2:]
+ stops = append(stops, <-handlerc)
+ stops = append(stops, <-handlerc)
+
+ // Make a bunch more requests.
+ // Eventually, the server tells us to go away.
+ for i := 0; i < 5*maxHandlers; i++ {
+ st.writeHeaders(HeadersFrameParam{
+ StreamID: streamID,
+ BlockFragment: st.encodeHeader(),
+ EndStream: true,
+ EndHeaders: true,
+ })
+ st.fr.WriteRSTStream(streamID, ErrCodeCancel)
+ streamID += 2
+ }
+Frames:
+ for {
+ f, err := st.readFrame()
+ if err != nil {
+ st.t.Fatal(err)
+ }
+ switch f := f.(type) {
+ case *GoAwayFrame:
+ if f.ErrCode != ErrCodeEnhanceYourCalm {
+ t.Errorf("err code = %v; want %v", f.ErrCode, ErrCodeEnhanceYourCalm)
+ }
+ break Frames
+ default:
+ }
+ }
+
+ for _, s := range stops {
+ close(s)
+ }
+}
diff --git a/http2/transport.go b/http2/transport.go
index b9632380e7..4515b22c4a 100644
--- a/http2/transport.go
+++ b/http2/transport.go
@@ -19,6 +19,7 @@ import (
"io/fs"
"log"
"math"
+ "math/bits"
mathrand "math/rand"
"net"
"net/http"
@@ -290,8 +291,7 @@ func (t *Transport) initConnPool() {
// HTTP/2 server.
type ClientConn struct {
t *Transport
- tconn net.Conn // usually *tls.Conn, except specialized impls
- tconnClosed bool
+ tconn net.Conn // usually *tls.Conn, except specialized impls
tlsState *tls.ConnectionState // nil only for specialized impls
reused uint32 // whether conn is being reused; atomic
singleUse bool // whether being used for a single http.Request
@@ -518,11 +518,14 @@ func (t *Transport) RoundTrip(req *http.Request) (*http.Response, error) {
func authorityAddr(scheme string, authority string) (addr string) {
host, port, err := net.SplitHostPort(authority)
if err != nil { // authority didn't have a port
+ host = authority
+ port = ""
+ }
+ if port == "" { // authority's port was empty
port = "443"
if scheme == "http" {
port = "80"
}
- host = authority
}
if a, err := idna.ToASCII(host); err == nil {
host = a
@@ -1677,7 +1680,27 @@ func (cs *clientStream) frameScratchBufferLen(maxFrameSize int) int {
return int(n) // doesn't truncate; max is 512K
}
-var bufPool sync.Pool // of *[]byte
+// Seven bufPools manage different frame sizes. This helps to avoid scenarios where long-running
+// streaming requests using small frame sizes occupy large buffers initially allocated for prior
+// requests needing big buffers. The size ranges are as follows:
+// {0 KB, 16 KB], {16 KB, 32 KB], {32 KB, 64 KB], {64 KB, 128 KB], {128 KB, 256 KB],
+// {256 KB, 512 KB], {512 KB, infinity}
+// In practice, the maximum scratch buffer size should not exceed 512 KB due to
+// frameScratchBufferLen(maxFrameSize), thus the "infinity pool" should never be used.
+// It exists mainly as a safety measure, for potential future increases in max buffer size.
+var bufPools [7]sync.Pool // of *[]byte
+func bufPoolIndex(size int) int {
+ if size <= 16384 {
+ return 0
+ }
+ size -= 1
+ bits := bits.Len(uint(size))
+ index := bits - 14
+ if index >= len(bufPools) {
+ return len(bufPools) - 1
+ }
+ return index
+}
func (cs *clientStream) writeRequestBody(req *http.Request) (err error) {
cc := cs.cc
@@ -1695,12 +1718,13 @@ func (cs *clientStream) writeRequestBody(req *http.Request) (err error) {
// Scratch buffer for reading into & writing from.
scratchLen := cs.frameScratchBufferLen(maxFrameSize)
var buf []byte
- if bp, ok := bufPool.Get().(*[]byte); ok && len(*bp) >= scratchLen {
- defer bufPool.Put(bp)
+ index := bufPoolIndex(scratchLen)
+ if bp, ok := bufPools[index].Get().(*[]byte); ok && len(*bp) >= scratchLen {
+ defer bufPools[index].Put(bp)
buf = *bp
} else {
buf = make([]byte, scratchLen)
- defer bufPool.Put(&buf)
+ defer bufPools[index].Put(&buf)
}
var sawEOF bool
diff --git a/http2/transport_test.go b/http2/transport_test.go
index d3156208cf..99848485b9 100644
--- a/http2/transport_test.go
+++ b/http2/transport_test.go
@@ -4456,11 +4456,14 @@ func TestAuthorityAddr(t *testing.T) {
}{
{"http", "foo.com", "foo.com:80"},
{"https", "foo.com", "foo.com:443"},
+ {"https", "foo.com:", "foo.com:443"},
{"https", "foo.com:1234", "foo.com:1234"},
{"https", "1.2.3.4:1234", "1.2.3.4:1234"},
{"https", "1.2.3.4", "1.2.3.4:443"},
+ {"https", "1.2.3.4:", "1.2.3.4:443"},
{"https", "[::1]:1234", "[::1]:1234"},
{"https", "[::1]", "[::1]:443"},
+ {"https", "[::1]:", "[::1]:443"},
}
for _, tt := range tests {
got := authorityAddr(tt.scheme, tt.authority)
diff --git a/internal/quic/acks.go b/internal/quic/acks.go
new file mode 100644
index 0000000000..ba860efb2b
--- /dev/null
+++ b/internal/quic/acks.go
@@ -0,0 +1,184 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build go1.21
+
+package quic
+
+import (
+ "time"
+)
+
+// ackState tracks packets received from a peer within a number space.
+// It handles packet deduplication (don't process the same packet twice) and
+// determines the timing and content of ACK frames.
+type ackState struct {
+ seen rangeset[packetNumber]
+
+ // The time at which we must send an ACK frame, even if we have no other data to send.
+ nextAck time.Time
+
+ // The time we received the largest-numbered packet in seen.
+ maxRecvTime time.Time
+
+ // The largest-numbered ack-eliciting packet in seen.
+ maxAckEliciting packetNumber
+
+ // The number of ack-eliciting packets in seen that we have not yet acknowledged.
+ unackedAckEliciting int
+}
+
+// shouldProcess reports whether a packet should be handled or discarded.
+func (acks *ackState) shouldProcess(num packetNumber) bool {
+ if packetNumber(acks.seen.min()) > num {
+ // We've discarded the state for this range of packet numbers.
+ // Discard the packet rather than potentially processing a duplicate.
+ // https://www.rfc-editor.org/rfc/rfc9000.html#section-13.2.3-5
+ return false
+ }
+ if acks.seen.contains(num) {
+ // Discard duplicate packets.
+ return false
+ }
+ return true
+}
+
+// receive records receipt of a packet.
+func (acks *ackState) receive(now time.Time, space numberSpace, num packetNumber, ackEliciting bool) {
+ if ackEliciting {
+ acks.unackedAckEliciting++
+ if acks.mustAckImmediately(space, num) {
+ acks.nextAck = now
+ } else if acks.nextAck.IsZero() {
+ // This packet does not need to be acknowledged immediately,
+ // but the ack must not be intentionally delayed by more than
+ // the max_ack_delay transport parameter we sent to the peer.
+ //
+ // We always delay acks by the maximum allowed, less the timer
+ // granularity. ("[max_ack_delay] SHOULD include the receiver's
+ // expected delays in alarms firing.")
+ //
+ // https://www.rfc-editor.org/rfc/rfc9000#section-18.2-4.28.1
+ acks.nextAck = now.Add(maxAckDelay - timerGranularity)
+ }
+ if num > acks.maxAckEliciting {
+ acks.maxAckEliciting = num
+ }
+ }
+
+ acks.seen.add(num, num+1)
+ if num == acks.seen.max() {
+ acks.maxRecvTime = now
+ }
+
+ // Limit the total number of ACK ranges by dropping older ranges.
+ //
+ // Remembering more ranges results in larger ACK frames.
+ //
+ // Remembering a large number of ranges could result in ACK frames becoming
+ // too large to fit in a packet, in which case we will silently drop older
+ // ranges during packet construction.
+ //
+ // Remembering fewer ranges can result in unnecessary retransmissions,
+ // since we cannot accept packets older than the oldest remembered range.
+ //
+ // The limit here is completely arbitrary. If it seems wrong, it probably is.
+ //
+ // https://www.rfc-editor.org/rfc/rfc9000#section-13.2.3
+ const maxAckRanges = 8
+ if overflow := acks.seen.numRanges() - maxAckRanges; overflow > 0 {
+ acks.seen.removeranges(0, overflow)
+ }
+}
+
+// mustAckImmediately reports whether an ack-eliciting packet must be acknowledged immediately,
+// or whether the ack may be deferred.
+func (acks *ackState) mustAckImmediately(space numberSpace, num packetNumber) bool {
+ // https://www.rfc-editor.org/rfc/rfc9000.html#section-13.2.1
+ if space != appDataSpace {
+ // "[...] all ack-eliciting Initial and Handshake packets [...]"
+ // https://www.rfc-editor.org/rfc/rfc9000.html#section-13.2.1-2
+ return true
+ }
+ if num < acks.maxAckEliciting {
+ // "[...] when the received packet has a packet number less than another
+ // ack-eliciting packet that has been received [...]"
+ // https://www.rfc-editor.org/rfc/rfc9000.html#section-13.2.1-8.1
+ return true
+ }
+ if acks.seen.rangeContaining(acks.maxAckEliciting).end != num {
+ // "[...] when the packet has a packet number larger than the highest-numbered
+ // ack-eliciting packet that has been received and there are missing packets
+ // between that packet and this packet."
+ // https://www.rfc-editor.org/rfc/rfc9000.html#section-13.2.1-8.2
+ //
+ // This case is a bit tricky. Let's say we've received:
+ // 0, ack-eliciting
+ // 1, ack-eliciting
+ // 3, NOT ack eliciting
+ //
+ // We have sent ACKs for 0 and 1. If we receive ack-eliciting packet 2,
+ // we do not need to send an immediate ACK, because there are no missing
+ // packets between it and the highest-numbered ack-eliciting packet (1).
+ // If we receive ack-eliciting packet 4, we do need to send an immediate ACK,
+ // because there's a gap (the missing packet 2).
+ //
+ // We check for this by looking up the ACK range which contains the
+ // highest-numbered ack-eliciting packet: [0, 1) in the above example.
+ // If the range ends just before the packet we are now processing,
+ // there are no gaps. If it does not, there must be a gap.
+ return true
+ }
+ if acks.unackedAckEliciting >= 2 {
+ // "[...] after receiving at least two ack-eliciting packets."
+ // https://www.rfc-editor.org/rfc/rfc9000.html#section-13.2.2
+ return true
+ }
+ return false
+}
+
+// shouldSendAck reports whether the connection should send an ACK frame at this time,
+// in an ACK-only packet if necessary.
+func (acks *ackState) shouldSendAck(now time.Time) bool {
+ return !acks.nextAck.IsZero() && !acks.nextAck.After(now)
+}
+
+// acksToSend returns the set of packet numbers to ACK at this time, and the current ack delay.
+// It may return acks even if shouldSendAck returns false, when there are unacked
+// ack-eliciting packets whose ack is being delayed.
+func (acks *ackState) acksToSend(now time.Time) (nums rangeset[packetNumber], ackDelay time.Duration) {
+ if acks.nextAck.IsZero() && acks.unackedAckEliciting == 0 {
+ return nil, 0
+ }
+ // "[...] the delays intentionally introduced between the time the packet with the
+ // largest packet number is received and the time an acknowledgement is sent."
+ // https://www.rfc-editor.org/rfc/rfc9000#section-13.2.5-1
+ delay := now.Sub(acks.maxRecvTime)
+ if delay < 0 {
+ delay = 0
+ }
+ return acks.seen, delay
+}
+
+// sentAck records that an ACK frame has been sent.
+func (acks *ackState) sentAck() {
+ acks.nextAck = time.Time{}
+ acks.unackedAckEliciting = 0
+}
+
+// handleAck records that an ack has been received for a ACK frame we sent
+// containing the given Largest Acknowledged field.
+func (acks *ackState) handleAck(largestAcked packetNumber) {
+ // We can stop acking packets less or equal to largestAcked.
+ // https://www.rfc-editor.org/rfc/rfc9000.html#section-13.2.4-1
+ //
+ // We rely on acks.seen containing the largest packet number that has been successfully
+ // processed, so we retain the range containing largestAcked and discard previous ones.
+ acks.seen.sub(0, acks.seen.rangeContaining(largestAcked).start)
+}
+
+// largestSeen reports the largest seen packet.
+func (acks *ackState) largestSeen() packetNumber {
+ return acks.seen.max()
+}
diff --git a/internal/quic/acks_test.go b/internal/quic/acks_test.go
new file mode 100644
index 0000000000..4f1032910f
--- /dev/null
+++ b/internal/quic/acks_test.go
@@ -0,0 +1,248 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build go1.21
+
+package quic
+
+import (
+ "testing"
+ "time"
+)
+
+func TestAcksDisallowDuplicate(t *testing.T) {
+ // Don't process a packet that we've seen before.
+ acks := ackState{}
+ now := time.Now()
+ receive := []packetNumber{0, 1, 2, 4, 7, 6, 9}
+ seen := map[packetNumber]bool{}
+ for i, pnum := range receive {
+ acks.receive(now, appDataSpace, pnum, true)
+ seen[pnum] = true
+ for ppnum := packetNumber(0); ppnum < 11; ppnum++ {
+ if got, want := acks.shouldProcess(ppnum), !seen[ppnum]; got != want {
+ t.Fatalf("after receiving %v: acks.shouldProcess(%v) = %v, want %v", receive[:i+1], ppnum, got, want)
+ }
+ }
+ }
+}
+
+func TestAcksDisallowDiscardedAckRanges(t *testing.T) {
+ // Don't process a packet with a number in a discarded range.
+ acks := ackState{}
+ now := time.Now()
+ for pnum := packetNumber(0); ; pnum += 2 {
+ acks.receive(now, appDataSpace, pnum, true)
+ send, _ := acks.acksToSend(now)
+ for ppnum := packetNumber(0); ppnum < packetNumber(send.min()); ppnum++ {
+ if acks.shouldProcess(ppnum) {
+ t.Fatalf("after limiting ack ranges to %v: acks.shouldProcess(%v) (in discarded range) = true, want false", send, ppnum)
+ }
+ }
+ if send.min() > 10 {
+ break
+ }
+ }
+}
+
+func TestAcksSent(t *testing.T) {
+ type packet struct {
+ pnum packetNumber
+ ackEliciting bool
+ }
+ for _, test := range []struct {
+ name string
+ space numberSpace
+
+ // ackedPackets and packets are packets that we receive.
+ // After receiving all packets in ackedPackets, we send an ack.
+ // Then we receive the subsequent packets in packets.
+ ackedPackets []packet
+ packets []packet
+
+ wantDelay time.Duration
+ wantAcks rangeset[packetNumber]
+ }{{
+ name: "no packets to ack",
+ space: initialSpace,
+ }, {
+ name: "non-ack-eliciting packets are not acked",
+ space: initialSpace,
+ packets: []packet{{
+ pnum: 0,
+ ackEliciting: false,
+ }},
+ }, {
+ name: "ack-eliciting Initial packets are acked immediately",
+ space: initialSpace,
+ packets: []packet{{
+ pnum: 0,
+ ackEliciting: true,
+ }},
+ wantAcks: rangeset[packetNumber]{{0, 1}},
+ wantDelay: 0,
+ }, {
+ name: "ack-eliciting Handshake packets are acked immediately",
+ space: handshakeSpace,
+ packets: []packet{{
+ pnum: 0,
+ ackEliciting: true,
+ }},
+ wantAcks: rangeset[packetNumber]{{0, 1}},
+ wantDelay: 0,
+ }, {
+ name: "ack-eliciting AppData packets are acked after max_ack_delay",
+ space: appDataSpace,
+ packets: []packet{{
+ pnum: 0,
+ ackEliciting: true,
+ }},
+ wantAcks: rangeset[packetNumber]{{0, 1}},
+ wantDelay: maxAckDelay - timerGranularity,
+ }, {
+ name: "reordered ack-eliciting packets are acked immediately",
+ space: appDataSpace,
+ ackedPackets: []packet{{
+ pnum: 1,
+ ackEliciting: true,
+ }},
+ packets: []packet{{
+ pnum: 0,
+ ackEliciting: true,
+ }},
+ wantAcks: rangeset[packetNumber]{{0, 2}},
+ wantDelay: 0,
+ }, {
+ name: "gaps in ack-eliciting packets are acked immediately",
+ space: appDataSpace,
+ packets: []packet{{
+ pnum: 1,
+ ackEliciting: true,
+ }},
+ wantAcks: rangeset[packetNumber]{{1, 2}},
+ wantDelay: 0,
+ }, {
+ name: "reordered non-ack-eliciting packets are not acked immediately",
+ space: appDataSpace,
+ ackedPackets: []packet{{
+ pnum: 1,
+ ackEliciting: true,
+ }},
+ packets: []packet{{
+ pnum: 2,
+ ackEliciting: true,
+ }, {
+ pnum: 0,
+ ackEliciting: false,
+ }, {
+ pnum: 4,
+ ackEliciting: false,
+ }},
+ wantAcks: rangeset[packetNumber]{{0, 3}, {4, 5}},
+ wantDelay: maxAckDelay - timerGranularity,
+ }, {
+ name: "immediate ack after two ack-eliciting packets are received",
+ space: appDataSpace,
+ packets: []packet{{
+ pnum: 0,
+ ackEliciting: true,
+ }, {
+ pnum: 1,
+ ackEliciting: true,
+ }},
+ wantAcks: rangeset[packetNumber]{{0, 2}},
+ wantDelay: 0,
+ }} {
+ t.Run(test.name, func(t *testing.T) {
+ acks := ackState{}
+ start := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)
+ for _, p := range test.ackedPackets {
+ t.Logf("receive %v.%v, ack-eliciting=%v", test.space, p.pnum, p.ackEliciting)
+ acks.receive(start, test.space, p.pnum, p.ackEliciting)
+ }
+ t.Logf("send an ACK frame")
+ acks.sentAck()
+ for _, p := range test.packets {
+ t.Logf("receive %v.%v, ack-eliciting=%v", test.space, p.pnum, p.ackEliciting)
+ acks.receive(start, test.space, p.pnum, p.ackEliciting)
+ }
+ switch {
+ case len(test.wantAcks) == 0:
+ // No ACK should be sent, even well after max_ack_delay.
+ if acks.shouldSendAck(start.Add(10 * maxAckDelay)) {
+ t.Errorf("acks.shouldSendAck(T+10*max_ack_delay) = true, want false")
+ }
+ case test.wantDelay > 0:
+ // No ACK should be sent before a delay.
+ if acks.shouldSendAck(start.Add(test.wantDelay - 1)) {
+ t.Errorf("acks.shouldSendAck(T+%v-1ns) = true, want false", test.wantDelay)
+ }
+ fallthrough
+ default:
+ // ACK should be sent after a delay.
+ if !acks.shouldSendAck(start.Add(test.wantDelay)) {
+ t.Errorf("acks.shouldSendAck(T+%v) = false, want true", test.wantDelay)
+ }
+ }
+ // acksToSend always reports the available packets that can be acked,
+ // and the amount of time that has passed since the most recent acked
+ // packet was received.
+ for _, delay := range []time.Duration{
+ 0,
+ test.wantDelay,
+ test.wantDelay + 1,
+ } {
+ gotNums, gotDelay := acks.acksToSend(start.Add(delay))
+ wantDelay := delay
+ if len(gotNums) == 0 {
+ wantDelay = 0
+ }
+ if !slicesEqual(gotNums, test.wantAcks) || gotDelay != wantDelay {
+ t.Errorf("acks.acksToSend(T+%v) = %v, %v; want %v, %v", delay, gotNums, gotDelay, test.wantAcks, wantDelay)
+ }
+ }
+ })
+ }
+}
+
+// slicesEqual reports whether two slices are equal.
+// Replace this with slices.Equal once the module go.mod is go1.17 or newer.
+func slicesEqual[E comparable](s1, s2 []E) bool {
+ if len(s1) != len(s2) {
+ return false
+ }
+ for i := range s1 {
+ if s1[i] != s2[i] {
+ return false
+ }
+ }
+ return true
+}
+
+func TestAcksDiscardAfterAck(t *testing.T) {
+ acks := ackState{}
+ now := time.Now()
+ acks.receive(now, appDataSpace, 0, true)
+ acks.receive(now, appDataSpace, 2, true)
+ acks.receive(now, appDataSpace, 4, true)
+ acks.receive(now, appDataSpace, 5, true)
+ acks.receive(now, appDataSpace, 6, true)
+ acks.handleAck(6) // discards all ranges prior to the one containing packet 6
+ acks.receive(now, appDataSpace, 7, true)
+ got, _ := acks.acksToSend(now)
+ if len(got) != 1 {
+ t.Errorf("acks.acksToSend contains ranges prior to last acknowledged ack; got %v, want 1 range", got)
+ }
+}
+
+func TestAcksLargestSeen(t *testing.T) {
+ acks := ackState{}
+ now := time.Now()
+ acks.receive(now, appDataSpace, 0, true)
+ acks.receive(now, appDataSpace, 4, true)
+ acks.receive(now, appDataSpace, 1, true)
+ if got, want := acks.largestSeen(), packetNumber(4); got != want {
+ t.Errorf("acks.largestSeen() = %v, want %v", got, want)
+ }
+}
diff --git a/internal/quic/atomic_bits.go b/internal/quic/atomic_bits.go
new file mode 100644
index 0000000000..e1e2594d15
--- /dev/null
+++ b/internal/quic/atomic_bits.go
@@ -0,0 +1,33 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build go1.21
+
+package quic
+
+import "sync/atomic"
+
+// atomicBits is an atomic uint32 that supports setting individual bits.
+type atomicBits[T ~uint32] struct {
+ bits atomic.Uint32
+}
+
+// set sets the bits in mask to the corresponding bits in v.
+// It returns the new value.
+func (a *atomicBits[T]) set(v, mask T) T {
+ if v&^mask != 0 {
+ panic("BUG: bits in v are not in mask")
+ }
+ for {
+ o := a.bits.Load()
+ n := (o &^ uint32(mask)) | uint32(v)
+ if a.bits.CompareAndSwap(o, n) {
+ return T(n)
+ }
+ }
+}
+
+func (a *atomicBits[T]) load() T {
+ return T(a.bits.Load())
+}
diff --git a/internal/quic/config.go b/internal/quic/config.go
new file mode 100644
index 0000000000..b390d6911e
--- /dev/null
+++ b/internal/quic/config.go
@@ -0,0 +1,81 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build go1.21
+
+package quic
+
+import (
+ "crypto/tls"
+)
+
+// A Config structure configures a QUIC endpoint.
+// A Config must not be modified after it has been passed to a QUIC function.
+// A Config may be reused; the quic package will also not modify it.
+type Config struct {
+ // TLSConfig is the endpoint's TLS configuration.
+ // It must be non-nil and include at least one certificate or else set GetCertificate.
+ TLSConfig *tls.Config
+
+ // MaxBidiRemoteStreams limits the number of simultaneous bidirectional streams
+ // a peer may open.
+ // If zero, the default value of 100 is used.
+ // If negative, the limit is zero.
+ MaxBidiRemoteStreams int64
+
+ // MaxUniRemoteStreams limits the number of simultaneous unidirectional streams
+ // a peer may open.
+ // If zero, the default value of 100 is used.
+ // If negative, the limit is zero.
+ MaxUniRemoteStreams int64
+
+ // MaxStreamReadBufferSize is the maximum amount of data sent by the peer that a
+ // stream will buffer for reading.
+ // If zero, the default value of 1MiB is used.
+ // If negative, the limit is zero.
+ MaxStreamReadBufferSize int64
+
+ // MaxStreamWriteBufferSize is the maximum amount of data a stream will buffer for
+ // sending to the peer.
+ // If zero, the default value of 1MiB is used.
+ // If negative, the limit is zero.
+ MaxStreamWriteBufferSize int64
+
+ // MaxConnReadBufferSize is the maximum amount of data sent by the peer that a
+ // connection will buffer for reading, across all streams.
+ // If zero, the default value of 1MiB is used.
+ // If negative, the limit is zero.
+ MaxConnReadBufferSize int64
+}
+
+func configDefault(v, def, limit int64) int64 {
+ switch {
+ case v == 0:
+ return def
+ case v < 0:
+ return 0
+ default:
+ return min(v, limit)
+ }
+}
+
+func (c *Config) maxBidiRemoteStreams() int64 {
+ return configDefault(c.MaxBidiRemoteStreams, 100, maxStreamsLimit)
+}
+
+func (c *Config) maxUniRemoteStreams() int64 {
+ return configDefault(c.MaxUniRemoteStreams, 100, maxStreamsLimit)
+}
+
+func (c *Config) maxStreamReadBufferSize() int64 {
+ return configDefault(c.MaxStreamReadBufferSize, 1<<20, maxVarint)
+}
+
+func (c *Config) maxStreamWriteBufferSize() int64 {
+ return configDefault(c.MaxStreamWriteBufferSize, 1<<20, maxVarint)
+}
+
+func (c *Config) maxConnReadBufferSize() int64 {
+ return configDefault(c.MaxConnReadBufferSize, 1<<20, maxVarint)
+}
diff --git a/internal/quic/config_test.go b/internal/quic/config_test.go
new file mode 100644
index 0000000000..d292854f54
--- /dev/null
+++ b/internal/quic/config_test.go
@@ -0,0 +1,47 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build go1.21
+
+package quic
+
+import "testing"
+
+func TestConfigTransportParameters(t *testing.T) {
+ const (
+ wantInitialMaxData = int64(1)
+ wantInitialMaxStreamData = int64(2)
+ wantInitialMaxStreamsBidi = int64(3)
+ wantInitialMaxStreamsUni = int64(4)
+ )
+ tc := newTestConn(t, clientSide, func(c *Config) {
+ c.MaxBidiRemoteStreams = wantInitialMaxStreamsBidi
+ c.MaxUniRemoteStreams = wantInitialMaxStreamsUni
+ c.MaxStreamReadBufferSize = wantInitialMaxStreamData
+ c.MaxConnReadBufferSize = wantInitialMaxData
+ })
+ tc.handshake()
+ if tc.sentTransportParameters == nil {
+ t.Fatalf("conn didn't send transport parameters during handshake")
+ }
+ p := tc.sentTransportParameters
+ if got, want := p.initialMaxData, wantInitialMaxData; got != want {
+ t.Errorf("initial_max_data = %v, want %v", got, want)
+ }
+ if got, want := p.initialMaxStreamDataBidiLocal, wantInitialMaxStreamData; got != want {
+ t.Errorf("initial_max_stream_data_bidi_local = %v, want %v", got, want)
+ }
+ if got, want := p.initialMaxStreamDataBidiRemote, wantInitialMaxStreamData; got != want {
+ t.Errorf("initial_max_stream_data_bidi_remote = %v, want %v", got, want)
+ }
+ if got, want := p.initialMaxStreamDataUni, wantInitialMaxStreamData; got != want {
+ t.Errorf("initial_max_stream_data_uni = %v, want %v", got, want)
+ }
+ if got, want := p.initialMaxStreamsBidi, wantInitialMaxStreamsBidi; got != want {
+ t.Errorf("initial_max_stream_data_uni = %v, want %v", got, want)
+ }
+ if got, want := p.initialMaxStreamsUni, wantInitialMaxStreamsUni; got != want {
+ t.Errorf("initial_max_stream_data_uni = %v, want %v", got, want)
+ }
+}
diff --git a/internal/quic/conn.go b/internal/quic/conn.go
new file mode 100644
index 0000000000..9db00fe092
--- /dev/null
+++ b/internal/quic/conn.go
@@ -0,0 +1,376 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build go1.21
+
+package quic
+
+import (
+ "context"
+ "crypto/tls"
+ "errors"
+ "fmt"
+ "net/netip"
+ "time"
+)
+
+// A Conn is a QUIC connection.
+//
+// Multiple goroutines may invoke methods on a Conn simultaneously.
+type Conn struct {
+ side connSide
+ listener *Listener
+ config *Config
+ testHooks connTestHooks
+ peerAddr netip.AddrPort
+
+ msgc chan any
+ donec chan struct{} // closed when conn loop exits
+ exited bool // set to make the conn loop exit immediately
+
+ w packetWriter
+ acks [numberSpaceCount]ackState // indexed by number space
+ lifetime lifetimeState
+ connIDState connIDState
+ loss lossState
+ streams streamsState
+
+ // idleTimeout is the time at which the connection will be closed due to inactivity.
+ // https://www.rfc-editor.org/rfc/rfc9000#section-10.1
+ maxIdleTimeout time.Duration
+ idleTimeout time.Time
+
+ // Packet protection keys, CRYPTO streams, and TLS state.
+ keysInitial fixedKeyPair
+ keysHandshake fixedKeyPair
+ keysAppData updatingKeyPair
+ crypto [numberSpaceCount]cryptoStream
+ tls *tls.QUICConn
+
+ // handshakeConfirmed is set when the handshake is confirmed.
+ // For server connections, it tracks sending HANDSHAKE_DONE.
+ handshakeConfirmed sentVal
+
+ peerAckDelayExponent int8 // -1 when unknown
+
+ // Tests only: Send a PING in a specific number space.
+ testSendPingSpace numberSpace
+ testSendPing sentVal
+}
+
+// connTestHooks override conn behavior in tests.
+type connTestHooks interface {
+ nextMessage(msgc chan any, nextTimeout time.Time) (now time.Time, message any)
+ handleTLSEvent(tls.QUICEvent)
+ newConnID(seq int64) ([]byte, error)
+ waitUntil(ctx context.Context, until func() bool) error
+ timeNow() time.Time
+}
+
+func newConn(now time.Time, side connSide, initialConnID []byte, peerAddr netip.AddrPort, config *Config, l *Listener, hooks connTestHooks) (*Conn, error) {
+ c := &Conn{
+ side: side,
+ listener: l,
+ config: config,
+ peerAddr: peerAddr,
+ msgc: make(chan any, 1),
+ donec: make(chan struct{}),
+ testHooks: hooks,
+ maxIdleTimeout: defaultMaxIdleTimeout,
+ idleTimeout: now.Add(defaultMaxIdleTimeout),
+ peerAckDelayExponent: -1,
+ }
+
+ // A one-element buffer allows us to wake a Conn's event loop as a
+ // non-blocking operation.
+ c.msgc = make(chan any, 1)
+
+ var originalDstConnID []byte
+ if c.side == clientSide {
+ if err := c.connIDState.initClient(c); err != nil {
+ return nil, err
+ }
+ initialConnID, _ = c.connIDState.dstConnID()
+ } else {
+ if err := c.connIDState.initServer(c, initialConnID); err != nil {
+ return nil, err
+ }
+ originalDstConnID = initialConnID
+ }
+
+ // The smallest allowed maximum QUIC datagram size is 1200 bytes.
+ // TODO: PMTU discovery.
+ const maxDatagramSize = 1200
+ c.keysAppData.init()
+ c.loss.init(c.side, maxDatagramSize, now)
+ c.streamsInit()
+ c.lifetimeInit()
+
+ // TODO: retry_source_connection_id
+ if err := c.startTLS(now, initialConnID, transportParameters{
+ initialSrcConnID: c.connIDState.srcConnID(),
+ originalDstConnID: originalDstConnID,
+ ackDelayExponent: ackDelayExponent,
+ maxUDPPayloadSize: maxUDPPayloadSize,
+ maxAckDelay: maxAckDelay,
+ disableActiveMigration: true,
+ initialMaxData: config.maxConnReadBufferSize(),
+ initialMaxStreamDataBidiLocal: config.maxStreamReadBufferSize(),
+ initialMaxStreamDataBidiRemote: config.maxStreamReadBufferSize(),
+ initialMaxStreamDataUni: config.maxStreamReadBufferSize(),
+ initialMaxStreamsBidi: c.streams.remoteLimit[bidiStream].max,
+ initialMaxStreamsUni: c.streams.remoteLimit[uniStream].max,
+ activeConnIDLimit: activeConnIDLimit,
+ }); err != nil {
+ return nil, err
+ }
+
+ go c.loop(now)
+ return c, nil
+}
+
+func (c *Conn) String() string {
+ return fmt.Sprintf("quic.Conn(%v,->%v)", c.side, c.peerAddr)
+}
+
+// confirmHandshake is called when the handshake is confirmed.
+// https://www.rfc-editor.org/rfc/rfc9001#section-4.1.2
+func (c *Conn) confirmHandshake(now time.Time) {
+ // If handshakeConfirmed is unset, the handshake is not confirmed.
+ // If it is unsent, the handshake is confirmed and we need to send a HANDSHAKE_DONE.
+ // If it is sent, we have sent a HANDSHAKE_DONE.
+ // If it is received, the handshake is confirmed and we do not need to send anything.
+ if c.handshakeConfirmed.isSet() {
+ return // already confirmed
+ }
+ if c.side == serverSide {
+ // When the server confirms the handshake, it sends a HANDSHAKE_DONE.
+ c.handshakeConfirmed.setUnsent()
+ c.listener.serverConnEstablished(c)
+ } else {
+ // The client never sends a HANDSHAKE_DONE, so we set handshakeConfirmed
+ // to the received state, indicating that the handshake is confirmed and we
+ // don't need to send anything.
+ c.handshakeConfirmed.setReceived()
+ }
+ c.loss.confirmHandshake()
+ // "An endpoint MUST discard its Handshake keys when the TLS handshake is confirmed"
+ // https://www.rfc-editor.org/rfc/rfc9001#section-4.9.2-1
+ c.discardKeys(now, handshakeSpace)
+}
+
+// discardKeys discards unused packet protection keys.
+// https://www.rfc-editor.org/rfc/rfc9001#section-4.9
+func (c *Conn) discardKeys(now time.Time, space numberSpace) {
+ switch space {
+ case initialSpace:
+ c.keysInitial.discard()
+ case handshakeSpace:
+ c.keysHandshake.discard()
+ }
+ c.loss.discardKeys(now, space)
+}
+
+// receiveTransportParameters applies transport parameters sent by the peer.
+func (c *Conn) receiveTransportParameters(p transportParameters) error {
+ if err := c.connIDState.validateTransportParameters(c.side, p); err != nil {
+ return err
+ }
+ c.streams.outflow.setMaxData(p.initialMaxData)
+ c.streams.localLimit[bidiStream].setMax(p.initialMaxStreamsBidi)
+ c.streams.localLimit[uniStream].setMax(p.initialMaxStreamsUni)
+ c.streams.peerInitialMaxStreamDataBidiLocal = p.initialMaxStreamDataBidiLocal
+ c.streams.peerInitialMaxStreamDataRemote[bidiStream] = p.initialMaxStreamDataBidiRemote
+ c.streams.peerInitialMaxStreamDataRemote[uniStream] = p.initialMaxStreamDataUni
+ c.peerAckDelayExponent = p.ackDelayExponent
+ c.loss.setMaxAckDelay(p.maxAckDelay)
+ if err := c.connIDState.setPeerActiveConnIDLimit(c, p.activeConnIDLimit); err != nil {
+ return err
+ }
+ if p.preferredAddrConnID != nil {
+ var (
+ seq int64 = 1 // sequence number of this conn id is 1
+ retirePriorTo int64 = 0 // retire nothing
+ resetToken [16]byte
+ )
+ copy(resetToken[:], p.preferredAddrResetToken)
+ if err := c.connIDState.handleNewConnID(seq, retirePriorTo, p.preferredAddrConnID, resetToken); err != nil {
+ return err
+ }
+ }
+
+ // TODO: Many more transport parameters to come.
+
+ return nil
+}
+
+type (
+ timerEvent struct{}
+ wakeEvent struct{}
+)
+
+// loop is the connection main loop.
+//
+// Except where otherwise noted, all connection state is owned by the loop goroutine.
+//
+// The loop processes messages from c.msgc and timer events.
+// Other goroutines may examine or modify conn state by sending the loop funcs to execute.
+func (c *Conn) loop(now time.Time) {
+ defer close(c.donec)
+ defer c.tls.Close()
+ defer c.listener.connDrained(c)
+
+ // The connection timer sends a message to the connection loop on expiry.
+ // We need to give it an expiry when creating it, so set the initial timeout to
+ // an arbitrary large value. The timer will be reset before this expires (and it
+ // isn't a problem if it does anyway). Skip creating the timer in tests which
+ // take control of the connection message loop.
+ var timer *time.Timer
+ var lastTimeout time.Time
+ hooks := c.testHooks
+ if hooks == nil {
+ timer = time.AfterFunc(1*time.Hour, func() {
+ c.sendMsg(timerEvent{})
+ })
+ defer timer.Stop()
+ }
+
+ for !c.exited {
+ sendTimeout := c.maybeSend(now) // try sending
+
+ // Note that we only need to consider the ack timer for the App Data space,
+ // since the Initial and Handshake spaces always ack immediately.
+ nextTimeout := sendTimeout
+ nextTimeout = firstTime(nextTimeout, c.idleTimeout)
+ if !c.isClosingOrDraining() {
+ nextTimeout = firstTime(nextTimeout, c.loss.timer)
+ nextTimeout = firstTime(nextTimeout, c.acks[appDataSpace].nextAck)
+ } else {
+ nextTimeout = firstTime(nextTimeout, c.lifetime.drainEndTime)
+ }
+
+ var m any
+ if hooks != nil {
+ // Tests only: Wait for the test to tell us to continue.
+ now, m = hooks.nextMessage(c.msgc, nextTimeout)
+ } else if !nextTimeout.IsZero() && nextTimeout.Before(now) {
+ // A connection timer has expired.
+ now = time.Now()
+ m = timerEvent{}
+ } else {
+ // Reschedule the connection timer if necessary
+ // and wait for the next event.
+ if !nextTimeout.Equal(lastTimeout) && !nextTimeout.IsZero() {
+ // Resetting a timer created with time.AfterFunc guarantees
+ // that the timer will run again. We might generate a spurious
+ // timer event under some circumstances, but that's okay.
+ timer.Reset(nextTimeout.Sub(now))
+ lastTimeout = nextTimeout
+ }
+ m = <-c.msgc
+ now = time.Now()
+ }
+ switch m := m.(type) {
+ case *datagram:
+ c.handleDatagram(now, m)
+ m.recycle()
+ case timerEvent:
+ // A connection timer has expired.
+ if !now.Before(c.idleTimeout) {
+ // "[...] the connection is silently closed and
+ // its state is discarded [...]"
+ // https://www.rfc-editor.org/rfc/rfc9000#section-10.1-1
+ c.exited = true
+ return
+ }
+ c.loss.advance(now, c.handleAckOrLoss)
+ if c.lifetimeAdvance(now) {
+ // The connection has completed the draining period,
+ // and may be shut down.
+ return
+ }
+ case wakeEvent:
+ // We're being woken up to try sending some frames.
+ case func(time.Time, *Conn):
+ // Send a func to msgc to run it on the main Conn goroutine
+ m(now, c)
+ default:
+ panic(fmt.Sprintf("quic: unrecognized conn message %T", m))
+ }
+ }
+}
+
+// sendMsg sends a message to the conn's loop.
+// It does not wait for the message to be processed.
+// The conn may close before processing the message, in which case it is lost.
+func (c *Conn) sendMsg(m any) {
+ select {
+ case c.msgc <- m:
+ case <-c.donec:
+ }
+}
+
+// wake wakes up the conn's loop.
+func (c *Conn) wake() {
+ select {
+ case c.msgc <- wakeEvent{}:
+ default:
+ }
+}
+
+// runOnLoop executes a function within the conn's loop goroutine.
+func (c *Conn) runOnLoop(f func(now time.Time, c *Conn)) error {
+ donec := make(chan struct{})
+ c.sendMsg(func(now time.Time, c *Conn) {
+ defer close(donec)
+ f(now, c)
+ })
+ select {
+ case <-donec:
+ case <-c.donec:
+ return errors.New("quic: connection closed")
+ }
+ return nil
+}
+
+func (c *Conn) waitOnDone(ctx context.Context, ch <-chan struct{}) error {
+ if c.testHooks != nil {
+ return c.testHooks.waitUntil(ctx, func() bool {
+ select {
+ case <-ch:
+ return true
+ default:
+ }
+ return false
+ })
+ }
+ // Check the channel before the context.
+ // We always prefer to return results when available,
+ // even when provided with an already-canceled context.
+ select {
+ case <-ch:
+ return nil
+ default:
+ }
+ select {
+ case <-ch:
+ case <-ctx.Done():
+ return ctx.Err()
+ }
+ return nil
+}
+
+// firstTime returns the earliest non-zero time, or zero if both times are zero.
+func firstTime(a, b time.Time) time.Time {
+ switch {
+ case a.IsZero():
+ return b
+ case b.IsZero():
+ return a
+ case a.Before(b):
+ return a
+ default:
+ return b
+ }
+}
diff --git a/internal/quic/conn_async_test.go b/internal/quic/conn_async_test.go
new file mode 100644
index 0000000000..dc2a57f9dd
--- /dev/null
+++ b/internal/quic/conn_async_test.go
@@ -0,0 +1,185 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build go1.21
+
+package quic
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "path/filepath"
+ "runtime"
+ "sync"
+)
+
+// asyncTestState permits handling asynchronous operations in a synchronous test.
+//
+// For example, a test may want to write to a stream and observe that
+// STREAM frames are sent with the contents of the write in response
+// to MAX_STREAM_DATA frames received from the peer.
+// The Stream.Write is an asynchronous operation, but the test is simpler
+// if we can start the write, observe the first STREAM frame sent,
+// send a MAX_STREAM_DATA frame, observe the next STREAM frame sent, etc.
+//
+// We do this by instrumenting points where operations can block.
+// We start async operations like Write in a goroutine,
+// and wait for the operation to either finish or hit a blocking point.
+// When the connection event loop is idle, we check a list of
+// blocked operations to see if any can be woken.
+type asyncTestState struct {
+ mu sync.Mutex
+ notify chan struct{}
+ blocked map[*blockedAsync]struct{}
+}
+
+// An asyncOp is an asynchronous operation that results in (T, error).
+type asyncOp[T any] struct {
+ v T
+ err error
+
+ caller string
+ state *asyncTestState
+ donec chan struct{}
+ cancelFunc context.CancelFunc
+}
+
+// cancel cancels the async operation's context, and waits for
+// the operation to complete.
+func (a *asyncOp[T]) cancel() {
+ select {
+ case <-a.donec:
+ return // already done
+ default:
+ }
+ a.cancelFunc()
+ <-a.state.notify
+ select {
+ case <-a.donec:
+ default:
+ panic(fmt.Errorf("%v: async op failed to finish after being canceled", a.caller))
+ }
+}
+
+var errNotDone = errors.New("async op is not done")
+
+// result returns the result of the async operation.
+// It returns errNotDone if the operation is still in progress.
+//
+// Note that unlike a traditional async/await, this doesn't block
+// waiting for the operation to complete. Since tests have full
+// control over the progress of operations, an asyncOp can only
+// become done in reaction to the test taking some action.
+func (a *asyncOp[T]) result() (v T, err error) {
+ select {
+ case <-a.donec:
+ return a.v, a.err
+ default:
+ return v, errNotDone
+ }
+}
+
+// A blockedAsync is a blocked async operation.
+type blockedAsync struct {
+ until func() bool // when this returns true, the operation is unblocked
+ donec chan struct{} // closed when the operation is unblocked
+}
+
+type asyncContextKey struct{}
+
+// runAsync starts an asynchronous operation.
+//
+// The function f should call a blocking function such as
+// Stream.Write or Conn.AcceptStream and return its result.
+// It must use the provided context.
+func runAsync[T any](ts *testConn, f func(context.Context) (T, error)) *asyncOp[T] {
+ as := &ts.asyncTestState
+ if as.notify == nil {
+ as.notify = make(chan struct{})
+ as.mu.Lock()
+ as.blocked = make(map[*blockedAsync]struct{})
+ as.mu.Unlock()
+ }
+ _, file, line, _ := runtime.Caller(1)
+ ctx := context.WithValue(context.Background(), asyncContextKey{}, true)
+ ctx, cancel := context.WithCancel(ctx)
+ a := &asyncOp[T]{
+ state: as,
+ caller: fmt.Sprintf("%v:%v", filepath.Base(file), line),
+ donec: make(chan struct{}),
+ cancelFunc: cancel,
+ }
+ go func() {
+ a.v, a.err = f(ctx)
+ close(a.donec)
+ as.notify <- struct{}{}
+ }()
+ ts.t.Cleanup(func() {
+ if _, err := a.result(); err == errNotDone {
+ ts.t.Errorf("%v: async operation is still executing at end of test", a.caller)
+ a.cancel()
+ }
+ })
+ // Wait for the operation to either finish or block.
+ <-as.notify
+ return a
+}
+
+// waitUntil waits for a blocked async operation to complete.
+// The operation is complete when the until func returns true.
+func (as *asyncTestState) waitUntil(ctx context.Context, until func() bool) error {
+ if until() {
+ return nil
+ }
+ if err := ctx.Err(); err != nil {
+ // Context has already expired.
+ return err
+ }
+ if ctx.Value(asyncContextKey{}) == nil {
+ // Context is not one that we've created, and hasn't expired.
+ // This probably indicates that we've tried to perform a
+ // blocking operation without using the async test harness here,
+ // which may have unpredictable results.
+ panic("blocking async point with unexpected Context")
+ }
+ b := &blockedAsync{
+ until: until,
+ donec: make(chan struct{}),
+ }
+ // Record this as a pending blocking operation.
+ as.mu.Lock()
+ as.blocked[b] = struct{}{}
+ as.mu.Unlock()
+ // Notify the creator of the operation that we're blocked,
+ // and wait to be woken up.
+ as.notify <- struct{}{}
+ select {
+ case <-b.donec:
+ case <-ctx.Done():
+ return ctx.Err()
+ }
+ return nil
+}
+
+// wakeAsync tries to wake up a blocked async operation.
+// It returns true if one was woken, false otherwise.
+func (as *asyncTestState) wakeAsync() bool {
+ as.mu.Lock()
+ var woken *blockedAsync
+ for w := range as.blocked {
+ if w.until() {
+ woken = w
+ delete(as.blocked, w)
+ break
+ }
+ }
+ as.mu.Unlock()
+ if woken == nil {
+ return false
+ }
+ close(woken.donec)
+ <-as.notify // must not hold as.mu while blocked here
+ return true
+}
diff --git a/internal/quic/conn_close.go b/internal/quic/conn_close.go
new file mode 100644
index 0000000000..b8b86fd6fb
--- /dev/null
+++ b/internal/quic/conn_close.go
@@ -0,0 +1,252 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build go1.21
+
+package quic
+
+import (
+ "context"
+ "errors"
+ "time"
+)
+
+// lifetimeState tracks the state of a connection.
+//
+// This is fairly coupled to the rest of a Conn, but putting it in a struct of its own helps
+// reason about operations that cause state transitions.
+type lifetimeState struct {
+ readyc chan struct{} // closed when TLS handshake completes
+ drainingc chan struct{} // closed when entering the draining state
+
+ // Possible states for the connection:
+ //
+ // Alive: localErr and finalErr are both nil.
+ //
+ // Closing: localErr is non-nil and finalErr is nil.
+ // We have sent a CONNECTION_CLOSE to the peer or are about to
+ // (if connCloseSentTime is zero) and are waiting for the peer to respond.
+ // drainEndTime is set to the time the closing state ends.
+ // https://www.rfc-editor.org/rfc/rfc9000.html#section-10.2.1
+ //
+ // Draining: finalErr is non-nil.
+ // If localErr is nil, we're waiting for the user to provide us with a final status
+ // to send to the peer.
+ // Otherwise, we've either sent a CONNECTION_CLOSE to the peer or are about to
+ // (if connCloseSentTime is zero).
+ // drainEndTime is set to the time the draining state ends.
+ // https://www.rfc-editor.org/rfc/rfc9000.html#section-10.2.2
+ localErr error // error sent to the peer
+ finalErr error // error sent by the peer, or transport error; always set before draining
+
+ connCloseSentTime time.Time // send time of last CONNECTION_CLOSE frame
+ connCloseDelay time.Duration // delay until next CONNECTION_CLOSE frame sent
+ drainEndTime time.Time // time the connection exits the draining state
+}
+
+func (c *Conn) lifetimeInit() {
+ c.lifetime.readyc = make(chan struct{})
+ c.lifetime.drainingc = make(chan struct{})
+}
+
+var errNoPeerResponse = errors.New("peer did not respond to CONNECTION_CLOSE")
+
+// advance is called when time passes.
+func (c *Conn) lifetimeAdvance(now time.Time) (done bool) {
+ if c.lifetime.drainEndTime.IsZero() || c.lifetime.drainEndTime.After(now) {
+ return false
+ }
+ // The connection drain period has ended, and we can shut down.
+ // https://www.rfc-editor.org/rfc/rfc9000.html#section-10.2-7
+ c.lifetime.drainEndTime = time.Time{}
+ if c.lifetime.finalErr == nil {
+ // The peer never responded to our CONNECTION_CLOSE.
+ c.enterDraining(errNoPeerResponse)
+ }
+ return true
+}
+
+// confirmHandshake is called when the TLS handshake completes.
+func (c *Conn) handshakeDone() {
+ close(c.lifetime.readyc)
+}
+
+// isDraining reports whether the conn is in the draining state.
+//
+// The draining state is entered once an endpoint receives a CONNECTION_CLOSE frame.
+// The endpoint will no longer send any packets, but we retain knowledge of the connection
+// until the end of the drain period to ensure we discard packets for the connection
+// rather than treating them as starting a new connection.
+//
+// https://www.rfc-editor.org/rfc/rfc9000.html#section-10.2.2
+func (c *Conn) isDraining() bool {
+ return c.lifetime.finalErr != nil
+}
+
+// isClosingOrDraining reports whether the conn is in the closing or draining states.
+func (c *Conn) isClosingOrDraining() bool {
+ return c.lifetime.localErr != nil || c.lifetime.finalErr != nil
+}
+
+// sendOK reports whether the conn can send frames at this time.
+func (c *Conn) sendOK(now time.Time) bool {
+ if !c.isClosingOrDraining() {
+ return true
+ }
+ // We are closing or draining.
+ if c.lifetime.localErr == nil {
+ // We're waiting for the user to close the connection, providing us with
+ // a final status to send to the peer.
+ return false
+ }
+ // Past this point, returning true will result in the conn sending a CONNECTION_CLOSE
+ // due to localErr being set.
+ if c.lifetime.drainEndTime.IsZero() {
+ // The closing and draining states should last for at least three times
+ // the current PTO interval. We currently use exactly that minimum.
+ // https://www.rfc-editor.org/rfc/rfc9000.html#section-10.2-5
+ //
+ // The drain period begins when we send or receive a CONNECTION_CLOSE,
+ // whichever comes first.
+ // https://www.rfc-editor.org/rfc/rfc9000.html#section-10.2.2-3
+ c.lifetime.drainEndTime = now.Add(3 * c.loss.ptoBasePeriod())
+ }
+ if c.lifetime.connCloseSentTime.IsZero() {
+ // We haven't sent a CONNECTION_CLOSE yet. Do so.
+ // Either we're initiating an immediate close
+ // (and will enter the closing state as soon as we send CONNECTION_CLOSE),
+ // or we've read a CONNECTION_CLOSE from our peer
+ // (and may send one CONNECTION_CLOSE before entering the draining state).
+ //
+ // Set the initial delay before we will send another CONNECTION_CLOSE.
+ //
+ // RFC 9000 states that we should rate limit CONNECTION_CLOSE frames,
+ // but leaves the implementation of the limit up to us. Here, we start
+ // with the same delay as the PTO timer (RFC 9002, Section 6.2.1),
+ // not including max_ack_delay, and double it on every CONNECTION_CLOSE sent.
+ c.lifetime.connCloseDelay = c.loss.rtt.smoothedRTT + max(4*c.loss.rtt.rttvar, timerGranularity)
+ c.lifetime.drainEndTime = now.Add(3 * c.loss.ptoBasePeriod())
+ return true
+ }
+ if c.isDraining() {
+ // We are in the draining state, and will send no more packets.
+ return false
+ }
+ maxRecvTime := c.acks[initialSpace].maxRecvTime
+ if t := c.acks[handshakeSpace].maxRecvTime; t.After(maxRecvTime) {
+ maxRecvTime = t
+ }
+ if t := c.acks[appDataSpace].maxRecvTime; t.After(maxRecvTime) {
+ maxRecvTime = t
+ }
+ if maxRecvTime.Before(c.lifetime.connCloseSentTime.Add(c.lifetime.connCloseDelay)) {
+ // After sending CONNECTION_CLOSE, ignore packets from the peer for
+ // a delay. On the next packet received after the delay, send another
+ // CONNECTION_CLOSE.
+ return false
+ }
+ c.lifetime.connCloseSentTime = now
+ c.lifetime.connCloseDelay *= 2
+ return true
+}
+
+// enterDraining enters the draining state.
+func (c *Conn) enterDraining(err error) {
+ if c.isDraining() {
+ return
+ }
+ if e, ok := c.lifetime.localErr.(localTransportError); ok && transportError(e) != errNo {
+ // If we've terminated the connection due to a peer protocol violation,
+ // record the final error on the connection as our reason for termination.
+ c.lifetime.finalErr = c.lifetime.localErr
+ } else {
+ c.lifetime.finalErr = err
+ }
+ close(c.lifetime.drainingc)
+ c.streams.queue.close(c.lifetime.finalErr)
+}
+
+func (c *Conn) waitReady(ctx context.Context) error {
+ select {
+ case <-c.lifetime.readyc:
+ return nil
+ case <-c.lifetime.drainingc:
+ return c.lifetime.finalErr
+ default:
+ }
+ select {
+ case <-c.lifetime.readyc:
+ return nil
+ case <-c.lifetime.drainingc:
+ return c.lifetime.finalErr
+ case <-ctx.Done():
+ return ctx.Err()
+ }
+}
+
+// Close closes the connection.
+//
+// Close is equivalent to:
+//
+// conn.Abort(nil)
+// err := conn.Wait(context.Background())
+func (c *Conn) Close() error {
+ c.Abort(nil)
+ <-c.lifetime.drainingc
+ return c.lifetime.finalErr
+}
+
+// Wait waits for the peer to close the connection.
+//
+// If the connection is closed locally and the peer does not close its end of the connection,
+// Wait will return with a non-nil error after the drain period expires.
+//
+// If the peer closes the connection with a NO_ERROR transport error, Wait returns nil.
+// If the peer closes the connection with an application error, Wait returns an ApplicationError
+// containing the peer's error code and reason.
+// If the peer closes the connection with any other status, Wait returns a non-nil error.
+func (c *Conn) Wait(ctx context.Context) error {
+ if err := c.waitOnDone(ctx, c.lifetime.drainingc); err != nil {
+ return err
+ }
+ return c.lifetime.finalErr
+}
+
+// Abort closes the connection and returns immediately.
+//
+// If err is nil, Abort sends a transport error of NO_ERROR to the peer.
+// If err is an ApplicationError, Abort sends its error code and text.
+// Otherwise, Abort sends a transport error of APPLICATION_ERROR with the error's text.
+func (c *Conn) Abort(err error) {
+ if err == nil {
+ err = localTransportError(errNo)
+ }
+ c.sendMsg(func(now time.Time, c *Conn) {
+ c.abort(now, err)
+ })
+}
+
+// abort terminates a connection with an error.
+func (c *Conn) abort(now time.Time, err error) {
+ if c.lifetime.localErr != nil {
+ return // already closing
+ }
+ c.lifetime.localErr = err
+}
+
+// abortImmediately terminates a connection.
+// The connection does not send a CONNECTION_CLOSE, and skips the draining period.
+func (c *Conn) abortImmediately(now time.Time, err error) {
+ c.abort(now, err)
+ c.enterDraining(err)
+ c.exited = true
+}
+
+// exit fully terminates a connection immediately.
+func (c *Conn) exit() {
+ c.sendMsg(func(now time.Time, c *Conn) {
+ c.enterDraining(errors.New("connection closed"))
+ c.exited = true
+ })
+}
diff --git a/internal/quic/conn_close_test.go b/internal/quic/conn_close_test.go
new file mode 100644
index 0000000000..20c00e754c
--- /dev/null
+++ b/internal/quic/conn_close_test.go
@@ -0,0 +1,186 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build go1.21
+
+package quic
+
+import (
+ "context"
+ "crypto/tls"
+ "errors"
+ "testing"
+ "time"
+)
+
+func TestConnCloseResponseBackoff(t *testing.T) {
+ tc := newTestConn(t, clientSide)
+ tc.handshake()
+
+ tc.conn.Abort(nil)
+ tc.wantFrame("aborting connection generates CONN_CLOSE",
+ packetType1RTT, debugFrameConnectionCloseTransport{
+ code: errNo,
+ })
+
+ waiting := runAsync(tc, func(ctx context.Context) (struct{}, error) {
+ return struct{}{}, tc.conn.Wait(ctx)
+ })
+ if _, err := waiting.result(); err != errNotDone {
+ t.Errorf("conn.Wait() = %v, want still waiting", err)
+ }
+
+ tc.writeFrames(packetType1RTT, debugFramePing{})
+ tc.wantIdle("packets received immediately after CONN_CLOSE receive no response")
+
+ tc.advance(1100 * time.Microsecond)
+ tc.writeFrames(packetType1RTT, debugFramePing{})
+ tc.wantFrame("receiving packet 1.1ms after CONN_CLOSE generates another CONN_CLOSE",
+ packetType1RTT, debugFrameConnectionCloseTransport{
+ code: errNo,
+ })
+
+ tc.advance(1100 * time.Microsecond)
+ tc.writeFrames(packetType1RTT, debugFramePing{})
+ tc.wantIdle("no response to packet, because CONN_CLOSE backoff is now 2ms")
+
+ tc.advance(1000 * time.Microsecond)
+ tc.writeFrames(packetType1RTT, debugFramePing{})
+ tc.wantFrame("2ms since last CONN_CLOSE, receiving a packet generates another CONN_CLOSE",
+ packetType1RTT, debugFrameConnectionCloseTransport{
+ code: errNo,
+ })
+ if _, err := waiting.result(); err != errNotDone {
+ t.Errorf("conn.Wait() = %v, want still waiting", err)
+ }
+
+ tc.advance(100000 * time.Microsecond)
+ tc.writeFrames(packetType1RTT, debugFramePing{})
+ tc.wantIdle("drain timer expired, no more responses")
+
+ if _, err := waiting.result(); !errors.Is(err, errNoPeerResponse) {
+ t.Errorf("blocked conn.Wait() = %v, want errNoPeerResponse", err)
+ }
+ if err := tc.conn.Wait(canceledContext()); !errors.Is(err, errNoPeerResponse) {
+ t.Errorf("non-blocking conn.Wait() = %v, want errNoPeerResponse", err)
+ }
+}
+
+func TestConnCloseWithPeerResponse(t *testing.T) {
+ tc := newTestConn(t, clientSide)
+ tc.handshake()
+
+ tc.conn.Abort(nil)
+ tc.wantFrame("aborting connection generates CONN_CLOSE",
+ packetType1RTT, debugFrameConnectionCloseTransport{
+ code: errNo,
+ })
+
+ waiting := runAsync(tc, func(ctx context.Context) (struct{}, error) {
+ return struct{}{}, tc.conn.Wait(ctx)
+ })
+ if _, err := waiting.result(); err != errNotDone {
+ t.Errorf("conn.Wait() = %v, want still waiting", err)
+ }
+
+ tc.writeFrames(packetType1RTT, debugFrameConnectionCloseApplication{
+ code: 20,
+ })
+
+ wantErr := &ApplicationError{
+ Code: 20,
+ }
+ if _, err := waiting.result(); !errors.Is(err, wantErr) {
+ t.Errorf("blocked conn.Wait() = %v, want %v", err, wantErr)
+ }
+ if err := tc.conn.Wait(canceledContext()); !errors.Is(err, wantErr) {
+ t.Errorf("non-blocking conn.Wait() = %v, want %v", err, wantErr)
+ }
+}
+
+func TestConnClosePeerCloses(t *testing.T) {
+ tc := newTestConn(t, clientSide)
+ tc.handshake()
+
+ wantErr := &ApplicationError{
+ Code: 42,
+ Reason: "why?",
+ }
+ tc.writeFrames(packetType1RTT, debugFrameConnectionCloseApplication{
+ code: wantErr.Code,
+ reason: wantErr.Reason,
+ })
+ tc.wantIdle("CONN_CLOSE response not sent until user closes this side")
+
+ if err := tc.conn.Wait(canceledContext()); !errors.Is(err, wantErr) {
+ t.Errorf("conn.Wait() = %v, want %v", err, wantErr)
+ }
+
+ tc.conn.Abort(&ApplicationError{
+ Code: 9,
+ Reason: "because",
+ })
+ tc.wantFrame("CONN_CLOSE sent after user closes connection",
+ packetType1RTT, debugFrameConnectionCloseApplication{
+ code: 9,
+ reason: "because",
+ })
+}
+
+func TestConnCloseReceiveInInitial(t *testing.T) {
+ tc := newTestConn(t, clientSide)
+ tc.wantFrame("client sends Initial CRYPTO frame",
+ packetTypeInitial, debugFrameCrypto{
+ data: tc.cryptoDataOut[tls.QUICEncryptionLevelInitial],
+ })
+ tc.writeFrames(packetTypeInitial, debugFrameConnectionCloseTransport{
+ code: errConnectionRefused,
+ })
+ tc.wantIdle("CONN_CLOSE response not sent until user closes this side")
+
+ wantErr := peerTransportError{code: errConnectionRefused}
+ if err := tc.conn.Wait(canceledContext()); !errors.Is(err, wantErr) {
+ t.Errorf("conn.Wait() = %v, want %v", err, wantErr)
+ }
+
+ tc.conn.Abort(&ApplicationError{Code: 1})
+ tc.wantFrame("CONN_CLOSE in Initial frame is APPLICATION_ERROR",
+ packetTypeInitial, debugFrameConnectionCloseTransport{
+ code: errApplicationError,
+ })
+ tc.wantIdle("no more frames to send")
+}
+
+func TestConnCloseReceiveInHandshake(t *testing.T) {
+ tc := newTestConn(t, clientSide)
+ tc.ignoreFrame(frameTypeAck)
+ tc.wantFrame("client sends Initial CRYPTO frame",
+ packetTypeInitial, debugFrameCrypto{
+ data: tc.cryptoDataOut[tls.QUICEncryptionLevelInitial],
+ })
+ tc.writeFrames(packetTypeInitial, debugFrameCrypto{
+ data: tc.cryptoDataIn[tls.QUICEncryptionLevelInitial],
+ })
+ tc.writeFrames(packetTypeHandshake, debugFrameConnectionCloseTransport{
+ code: errConnectionRefused,
+ })
+ tc.wantIdle("CONN_CLOSE response not sent until user closes this side")
+
+ wantErr := peerTransportError{code: errConnectionRefused}
+ if err := tc.conn.Wait(canceledContext()); !errors.Is(err, wantErr) {
+ t.Errorf("conn.Wait() = %v, want %v", err, wantErr)
+ }
+
+ // The conn has Initial and Handshake keys, so it will send CONN_CLOSE in both spaces.
+ tc.conn.Abort(&ApplicationError{Code: 1})
+ tc.wantFrame("CONN_CLOSE in Initial frame is APPLICATION_ERROR",
+ packetTypeInitial, debugFrameConnectionCloseTransport{
+ code: errApplicationError,
+ })
+ tc.wantFrame("CONN_CLOSE in Handshake frame is APPLICATION_ERROR",
+ packetTypeHandshake, debugFrameConnectionCloseTransport{
+ code: errApplicationError,
+ })
+ tc.wantIdle("no more frames to send")
+}
diff --git a/internal/quic/conn_flow.go b/internal/quic/conn_flow.go
new file mode 100644
index 0000000000..4f1ab6eafc
--- /dev/null
+++ b/internal/quic/conn_flow.go
@@ -0,0 +1,141 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build go1.21
+
+package quic
+
+import (
+ "sync/atomic"
+ "time"
+)
+
+// connInflow tracks connection-level flow control for data sent by the peer to us.
+//
+// There are four byte offsets of significance in the stream of data received from the peer,
+// each >= to the previous:
+//
+// - bytes read by the user
+// - bytes received from the peer
+// - limit sent to the peer in a MAX_DATA frame
+// - potential new limit to sent to the peer
+//
+// We maintain a flow control window, so as bytes are read by the user
+// the potential limit is extended correspondingly.
+//
+// We keep an atomic counter of bytes read by the user and not yet applied to the
+// potential limit (credit). When this count grows large enough, we update the
+// new limit to send and mark that we need to send a new MAX_DATA frame.
+type connInflow struct {
+ sent sentVal // set when we need to send a MAX_DATA update to the peer
+ usedLimit int64 // total bytes sent by the peer, must be less than sentLimit
+ sentLimit int64 // last MAX_DATA sent to the peer
+ newLimit int64 // new MAX_DATA to send
+
+ credit atomic.Int64 // bytes read but not yet applied to extending the flow-control window
+}
+
+func (c *Conn) inflowInit() {
+ // The initial MAX_DATA limit is sent as a transport parameter.
+ c.streams.inflow.sentLimit = c.config.maxConnReadBufferSize()
+ c.streams.inflow.newLimit = c.streams.inflow.sentLimit
+}
+
+// handleStreamBytesReadOffLoop records that the user has consumed bytes from a stream.
+// We may extend the peer's flow control window.
+//
+// This is called indirectly by the user, via Read or CloseRead.
+func (c *Conn) handleStreamBytesReadOffLoop(n int64) {
+ if n == 0 {
+ return
+ }
+ if c.shouldUpdateFlowControl(c.streams.inflow.credit.Add(n)) {
+ // We should send a MAX_DATA update to the peer.
+ // Record this on the Conn's main loop.
+ c.sendMsg(func(now time.Time, c *Conn) {
+ // A MAX_DATA update may have already happened, so check again.
+ if c.shouldUpdateFlowControl(c.streams.inflow.credit.Load()) {
+ c.sendMaxDataUpdate()
+ }
+ })
+ }
+}
+
+// handleStreamBytesReadOnLoop extends the peer's flow control window after
+// data has been discarded due to a RESET_STREAM frame.
+//
+// This is called on the conn's loop.
+func (c *Conn) handleStreamBytesReadOnLoop(n int64) {
+ if c.shouldUpdateFlowControl(c.streams.inflow.credit.Add(n)) {
+ c.sendMaxDataUpdate()
+ }
+}
+
+func (c *Conn) sendMaxDataUpdate() {
+ c.streams.inflow.sent.setUnsent()
+ // Apply current credit to the limit.
+ // We don't strictly need to do this here
+ // since appendMaxDataFrame will do so as well,
+ // but this avoids redundant trips down this path
+ // if the MAX_DATA frame doesn't go out right away.
+ c.streams.inflow.newLimit += c.streams.inflow.credit.Swap(0)
+}
+
+func (c *Conn) shouldUpdateFlowControl(credit int64) bool {
+ return shouldUpdateFlowControl(c.config.maxConnReadBufferSize(), credit)
+}
+
+// handleStreamBytesReceived records that the peer has sent us stream data.
+func (c *Conn) handleStreamBytesReceived(n int64) error {
+ c.streams.inflow.usedLimit += n
+ if c.streams.inflow.usedLimit > c.streams.inflow.sentLimit {
+ return localTransportError(errFlowControl)
+ }
+ return nil
+}
+
+// appendMaxDataFrame appends a MAX_DATA frame to the current packet.
+//
+// It returns true if no more frames need appending,
+// false if it could not fit a frame in the current packet.
+func (c *Conn) appendMaxDataFrame(w *packetWriter, pnum packetNumber, pto bool) bool {
+ if c.streams.inflow.sent.shouldSendPTO(pto) {
+ // Add any unapplied credit to the new limit now.
+ c.streams.inflow.newLimit += c.streams.inflow.credit.Swap(0)
+ if !w.appendMaxDataFrame(c.streams.inflow.newLimit) {
+ return false
+ }
+ c.streams.inflow.sentLimit += c.streams.inflow.newLimit
+ c.streams.inflow.sent.setSent(pnum)
+ }
+ return true
+}
+
+// ackOrLossMaxData records the fate of a MAX_DATA frame.
+func (c *Conn) ackOrLossMaxData(pnum packetNumber, fate packetFate) {
+ c.streams.inflow.sent.ackLatestOrLoss(pnum, fate)
+}
+
+// connOutflow tracks connection-level flow control for data sent by us to the peer.
+type connOutflow struct {
+ max int64 // largest MAX_DATA received from peer
+ used int64 // total bytes of STREAM data sent to peer
+}
+
+// setMaxData updates the connection-level flow control limit
+// with the initial limit conveyed in transport parameters
+// or an update from a MAX_DATA frame.
+func (f *connOutflow) setMaxData(maxData int64) {
+ f.max = max(f.max, maxData)
+}
+
+// avail returns the number of connection-level flow control bytes available.
+func (f *connOutflow) avail() int64 {
+ return f.max - f.used
+}
+
+// consume records consumption of n bytes of flow.
+func (f *connOutflow) consume(n int64) {
+ f.used += n
+}
diff --git a/internal/quic/conn_flow_test.go b/internal/quic/conn_flow_test.go
new file mode 100644
index 0000000000..03e0757a6d
--- /dev/null
+++ b/internal/quic/conn_flow_test.go
@@ -0,0 +1,430 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build go1.21
+
+package quic
+
+import (
+ "context"
+ "testing"
+)
+
+func TestConnInflowReturnOnRead(t *testing.T) {
+ ctx := canceledContext()
+ tc, s := newTestConnAndRemoteStream(t, serverSide, uniStream, func(c *Config) {
+ c.MaxConnReadBufferSize = 64
+ })
+ tc.writeFrames(packetType1RTT, debugFrameStream{
+ id: s.id,
+ data: make([]byte, 64),
+ })
+ const readSize = 8
+ if n, err := s.ReadContext(ctx, make([]byte, readSize)); n != readSize || err != nil {
+ t.Fatalf("s.Read() = %v, %v; want %v, nil", n, err, readSize)
+ }
+ tc.wantFrame("available window increases, send a MAX_DATA",
+ packetType1RTT, debugFrameMaxData{
+ max: 64 + readSize,
+ })
+ if n, err := s.ReadContext(ctx, make([]byte, 64)); n != 64-readSize || err != nil {
+ t.Fatalf("s.Read() = %v, %v; want %v, nil", n, err, 64-readSize)
+ }
+ tc.wantFrame("available window increases, send a MAX_DATA",
+ packetType1RTT, debugFrameMaxData{
+ max: 128,
+ })
+ // Peer can write up to the new limit.
+ tc.writeFrames(packetType1RTT, debugFrameStream{
+ id: s.id,
+ off: 64,
+ data: make([]byte, 64),
+ })
+ tc.wantIdle("connection is idle")
+ if n, err := s.ReadContext(ctx, make([]byte, 64)); n != 64 || err != nil {
+ t.Fatalf("offset 64: s.Read() = %v, %v; want %v, nil", n, err, 64)
+ }
+}
+
+func TestConnInflowReturnOnRacingReads(t *testing.T) {
+ // Perform two reads at the same time,
+ // one for half of MaxConnReadBufferSize
+ // and one for one byte.
+ //
+ // We should observe a single MAX_DATA update.
+ // Depending on the ordering of events,
+ // this may include the credit from just the larger read
+ // or the credit from both.
+ ctx := canceledContext()
+ tc := newTestConn(t, serverSide, func(c *Config) {
+ c.MaxConnReadBufferSize = 64
+ })
+ tc.handshake()
+ tc.ignoreFrame(frameTypeAck)
+ tc.writeFrames(packetType1RTT, debugFrameStream{
+ id: newStreamID(clientSide, uniStream, 0),
+ data: make([]byte, 32),
+ })
+ tc.writeFrames(packetType1RTT, debugFrameStream{
+ id: newStreamID(clientSide, uniStream, 1),
+ data: make([]byte, 32),
+ })
+ s1, err := tc.conn.AcceptStream(ctx)
+ if err != nil {
+ t.Fatalf("conn.AcceptStream() = %v", err)
+ }
+ s2, err := tc.conn.AcceptStream(ctx)
+ if err != nil {
+ t.Fatalf("conn.AcceptStream() = %v", err)
+ }
+ read1 := runAsync(tc, func(ctx context.Context) (int, error) {
+ return s1.ReadContext(ctx, make([]byte, 16))
+ })
+ read2 := runAsync(tc, func(ctx context.Context) (int, error) {
+ return s2.ReadContext(ctx, make([]byte, 1))
+ })
+ // This MAX_DATA might extend the window by 16 or 17, depending on
+ // whether the second write occurs before the update happens.
+ tc.wantFrameType("MAX_DATA update is sent",
+ packetType1RTT, debugFrameMaxData{})
+ tc.wantIdle("redundant MAX_DATA is not sent")
+ if _, err := read1.result(); err != nil {
+ t.Errorf("ReadContext #1 = %v", err)
+ }
+ if _, err := read2.result(); err != nil {
+ t.Errorf("ReadContext #2 = %v", err)
+ }
+}
+
+func TestConnInflowReturnOnClose(t *testing.T) {
+ tc, s := newTestConnAndRemoteStream(t, serverSide, uniStream, func(c *Config) {
+ c.MaxConnReadBufferSize = 64
+ })
+ tc.ignoreFrame(frameTypeStopSending)
+ tc.writeFrames(packetType1RTT, debugFrameStream{
+ id: s.id,
+ data: make([]byte, 64),
+ })
+ s.CloseRead()
+ tc.wantFrame("closing stream updates connection-level flow control",
+ packetType1RTT, debugFrameMaxData{
+ max: 128,
+ })
+}
+
+func TestConnInflowReturnOnReset(t *testing.T) {
+ tc, s := newTestConnAndRemoteStream(t, serverSide, uniStream, func(c *Config) {
+ c.MaxConnReadBufferSize = 64
+ })
+ tc.ignoreFrame(frameTypeStopSending)
+ tc.writeFrames(packetType1RTT, debugFrameStream{
+ id: s.id,
+ data: make([]byte, 32),
+ })
+ tc.writeFrames(packetType1RTT, debugFrameResetStream{
+ id: s.id,
+ finalSize: 64,
+ })
+ s.CloseRead()
+ tc.wantFrame("receiving stream reseet updates connection-level flow control",
+ packetType1RTT, debugFrameMaxData{
+ max: 128,
+ })
+}
+
+func TestConnInflowStreamViolation(t *testing.T) {
+ tc := newTestConn(t, serverSide, func(c *Config) {
+ c.MaxConnReadBufferSize = 100
+ })
+ tc.handshake()
+ tc.ignoreFrame(frameTypeAck)
+ // Total MAX_DATA consumed: 50
+ tc.writeFrames(packetType1RTT, debugFrameStream{
+ id: newStreamID(clientSide, bidiStream, 0),
+ data: make([]byte, 50),
+ })
+ // Total MAX_DATA consumed: 80
+ tc.writeFrames(packetType1RTT, debugFrameStream{
+ id: newStreamID(clientSide, uniStream, 0),
+ off: 20,
+ data: make([]byte, 10),
+ })
+ // Total MAX_DATA consumed: 100
+ tc.writeFrames(packetType1RTT, debugFrameStream{
+ id: newStreamID(clientSide, bidiStream, 0),
+ off: 70,
+ fin: true,
+ })
+ // This stream has already consumed quota for these bytes.
+ // Total MAX_DATA consumed: 100
+ tc.writeFrames(packetType1RTT, debugFrameStream{
+ id: newStreamID(clientSide, uniStream, 0),
+ data: make([]byte, 20),
+ })
+ tc.wantIdle("peer has consumed all MAX_DATA quota")
+
+ // Total MAX_DATA consumed: 101
+ tc.writeFrames(packetType1RTT, debugFrameStream{
+ id: newStreamID(clientSide, bidiStream, 2),
+ data: make([]byte, 1),
+ })
+ tc.wantFrame("peer violates MAX_DATA limit",
+ packetType1RTT, debugFrameConnectionCloseTransport{
+ code: errFlowControl,
+ })
+}
+
+func TestConnInflowResetViolation(t *testing.T) {
+ tc := newTestConn(t, serverSide, func(c *Config) {
+ c.MaxConnReadBufferSize = 100
+ })
+ tc.handshake()
+ tc.ignoreFrame(frameTypeAck)
+ tc.writeFrames(packetType1RTT, debugFrameStream{
+ id: newStreamID(clientSide, bidiStream, 0),
+ data: make([]byte, 100),
+ })
+ tc.wantIdle("peer has consumed all MAX_DATA quota")
+
+ tc.writeFrames(packetType1RTT, debugFrameResetStream{
+ id: newStreamID(clientSide, uniStream, 0),
+ finalSize: 0,
+ })
+ tc.wantIdle("stream reset does not consume MAX_DATA quota, no error")
+
+ tc.writeFrames(packetType1RTT, debugFrameResetStream{
+ id: newStreamID(clientSide, uniStream, 1),
+ finalSize: 1,
+ })
+ tc.wantFrame("RESET_STREAM final size violates MAX_DATA limit",
+ packetType1RTT, debugFrameConnectionCloseTransport{
+ code: errFlowControl,
+ })
+}
+
+func TestConnInflowMultipleStreams(t *testing.T) {
+ ctx := canceledContext()
+ tc := newTestConn(t, serverSide, func(c *Config) {
+ c.MaxConnReadBufferSize = 128
+ })
+ tc.handshake()
+ tc.ignoreFrame(frameTypeAck)
+
+ var streams []*Stream
+ for _, id := range []streamID{
+ newStreamID(clientSide, uniStream, 0),
+ newStreamID(clientSide, uniStream, 1),
+ newStreamID(clientSide, bidiStream, 0),
+ newStreamID(clientSide, bidiStream, 1),
+ } {
+ tc.writeFrames(packetType1RTT, debugFrameStream{
+ id: id,
+ data: make([]byte, 32),
+ })
+ s, err := tc.conn.AcceptStream(ctx)
+ if err != nil {
+ t.Fatalf("AcceptStream() = %v", err)
+ }
+ streams = append(streams, s)
+ if n, err := s.ReadContext(ctx, make([]byte, 1)); err != nil || n != 1 {
+ t.Fatalf("s.Read() = %v, %v; want 1, nil", n, err)
+ }
+ }
+ tc.wantIdle("streams have read data, but not enough to update MAX_DATA")
+
+ if n, err := streams[0].ReadContext(ctx, make([]byte, 32)); err != nil || n != 31 {
+ t.Fatalf("s.Read() = %v, %v; want 31, nil", n, err)
+ }
+ tc.wantFrame("read enough data to trigger a MAX_DATA update",
+ packetType1RTT, debugFrameMaxData{
+ max: 128 + 32 + 1 + 1 + 1,
+ })
+
+ tc.ignoreFrame(frameTypeStopSending)
+ streams[2].CloseRead()
+ tc.wantFrame("closed stream triggers another MAX_DATA update",
+ packetType1RTT, debugFrameMaxData{
+ max: 128 + 32 + 1 + 32 + 1,
+ })
+}
+
+func TestConnOutflowBlocked(t *testing.T) {
+ tc, s := newTestConnAndLocalStream(t, clientSide, uniStream,
+ permissiveTransportParameters,
+ func(p *transportParameters) {
+ p.initialMaxData = 10
+ })
+ tc.ignoreFrame(frameTypeAck)
+
+ data := makeTestData(32)
+ n, err := s.Write(data)
+ if n != len(data) || err != nil {
+ t.Fatalf("s.Write() = %v, %v; want %v, nil", n, err, len(data))
+ }
+
+ tc.wantFrame("stream writes data up to MAX_DATA limit",
+ packetType1RTT, debugFrameStream{
+ id: s.id,
+ data: data[:10],
+ })
+ tc.wantIdle("stream is blocked by MAX_DATA limit")
+
+ tc.writeFrames(packetType1RTT, debugFrameMaxData{
+ max: 20,
+ })
+ tc.wantFrame("stream writes data up to new MAX_DATA limit",
+ packetType1RTT, debugFrameStream{
+ id: s.id,
+ off: 10,
+ data: data[10:20],
+ })
+ tc.wantIdle("stream is blocked by new MAX_DATA limit")
+
+ tc.writeFrames(packetType1RTT, debugFrameMaxData{
+ max: 100,
+ })
+ tc.wantFrame("stream writes remaining data",
+ packetType1RTT, debugFrameStream{
+ id: s.id,
+ off: 20,
+ data: data[20:],
+ })
+}
+
+func TestConnOutflowMaxDataDecreases(t *testing.T) {
+ tc, s := newTestConnAndLocalStream(t, clientSide, uniStream,
+ permissiveTransportParameters,
+ func(p *transportParameters) {
+ p.initialMaxData = 10
+ })
+ tc.ignoreFrame(frameTypeAck)
+
+ // Decrease in MAX_DATA is ignored.
+ tc.writeFrames(packetType1RTT, debugFrameMaxData{
+ max: 5,
+ })
+
+ data := makeTestData(32)
+ n, err := s.Write(data)
+ if n != len(data) || err != nil {
+ t.Fatalf("s.Write() = %v, %v; want %v, nil", n, err, len(data))
+ }
+
+ tc.wantFrame("stream writes data up to MAX_DATA limit",
+ packetType1RTT, debugFrameStream{
+ id: s.id,
+ data: data[:10],
+ })
+}
+
+func TestConnOutflowMaxDataRoundRobin(t *testing.T) {
+ ctx := canceledContext()
+ tc := newTestConn(t, clientSide, permissiveTransportParameters,
+ func(p *transportParameters) {
+ p.initialMaxData = 0
+ })
+ tc.handshake()
+ tc.ignoreFrame(frameTypeAck)
+
+ s1, err := tc.conn.newLocalStream(ctx, uniStream)
+ if err != nil {
+ t.Fatalf("conn.newLocalStream(%v) = %v", uniStream, err)
+ }
+ s2, err := tc.conn.newLocalStream(ctx, uniStream)
+ if err != nil {
+ t.Fatalf("conn.newLocalStream(%v) = %v", uniStream, err)
+ }
+
+ s1.Write(make([]byte, 10))
+ s2.Write(make([]byte, 10))
+
+ tc.writeFrames(packetType1RTT, debugFrameMaxData{
+ max: 1,
+ })
+ tc.wantFrame("stream 1 writes data up to MAX_DATA limit",
+ packetType1RTT, debugFrameStream{
+ id: s1.id,
+ data: []byte{0},
+ })
+
+ tc.writeFrames(packetType1RTT, debugFrameMaxData{
+ max: 2,
+ })
+ tc.wantFrame("stream 2 writes data up to MAX_DATA limit",
+ packetType1RTT, debugFrameStream{
+ id: s2.id,
+ data: []byte{0},
+ })
+
+ tc.writeFrames(packetType1RTT, debugFrameMaxData{
+ max: 3,
+ })
+ tc.wantFrame("stream 1 writes data up to MAX_DATA limit",
+ packetType1RTT, debugFrameStream{
+ id: s1.id,
+ off: 1,
+ data: []byte{0},
+ })
+}
+
+func TestConnOutflowMetaAndData(t *testing.T) {
+ tc, s := newTestConnAndLocalStream(t, clientSide, bidiStream,
+ permissiveTransportParameters,
+ func(p *transportParameters) {
+ p.initialMaxData = 0
+ })
+ tc.ignoreFrame(frameTypeAck)
+
+ data := makeTestData(32)
+ s.Write(data)
+
+ s.CloseRead()
+ tc.wantFrame("CloseRead sends a STOP_SENDING, not flow controlled",
+ packetType1RTT, debugFrameStopSending{
+ id: s.id,
+ })
+
+ tc.writeFrames(packetType1RTT, debugFrameMaxData{
+ max: 100,
+ })
+ tc.wantFrame("unblocked MAX_DATA",
+ packetType1RTT, debugFrameStream{
+ id: s.id,
+ data: data,
+ })
+}
+
+func TestConnOutflowResentData(t *testing.T) {
+ tc, s := newTestConnAndLocalStream(t, clientSide, bidiStream,
+ permissiveTransportParameters,
+ func(p *transportParameters) {
+ p.initialMaxData = 10
+ })
+ tc.ignoreFrame(frameTypeAck)
+
+ data := makeTestData(15)
+ s.Write(data[:8])
+ tc.wantFrame("data is under MAX_DATA limit, all sent",
+ packetType1RTT, debugFrameStream{
+ id: s.id,
+ data: data[:8],
+ })
+
+ // Lose the last STREAM packet.
+ const pto = false
+ tc.triggerLossOrPTO(packetType1RTT, false)
+ tc.wantFrame("lost STREAM data is retransmitted",
+ packetType1RTT, debugFrameStream{
+ id: s.id,
+ data: data[:8],
+ })
+
+ s.Write(data[8:])
+ tc.wantFrame("new data is sent up to the MAX_DATA limit",
+ packetType1RTT, debugFrameStream{
+ id: s.id,
+ off: 8,
+ data: data[8:10],
+ })
+}
diff --git a/internal/quic/conn_id.go b/internal/quic/conn_id.go
new file mode 100644
index 0000000000..045e646ac1
--- /dev/null
+++ b/internal/quic/conn_id.go
@@ -0,0 +1,423 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build go1.21
+
+package quic
+
+import (
+ "bytes"
+ "crypto/rand"
+)
+
+// connIDState is a conn's connection IDs.
+type connIDState struct {
+ // The destination connection IDs of packets we receive are local.
+ // The destination connection IDs of packets we send are remote.
+ //
+ // Local IDs are usually issued by us, and remote IDs by the peer.
+ // The exception is the transient destination connection ID sent in
+ // a client's Initial packets, which is chosen by the client.
+ //
+ // These are []connID rather than []*connID to minimize allocations.
+ local []connID
+ remote []connID
+
+ nextLocalSeq int64
+ retireRemotePriorTo int64 // largest Retire Prior To value sent by the peer
+ peerActiveConnIDLimit int64 // peer's active_connection_id_limit transport parameter
+
+ needSend bool
+}
+
+// A connID is a connection ID and associated metadata.
+type connID struct {
+ // cid is the connection ID itself.
+ cid []byte
+
+ // seq is the connection ID's sequence number:
+ // https://www.rfc-editor.org/rfc/rfc9000.html#section-5.1.1-1
+ //
+ // For the transient destination ID in a client's Initial packet, this is -1.
+ seq int64
+
+ // retired is set when the connection ID is retired.
+ retired bool
+
+ // send is set when the connection ID's state needs to be sent to the peer.
+ //
+ // For local IDs, this indicates a new ID that should be sent
+ // in a NEW_CONNECTION_ID frame.
+ //
+ // For remote IDs, this indicates a retired ID that should be sent
+ // in a RETIRE_CONNECTION_ID frame.
+ send sentVal
+}
+
+func (s *connIDState) initClient(c *Conn) error {
+ // Client chooses its initial connection ID, and sends it
+ // in the Source Connection ID field of the first Initial packet.
+ locid, err := c.newConnID(0)
+ if err != nil {
+ return err
+ }
+ s.local = append(s.local, connID{
+ seq: 0,
+ cid: locid,
+ })
+ s.nextLocalSeq = 1
+
+ // Client chooses an initial, transient connection ID for the server,
+ // and sends it in the Destination Connection ID field of the first Initial packet.
+ remid, err := c.newConnID(-1)
+ if err != nil {
+ return err
+ }
+ s.remote = append(s.remote, connID{
+ seq: -1,
+ cid: remid,
+ })
+ const retired = false
+ c.listener.connIDsChanged(c, retired, s.local[:])
+ return nil
+}
+
+func (s *connIDState) initServer(c *Conn, dstConnID []byte) error {
+ // Client-chosen, transient connection ID received in the first Initial packet.
+ // The server will not use this as the Source Connection ID of packets it sends,
+ // but remembers it because it may receive packets sent to this destination.
+ s.local = append(s.local, connID{
+ seq: -1,
+ cid: cloneBytes(dstConnID),
+ })
+
+ // Server chooses a connection ID, and sends it in the Source Connection ID of
+ // the response to the clent.
+ locid, err := c.newConnID(0)
+ if err != nil {
+ return err
+ }
+ s.local = append(s.local, connID{
+ seq: 0,
+ cid: locid,
+ })
+ s.nextLocalSeq = 1
+ const retired = false
+ c.listener.connIDsChanged(c, retired, s.local[:])
+ return nil
+}
+
+// srcConnID is the Source Connection ID to use in a sent packet.
+func (s *connIDState) srcConnID() []byte {
+ if s.local[0].seq == -1 && len(s.local) > 1 {
+ // Don't use the transient connection ID if another is available.
+ return s.local[1].cid
+ }
+ return s.local[0].cid
+}
+
+// dstConnID is the Destination Connection ID to use in a sent packet.
+func (s *connIDState) dstConnID() (cid []byte, ok bool) {
+ for i := range s.remote {
+ if !s.remote[i].retired {
+ return s.remote[i].cid, true
+ }
+ }
+ return nil, false
+}
+
+// setPeerActiveConnIDLimit sets the active_connection_id_limit
+// transport parameter received from the peer.
+func (s *connIDState) setPeerActiveConnIDLimit(c *Conn, lim int64) error {
+ s.peerActiveConnIDLimit = lim
+ return s.issueLocalIDs(c)
+}
+
+func (s *connIDState) issueLocalIDs(c *Conn) error {
+ toIssue := min(int(s.peerActiveConnIDLimit), maxPeerActiveConnIDLimit)
+ for i := range s.local {
+ if s.local[i].seq != -1 && !s.local[i].retired {
+ toIssue--
+ }
+ }
+ prev := len(s.local)
+ for toIssue > 0 {
+ cid, err := c.newConnID(s.nextLocalSeq)
+ if err != nil {
+ return err
+ }
+ s.local = append(s.local, connID{
+ seq: s.nextLocalSeq,
+ cid: cid,
+ })
+ s.local[len(s.local)-1].send.setUnsent()
+ s.nextLocalSeq++
+ s.needSend = true
+ toIssue--
+ }
+ const retired = false
+ c.listener.connIDsChanged(c, retired, s.local[prev:])
+ return nil
+}
+
+// validateTransportParameters verifies the original_destination_connection_id and
+// initial_source_connection_id transport parameters match the expected values.
+func (s *connIDState) validateTransportParameters(side connSide, p transportParameters) error {
+ // TODO: Consider returning more detailed errors, for debugging.
+ switch side {
+ case clientSide:
+ // Verify original_destination_connection_id matches
+ // the transient remote connection ID we chose.
+ if len(s.remote) == 0 || s.remote[0].seq != -1 {
+ return localTransportError(errInternal)
+ }
+ if !bytes.Equal(s.remote[0].cid, p.originalDstConnID) {
+ return localTransportError(errTransportParameter)
+ }
+ // Remove the transient remote connection ID.
+ // We have no further need for it.
+ s.remote = append(s.remote[:0], s.remote[1:]...)
+ case serverSide:
+ if p.originalDstConnID != nil {
+ // Clients do not send original_destination_connection_id.
+ return localTransportError(errTransportParameter)
+ }
+ }
+ // Verify initial_source_connection_id matches the first remote connection ID.
+ if len(s.remote) == 0 || s.remote[0].seq != 0 {
+ return localTransportError(errInternal)
+ }
+ if !bytes.Equal(p.initialSrcConnID, s.remote[0].cid) {
+ return localTransportError(errTransportParameter)
+ }
+ return nil
+}
+
+// handlePacket updates the connection ID state during the handshake
+// (Initial and Handshake packets).
+func (s *connIDState) handlePacket(c *Conn, ptype packetType, srcConnID []byte) {
+ switch {
+ case ptype == packetTypeInitial && c.side == clientSide:
+ if len(s.remote) == 1 && s.remote[0].seq == -1 {
+ // We're a client connection processing the first Initial packet
+ // from the server. Replace the transient remote connection ID
+ // with the Source Connection ID from the packet.
+ // Leave the transient ID the list for now, since we'll need it when
+ // processing the transport parameters.
+ s.remote[0].retired = true
+ s.remote = append(s.remote, connID{
+ seq: 0,
+ cid: cloneBytes(srcConnID),
+ })
+ }
+ case ptype == packetTypeInitial && c.side == serverSide:
+ if len(s.remote) == 0 {
+ // We're a server connection processing the first Initial packet
+ // from the client. Set the client's connection ID.
+ s.remote = append(s.remote, connID{
+ seq: 0,
+ cid: cloneBytes(srcConnID),
+ })
+ }
+ case ptype == packetTypeHandshake && c.side == serverSide:
+ if len(s.local) > 0 && s.local[0].seq == -1 && !s.local[0].retired {
+ // We're a server connection processing the first Handshake packet from
+ // the client. Discard the transient, client-chosen connection ID used
+ // for Initial packets; the client will never send it again.
+ const retired = true
+ c.listener.connIDsChanged(c, retired, s.local[0:1])
+ s.local = append(s.local[:0], s.local[1:]...)
+ }
+ }
+}
+
+func (s *connIDState) handleNewConnID(seq, retire int64, cid []byte, resetToken [16]byte) error {
+ if len(s.remote[0].cid) == 0 {
+ // "An endpoint that is sending packets with a zero-length
+ // Destination Connection ID MUST treat receipt of a NEW_CONNECTION_ID
+ // frame as a connection error of type PROTOCOL_VIOLATION."
+ // https://www.rfc-editor.org/rfc/rfc9000.html#section-19.15-6
+ return localTransportError(errProtocolViolation)
+ }
+
+ if retire > s.retireRemotePriorTo {
+ s.retireRemotePriorTo = retire
+ }
+
+ have := false // do we already have this connection ID?
+ active := 0
+ for i := range s.remote {
+ rcid := &s.remote[i]
+ if !rcid.retired && rcid.seq >= 0 && rcid.seq < s.retireRemotePriorTo {
+ s.retireRemote(rcid)
+ }
+ if !rcid.retired {
+ active++
+ }
+ if rcid.seq == seq {
+ if !bytes.Equal(rcid.cid, cid) {
+ return localTransportError(errProtocolViolation)
+ }
+ have = true // yes, we've seen this sequence number
+ }
+ }
+
+ if !have {
+ // This is a new connection ID that we have not seen before.
+ //
+ // We could take steps to keep the list of remote connection IDs
+ // sorted by sequence number, but there's no particular need
+ // so we don't bother.
+ s.remote = append(s.remote, connID{
+ seq: seq,
+ cid: cloneBytes(cid),
+ })
+ if seq < s.retireRemotePriorTo {
+ // This ID was already retired by a previous NEW_CONNECTION_ID frame.
+ s.retireRemote(&s.remote[len(s.remote)-1])
+ } else {
+ active++
+ }
+ }
+
+ if active > activeConnIDLimit {
+ // Retired connection IDs (including newly-retired ones) do not count
+ // against the limit.
+ // https://www.rfc-editor.org/rfc/rfc9000.html#section-5.1.1-5
+ return localTransportError(errConnectionIDLimit)
+ }
+
+ // "An endpoint SHOULD limit the number of connection IDs it has retired locally
+ // for which RETIRE_CONNECTION_ID frames have not yet been acknowledged."
+ // https://www.rfc-editor.org/rfc/rfc9000#section-5.1.2-6
+ //
+ // Set a limit of four times the active_connection_id_limit for
+ // the total number of remote connection IDs we keep state for locally.
+ if len(s.remote) > 4*activeConnIDLimit {
+ return localTransportError(errConnectionIDLimit)
+ }
+
+ return nil
+}
+
+// retireRemote marks a remote connection ID as retired.
+func (s *connIDState) retireRemote(rcid *connID) {
+ rcid.retired = true
+ rcid.send.setUnsent()
+ s.needSend = true
+}
+
+func (s *connIDState) handleRetireConnID(c *Conn, seq int64) error {
+ if seq >= s.nextLocalSeq {
+ return localTransportError(errProtocolViolation)
+ }
+ for i := range s.local {
+ if s.local[i].seq == seq {
+ const retired = true
+ c.listener.connIDsChanged(c, retired, s.local[i:i+1])
+ s.local = append(s.local[:i], s.local[i+1:]...)
+ break
+ }
+ }
+ s.issueLocalIDs(c)
+ return nil
+}
+
+func (s *connIDState) ackOrLossNewConnectionID(pnum packetNumber, seq int64, fate packetFate) {
+ for i := range s.local {
+ if s.local[i].seq != seq {
+ continue
+ }
+ s.local[i].send.ackOrLoss(pnum, fate)
+ if fate != packetAcked {
+ s.needSend = true
+ }
+ return
+ }
+}
+
+func (s *connIDState) ackOrLossRetireConnectionID(pnum packetNumber, seq int64, fate packetFate) {
+ for i := 0; i < len(s.remote); i++ {
+ if s.remote[i].seq != seq {
+ continue
+ }
+ if fate == packetAcked {
+ // We have retired this connection ID, and the peer has acked.
+ // Discard its state completely.
+ s.remote = append(s.remote[:i], s.remote[i+1:]...)
+ } else {
+ // RETIRE_CONNECTION_ID frame was lost, mark for retransmission.
+ s.needSend = true
+ s.remote[i].send.ackOrLoss(pnum, fate)
+ }
+ return
+ }
+}
+
+// appendFrames appends NEW_CONNECTION_ID and RETIRE_CONNECTION_ID frames
+// to the current packet.
+//
+// It returns true if no more frames need appending,
+// false if not everything fit in the current packet.
+func (s *connIDState) appendFrames(w *packetWriter, pnum packetNumber, pto bool) bool {
+ if !s.needSend && !pto {
+ // Fast path: We don't need to send anything.
+ return true
+ }
+ retireBefore := int64(0)
+ if s.local[0].seq != -1 {
+ retireBefore = s.local[0].seq
+ }
+ for i := range s.local {
+ if !s.local[i].send.shouldSendPTO(pto) {
+ continue
+ }
+ if !w.appendNewConnectionIDFrame(
+ s.local[i].seq,
+ retireBefore,
+ s.local[i].cid,
+ [16]byte{}, // TODO: stateless reset token
+ ) {
+ return false
+ }
+ s.local[i].send.setSent(pnum)
+ }
+ for i := range s.remote {
+ if !s.remote[i].send.shouldSendPTO(pto) {
+ continue
+ }
+ if !w.appendRetireConnectionIDFrame(s.remote[i].seq) {
+ return false
+ }
+ s.remote[i].send.setSent(pnum)
+ }
+ s.needSend = false
+ return true
+}
+
+func cloneBytes(b []byte) []byte {
+ n := make([]byte, len(b))
+ copy(n, b)
+ return n
+}
+
+func (c *Conn) newConnID(seq int64) ([]byte, error) {
+ if c.testHooks != nil {
+ return c.testHooks.newConnID(seq)
+ }
+ return newRandomConnID(seq)
+}
+
+func newRandomConnID(_ int64) ([]byte, error) {
+ // It is not necessary for connection IDs to be cryptographically secure,
+ // but it doesn't hurt.
+ id := make([]byte, connIDLen)
+ if _, err := rand.Read(id); err != nil {
+ // TODO: Surface this error as a metric or log event or something.
+ // rand.Read really shouldn't ever fail, but if it does, we should
+ // have a way to inform the user.
+ return nil, err
+ }
+ return id, nil
+}
diff --git a/internal/quic/conn_id_test.go b/internal/quic/conn_id_test.go
new file mode 100644
index 0000000000..44755ecf45
--- /dev/null
+++ b/internal/quic/conn_id_test.go
@@ -0,0 +1,588 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build go1.21
+
+package quic
+
+import (
+ "bytes"
+ "crypto/tls"
+ "fmt"
+ "net/netip"
+ "strings"
+ "testing"
+)
+
+func TestConnIDClientHandshake(t *testing.T) {
+ tc := newTestConn(t, clientSide)
+ // On initialization, the client chooses local and remote IDs.
+ //
+ // The order in which we allocate the two isn't actually important,
+ // but test is a lot simpler if we assume.
+ if got, want := tc.conn.connIDState.srcConnID(), testLocalConnID(0); !bytes.Equal(got, want) {
+ t.Errorf("after initialization: srcConnID = %x, want %x", got, want)
+ }
+ dstConnID, _ := tc.conn.connIDState.dstConnID()
+ if got, want := dstConnID, testLocalConnID(-1); !bytes.Equal(got, want) {
+ t.Errorf("after initialization: dstConnID = %x, want %x", got, want)
+ }
+
+ // The server's first Initial packet provides the client with a
+ // non-transient remote connection ID.
+ tc.writeFrames(packetTypeInitial,
+ debugFrameCrypto{
+ data: tc.cryptoDataIn[tls.QUICEncryptionLevelInitial],
+ })
+ dstConnID, _ = tc.conn.connIDState.dstConnID()
+ if got, want := dstConnID, testPeerConnID(0); !bytes.Equal(got, want) {
+ t.Errorf("after receiving Initial: dstConnID = %x, want %x", got, want)
+ }
+
+ wantLocal := []connID{{
+ cid: testLocalConnID(0),
+ seq: 0,
+ }}
+ if got := tc.conn.connIDState.local; !connIDListEqual(got, wantLocal) {
+ t.Errorf("local ids: %v, want %v", fmtConnIDList(got), fmtConnIDList(wantLocal))
+ }
+ wantRemote := []connID{{
+ cid: testLocalConnID(-1),
+ seq: -1,
+ }, {
+ cid: testPeerConnID(0),
+ seq: 0,
+ }}
+ if got := tc.conn.connIDState.remote; !connIDListEqual(got, wantRemote) {
+ t.Errorf("remote ids: %v, want %v", fmtConnIDList(got), fmtConnIDList(wantRemote))
+ }
+}
+
+func TestConnIDServerHandshake(t *testing.T) {
+ tc := newTestConn(t, serverSide)
+ // On initialization, the server is provided with the client-chosen
+ // transient connection ID, and allocates an ID of its own.
+ // The Initial packet sets the remote connection ID.
+ tc.writeFrames(packetTypeInitial,
+ debugFrameCrypto{
+ data: tc.cryptoDataIn[tls.QUICEncryptionLevelInitial][:1],
+ })
+ if got, want := tc.conn.connIDState.srcConnID(), testLocalConnID(0); !bytes.Equal(got, want) {
+ t.Errorf("after initClient: srcConnID = %q, want %q", got, want)
+ }
+ dstConnID, _ := tc.conn.connIDState.dstConnID()
+ if got, want := dstConnID, testPeerConnID(0); !bytes.Equal(got, want) {
+ t.Errorf("after initClient: dstConnID = %q, want %q", got, want)
+ }
+
+ // The Initial flight of CRYPTO data includes transport parameters,
+ // which cause us to allocate another local connection ID.
+ tc.writeFrames(packetTypeInitial,
+ debugFrameCrypto{
+ off: 1,
+ data: tc.cryptoDataIn[tls.QUICEncryptionLevelInitial][1:],
+ })
+ wantLocal := []connID{{
+ cid: testPeerConnID(-1),
+ seq: -1,
+ }, {
+ cid: testLocalConnID(0),
+ seq: 0,
+ }, {
+ cid: testLocalConnID(1),
+ seq: 1,
+ }}
+ if got := tc.conn.connIDState.local; !connIDListEqual(got, wantLocal) {
+ t.Errorf("local ids: %v, want %v", fmtConnIDList(got), fmtConnIDList(wantLocal))
+ }
+ wantRemote := []connID{{
+ cid: testPeerConnID(0),
+ seq: 0,
+ }}
+ if got := tc.conn.connIDState.remote; !connIDListEqual(got, wantRemote) {
+ t.Errorf("remote ids: %v, want %v", fmtConnIDList(got), fmtConnIDList(wantRemote))
+ }
+
+ // The client's first Handshake packet permits the server to discard the
+ // transient connection ID.
+ tc.writeFrames(packetTypeHandshake,
+ debugFrameCrypto{
+ data: tc.cryptoDataIn[tls.QUICEncryptionLevelHandshake],
+ })
+ wantLocal = []connID{{
+ cid: testLocalConnID(0),
+ seq: 0,
+ }, {
+ cid: testLocalConnID(1),
+ seq: 1,
+ }}
+ if got := tc.conn.connIDState.local; !connIDListEqual(got, wantLocal) {
+ t.Errorf("local ids: %v, want %v", fmtConnIDList(got), fmtConnIDList(wantLocal))
+ }
+}
+
+func connIDListEqual(a, b []connID) bool {
+ if len(a) != len(b) {
+ return false
+ }
+ for i := range a {
+ if a[i].seq != b[i].seq {
+ return false
+ }
+ if !bytes.Equal(a[i].cid, b[i].cid) {
+ return false
+ }
+ }
+ return true
+}
+
+func fmtConnIDList(s []connID) string {
+ var strs []string
+ for _, cid := range s {
+ strs = append(strs, fmt.Sprintf("[seq:%v cid:{%x}]", cid.seq, cid.cid))
+ }
+ return "{" + strings.Join(strs, " ") + "}"
+}
+
+func TestNewRandomConnID(t *testing.T) {
+ cid, err := newRandomConnID(0)
+ if len(cid) != connIDLen || err != nil {
+ t.Fatalf("newConnID() = %x, %v; want %v bytes", cid, connIDLen, err)
+ }
+}
+
+func TestConnIDPeerRequestsManyIDs(t *testing.T) {
+ // "An endpoint SHOULD ensure that its peer has a sufficient number
+ // of available and unused connection IDs."
+ // https://www.rfc-editor.org/rfc/rfc9000#section-5.1.1-4
+ //
+ // "An endpoint MAY limit the total number of connection IDs
+ // issued for each connection [...]"
+ // https://www.rfc-editor.org/rfc/rfc9000#section-5.1.1-6
+ //
+ // Peer requests 100 connection IDs.
+ // We give them 4 in total.
+ tc := newTestConn(t, serverSide, func(p *transportParameters) {
+ p.activeConnIDLimit = 100
+ })
+ tc.ignoreFrame(frameTypeAck)
+ tc.ignoreFrame(frameTypeCrypto)
+
+ tc.writeFrames(packetTypeInitial,
+ debugFrameCrypto{
+ data: tc.cryptoDataIn[tls.QUICEncryptionLevelInitial],
+ })
+ tc.wantFrame("provide additional connection ID 1",
+ packetType1RTT, debugFrameNewConnectionID{
+ seq: 1,
+ connID: testLocalConnID(1),
+ })
+ tc.wantFrame("provide additional connection ID 2",
+ packetType1RTT, debugFrameNewConnectionID{
+ seq: 2,
+ connID: testLocalConnID(2),
+ })
+ tc.wantFrame("provide additional connection ID 3",
+ packetType1RTT, debugFrameNewConnectionID{
+ seq: 3,
+ connID: testLocalConnID(3),
+ })
+ tc.wantIdle("connection ID limit reached, no more to provide")
+}
+
+func TestConnIDPeerProvidesTooManyIDs(t *testing.T) {
+ // "An endpoint MUST NOT provide more connection IDs than the peer's limit."
+ // https://www.rfc-editor.org/rfc/rfc9000#section-5.1.1-4
+ tc := newTestConn(t, serverSide)
+ tc.handshake()
+ tc.ignoreFrame(frameTypeAck)
+
+ tc.writeFrames(packetType1RTT,
+ debugFrameNewConnectionID{
+ seq: 2,
+ connID: testLocalConnID(2),
+ })
+ tc.wantFrame("peer provided 3 connection IDs, our limit is 2",
+ packetType1RTT, debugFrameConnectionCloseTransport{
+ code: errConnectionIDLimit,
+ })
+}
+
+func TestConnIDPeerTemporarilyExceedsActiveConnIDLimit(t *testing.T) {
+ // "An endpoint MAY send connection IDs that temporarily exceed a peer's limit
+ // if the NEW_CONNECTION_ID frame also requires the retirement of any excess [...]"
+ // https://www.rfc-editor.org/rfc/rfc9000#section-5.1.1-4
+ tc := newTestConn(t, serverSide)
+ tc.handshake()
+ tc.ignoreFrame(frameTypeAck)
+
+ tc.writeFrames(packetType1RTT,
+ debugFrameNewConnectionID{
+ retirePriorTo: 2,
+ seq: 2,
+ connID: testPeerConnID(2),
+ }, debugFrameNewConnectionID{
+ retirePriorTo: 2,
+ seq: 3,
+ connID: testPeerConnID(3),
+ })
+ tc.wantFrame("peer requested we retire conn id 0",
+ packetType1RTT, debugFrameRetireConnectionID{
+ seq: 0,
+ })
+ tc.wantFrame("peer requested we retire conn id 1",
+ packetType1RTT, debugFrameRetireConnectionID{
+ seq: 1,
+ })
+}
+
+func TestConnIDPeerRetiresConnID(t *testing.T) {
+ // "An endpoint SHOULD supply a new connection ID when the peer retires a connection ID."
+ // https://www.rfc-editor.org/rfc/rfc9000#section-5.1.1-6
+ for _, side := range []connSide{
+ clientSide,
+ serverSide,
+ } {
+ t.Run(side.String(), func(t *testing.T) {
+ tc := newTestConn(t, side)
+ tc.handshake()
+ tc.ignoreFrame(frameTypeAck)
+
+ tc.writeFrames(packetType1RTT,
+ debugFrameRetireConnectionID{
+ seq: 0,
+ })
+ tc.wantFrame("provide replacement connection ID",
+ packetType1RTT, debugFrameNewConnectionID{
+ seq: 2,
+ retirePriorTo: 1,
+ connID: testLocalConnID(2),
+ })
+ })
+ }
+}
+
+func TestConnIDPeerWithZeroLengthConnIDSendsNewConnectionID(t *testing.T) {
+ // "An endpoint that selects a zero-length connection ID during the handshake
+ // cannot issue a new connection ID."
+ // https://www.rfc-editor.org/rfc/rfc9000#section-5.1.1-8
+ tc := newTestConn(t, clientSide, func(p *transportParameters) {
+ p.initialSrcConnID = []byte{}
+ })
+ tc.peerConnID = []byte{}
+ tc.ignoreFrame(frameTypeAck)
+ tc.uncheckedHandshake()
+
+ tc.writeFrames(packetType1RTT,
+ debugFrameNewConnectionID{
+ seq: 1,
+ connID: testPeerConnID(1),
+ })
+ tc.wantFrame("invalid NEW_CONNECTION_ID: previous conn id is zero-length",
+ packetType1RTT, debugFrameConnectionCloseTransport{
+ code: errProtocolViolation,
+ })
+}
+
+func TestConnIDPeerRequestsRetirement(t *testing.T) {
+ // "Upon receipt of an increased Retire Prior To field, the peer MUST
+ // stop using the corresponding connection IDs and retire them with
+ // RETIRE_CONNECTION_ID frames [...]"
+ // https://www.rfc-editor.org/rfc/rfc9000#section-5.1.2-5
+ tc := newTestConn(t, clientSide)
+ tc.handshake()
+ tc.ignoreFrame(frameTypeAck)
+
+ tc.writeFrames(packetType1RTT,
+ debugFrameNewConnectionID{
+ seq: 2,
+ retirePriorTo: 1,
+ connID: testPeerConnID(2),
+ })
+ tc.wantFrame("peer asked for conn id 0 to be retired",
+ packetType1RTT, debugFrameRetireConnectionID{
+ seq: 0,
+ })
+ if got, want := tc.lastPacket.dstConnID, testPeerConnID(1); !bytes.Equal(got, want) {
+ t.Fatalf("used destination conn id {%x}, want {%x}", got, want)
+ }
+}
+
+func TestConnIDPeerDoesNotAcknowledgeRetirement(t *testing.T) {
+ // "An endpoint SHOULD limit the number of connection IDs it has retired locally
+ // for which RETIRE_CONNECTION_ID frames have not yet been acknowledged."
+ // https://www.rfc-editor.org/rfc/rfc9000#section-5.1.2-6
+ tc := newTestConn(t, clientSide)
+ tc.handshake()
+ tc.ignoreFrame(frameTypeAck)
+ tc.ignoreFrame(frameTypeRetireConnectionID)
+
+ // Send a number of NEW_CONNECTION_ID frames, each retiring an old one.
+ for seq := int64(0); seq < 7; seq++ {
+ tc.writeFrames(packetType1RTT,
+ debugFrameNewConnectionID{
+ seq: seq + 2,
+ retirePriorTo: seq + 1,
+ connID: testPeerConnID(seq + 2),
+ })
+ // We're ignoring the RETIRE_CONNECTION_ID frames.
+ }
+ tc.wantFrame("number of retired, unacked conn ids is too large",
+ packetType1RTT, debugFrameConnectionCloseTransport{
+ code: errConnectionIDLimit,
+ })
+}
+
+func TestConnIDRepeatedNewConnectionIDFrame(t *testing.T) {
+ // "Receipt of the same [NEW_CONNECTION_ID] frame multiple times
+ // MUST NOT be treated as a connection error.
+ // https://www.rfc-editor.org/rfc/rfc9000#section-19.15-7
+ tc := newTestConn(t, clientSide)
+ tc.handshake()
+ tc.ignoreFrame(frameTypeAck)
+
+ for i := 0; i < 4; i++ {
+ tc.writeFrames(packetType1RTT,
+ debugFrameNewConnectionID{
+ seq: 2,
+ retirePriorTo: 1,
+ connID: testPeerConnID(2),
+ })
+ }
+ tc.wantFrame("peer asked for conn id to be retired",
+ packetType1RTT, debugFrameRetireConnectionID{
+ seq: 0,
+ })
+ tc.wantIdle("repeated NEW_CONNECTION_ID frames are not an error")
+}
+
+func TestConnIDForSequenceNumberChanges(t *testing.T) {
+ // "[...] if a sequence number is used for different connection IDs,
+ // the endpoint MAY treat that receipt as a connection error
+ // of type PROTOCOL_VIOLATION."
+ // https://www.rfc-editor.org/rfc/rfc9000#section-19.15-8
+ tc := newTestConn(t, clientSide)
+ tc.handshake()
+ tc.ignoreFrame(frameTypeAck)
+ tc.ignoreFrame(frameTypeRetireConnectionID)
+
+ tc.writeFrames(packetType1RTT,
+ debugFrameNewConnectionID{
+ seq: 2,
+ retirePriorTo: 1,
+ connID: testPeerConnID(2),
+ })
+ tc.writeFrames(packetType1RTT,
+ debugFrameNewConnectionID{
+ seq: 2,
+ retirePriorTo: 1,
+ connID: testPeerConnID(3),
+ })
+ tc.wantFrame("connection ID for sequence 0 has changed",
+ packetType1RTT, debugFrameConnectionCloseTransport{
+ code: errProtocolViolation,
+ })
+}
+
+func TestConnIDRetirePriorToAfterNewConnID(t *testing.T) {
+ // "Receiving a value in the Retire Prior To field that is greater than
+ // that in the Sequence Number field MUST be treated as a connection error
+ // of type FRAME_ENCODING_ERROR.
+ // https://www.rfc-editor.org/rfc/rfc9000#section-19.15-9
+ tc := newTestConn(t, serverSide)
+ tc.handshake()
+ tc.ignoreFrame(frameTypeAck)
+
+ tc.writeFrames(packetType1RTT,
+ debugFrameNewConnectionID{
+ retirePriorTo: 3,
+ seq: 2,
+ connID: testPeerConnID(2),
+ })
+ tc.wantFrame("invalid NEW_CONNECTION_ID: retired the new conn id",
+ packetType1RTT, debugFrameConnectionCloseTransport{
+ code: errFrameEncoding,
+ })
+}
+
+func TestConnIDAlreadyRetired(t *testing.T) {
+ // "An endpoint that receives a NEW_CONNECTION_ID frame with a
+ // sequence number smaller than the Retire Prior To field of a
+ // previously received NEW_CONNECTION_ID frame MUST send a
+ // corresponding RETIRE_CONNECTION_ID frame [...]"
+ // https://www.rfc-editor.org/rfc/rfc9000#section-19.15-11
+ tc := newTestConn(t, clientSide)
+ tc.handshake()
+ tc.ignoreFrame(frameTypeAck)
+
+ tc.writeFrames(packetType1RTT,
+ debugFrameNewConnectionID{
+ seq: 4,
+ retirePriorTo: 3,
+ connID: testPeerConnID(4),
+ })
+ tc.wantFrame("peer asked for conn id to be retired",
+ packetType1RTT, debugFrameRetireConnectionID{
+ seq: 0,
+ })
+ tc.wantFrame("peer asked for conn id to be retired",
+ packetType1RTT, debugFrameRetireConnectionID{
+ seq: 1,
+ })
+ tc.writeFrames(packetType1RTT,
+ debugFrameNewConnectionID{
+ seq: 2,
+ retirePriorTo: 0,
+ connID: testPeerConnID(2),
+ })
+ tc.wantFrame("NEW_CONNECTION_ID was for an already-retired ID",
+ packetType1RTT, debugFrameRetireConnectionID{
+ seq: 2,
+ })
+}
+
+func TestConnIDRepeatedRetireConnectionIDFrame(t *testing.T) {
+ tc := newTestConn(t, clientSide)
+ tc.handshake()
+ tc.ignoreFrame(frameTypeAck)
+
+ for i := 0; i < 4; i++ {
+ tc.writeFrames(packetType1RTT,
+ debugFrameRetireConnectionID{
+ seq: 0,
+ })
+ }
+ tc.wantFrame("issue new conn id after peer retires one",
+ packetType1RTT, debugFrameNewConnectionID{
+ retirePriorTo: 1,
+ seq: 2,
+ connID: testLocalConnID(2),
+ })
+ tc.wantIdle("repeated RETIRE_CONNECTION_ID frames are not an error")
+}
+
+func TestConnIDRetiredUnsent(t *testing.T) {
+ // "Receipt of a RETIRE_CONNECTION_ID frame containing a sequence number
+ // greater than any previously sent to the peer MUST be treated as a
+ // connection error of type PROTOCOL_VIOLATION."
+ // https://www.rfc-editor.org/rfc/rfc9000#section-19.16-7
+ tc := newTestConn(t, clientSide)
+ tc.handshake()
+ tc.ignoreFrame(frameTypeAck)
+
+ tc.writeFrames(packetType1RTT,
+ debugFrameRetireConnectionID{
+ seq: 2,
+ })
+ tc.wantFrame("invalid NEW_CONNECTION_ID: previous conn id is zero-length",
+ packetType1RTT, debugFrameConnectionCloseTransport{
+ code: errProtocolViolation,
+ })
+}
+
+func TestConnIDUsePreferredAddressConnID(t *testing.T) {
+ // Peer gives us a connection ID in the preferred address transport parameter.
+ // We don't use the preferred address at this time, but we should use the
+ // connection ID. (It isn't tied to any specific address.)
+ //
+ // This test will probably need updating if/when we start using the preferred address.
+ cid := testPeerConnID(10)
+ tc := newTestConn(t, serverSide, func(p *transportParameters) {
+ p.preferredAddrV4 = netip.MustParseAddrPort("0.0.0.0:0")
+ p.preferredAddrV6 = netip.MustParseAddrPort("[::0]:0")
+ p.preferredAddrConnID = cid
+ p.preferredAddrResetToken = make([]byte, 16)
+ })
+ tc.uncheckedHandshake()
+ tc.ignoreFrame(frameTypeAck)
+
+ tc.writeFrames(packetType1RTT,
+ debugFrameNewConnectionID{
+ seq: 2,
+ retirePriorTo: 1,
+ connID: []byte{0xff},
+ })
+ tc.wantFrame("peer asked for conn id 0 to be retired",
+ packetType1RTT, debugFrameRetireConnectionID{
+ seq: 0,
+ })
+ if got, want := tc.lastPacket.dstConnID, cid; !bytes.Equal(got, want) {
+ t.Fatalf("used destination conn id {%x}, want {%x} from preferred address transport parameter", got, want)
+ }
+}
+
+func TestConnIDPeerProvidesPreferredAddrAndTooManyConnIDs(t *testing.T) {
+ // Peer gives us more conn ids than our advertised limit,
+ // including a conn id in the preferred address transport parameter.
+ cid := testPeerConnID(10)
+ tc := newTestConn(t, serverSide, func(p *transportParameters) {
+ p.preferredAddrV4 = netip.MustParseAddrPort("0.0.0.0:0")
+ p.preferredAddrV6 = netip.MustParseAddrPort("[::0]:0")
+ p.preferredAddrConnID = cid
+ p.preferredAddrResetToken = make([]byte, 16)
+ })
+ tc.uncheckedHandshake()
+ tc.ignoreFrame(frameTypeAck)
+
+ tc.writeFrames(packetType1RTT,
+ debugFrameNewConnectionID{
+ seq: 2,
+ retirePriorTo: 0,
+ connID: testPeerConnID(2),
+ })
+ tc.wantFrame("peer provided 3 connection IDs, our limit is 2",
+ packetType1RTT, debugFrameConnectionCloseTransport{
+ code: errConnectionIDLimit,
+ })
+}
+
+func TestConnIDPeerWithZeroLengthIDProvidesPreferredAddr(t *testing.T) {
+ // Peer gives us more conn ids than our advertised limit,
+ // including a conn id in the preferred address transport parameter.
+ tc := newTestConn(t, serverSide, func(p *transportParameters) {
+ p.initialSrcConnID = []byte{}
+ p.preferredAddrV4 = netip.MustParseAddrPort("0.0.0.0:0")
+ p.preferredAddrV6 = netip.MustParseAddrPort("[::0]:0")
+ p.preferredAddrConnID = testPeerConnID(1)
+ p.preferredAddrResetToken = make([]byte, 16)
+ })
+ tc.peerConnID = []byte{}
+
+ tc.writeFrames(packetTypeInitial,
+ debugFrameCrypto{
+ data: tc.cryptoDataIn[tls.QUICEncryptionLevelInitial],
+ })
+ tc.wantFrame("peer with zero-length connection ID tried to provide another in transport parameters",
+ packetTypeInitial, debugFrameConnectionCloseTransport{
+ code: errProtocolViolation,
+ })
+}
+
+func TestConnIDInitialSrcConnIDMismatch(t *testing.T) {
+ // "Endpoints MUST validate that received [initial_source_connection_id]
+ // parameters match received connection ID values."
+ // https://www.rfc-editor.org/rfc/rfc9000#section-7.3-3
+ testSides(t, "", func(t *testing.T, side connSide) {
+ tc := newTestConn(t, side, func(p *transportParameters) {
+ p.initialSrcConnID = []byte("invalid")
+ })
+ tc.ignoreFrame(frameTypeAck)
+ tc.ignoreFrame(frameTypeCrypto)
+ tc.writeFrames(packetTypeInitial,
+ debugFrameCrypto{
+ data: tc.cryptoDataIn[tls.QUICEncryptionLevelInitial],
+ })
+ if side == clientSide {
+ // Server transport parameters are carried in the Handshake packet.
+ tc.writeFrames(packetTypeHandshake,
+ debugFrameCrypto{
+ data: tc.cryptoDataIn[tls.QUICEncryptionLevelHandshake],
+ })
+ }
+ tc.wantFrame("initial_source_connection_id transport parameter mismatch",
+ packetTypeInitial, debugFrameConnectionCloseTransport{
+ code: errTransportParameter,
+ })
+ })
+}
diff --git a/internal/quic/conn_loss.go b/internal/quic/conn_loss.go
new file mode 100644
index 0000000000..85bda314ec
--- /dev/null
+++ b/internal/quic/conn_loss.go
@@ -0,0 +1,83 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build go1.21
+
+package quic
+
+import "fmt"
+
+// handleAckOrLoss deals with the final fate of a packet we sent:
+// Either the peer acknowledges it, or we declare it lost.
+//
+// In order to handle packet loss, we must retain any information sent to the peer
+// until the peer has acknowledged it.
+//
+// When information is acknowledged, we can discard it.
+//
+// When information is lost, we mark it for retransmission.
+// See RFC 9000, Section 13.3 for a complete list of information which is retransmitted on loss.
+// https://www.rfc-editor.org/rfc/rfc9000#section-13.3
+func (c *Conn) handleAckOrLoss(space numberSpace, sent *sentPacket, fate packetFate) {
+ // The list of frames in a sent packet is marshaled into a buffer in the sentPacket
+ // by the packetWriter. Unmarshal that buffer here. This code must be kept in sync with
+ // packetWriter.append*.
+ //
+ // A sent packet meets its fate (acked or lost) only once, so it's okay to consume
+ // the sentPacket's buffer here.
+ for !sent.done() {
+ switch f := sent.next(); f {
+ default:
+ panic(fmt.Sprintf("BUG: unhandled acked/lost frame type %x", f))
+ case frameTypeAck:
+ // Unlike most information, loss of an ACK frame does not trigger
+ // retransmission. ACKs are sent in response to ack-eliciting packets,
+ // and always contain the latest information available.
+ //
+ // Acknowledgement of an ACK frame may allow us to discard information
+ // about older packets.
+ largest := packetNumber(sent.nextInt())
+ if fate == packetAcked {
+ c.acks[space].handleAck(largest)
+ }
+ case frameTypeCrypto:
+ start, end := sent.nextRange()
+ c.crypto[space].ackOrLoss(start, end, fate)
+ case frameTypeMaxData:
+ c.ackOrLossMaxData(sent.num, fate)
+ case frameTypeResetStream,
+ frameTypeStopSending,
+ frameTypeMaxStreamData,
+ frameTypeStreamDataBlocked:
+ id := streamID(sent.nextInt())
+ s := c.streamForID(id)
+ if s == nil {
+ continue
+ }
+ s.ackOrLoss(sent.num, f, fate)
+ case frameTypeStreamBase,
+ frameTypeStreamBase | streamFinBit:
+ id := streamID(sent.nextInt())
+ start, end := sent.nextRange()
+ s := c.streamForID(id)
+ if s == nil {
+ continue
+ }
+ fin := f&streamFinBit != 0
+ s.ackOrLossData(sent.num, start, end, fin, fate)
+ case frameTypeMaxStreamsBidi:
+ c.streams.remoteLimit[bidiStream].sendMax.ackLatestOrLoss(sent.num, fate)
+ case frameTypeMaxStreamsUni:
+ c.streams.remoteLimit[uniStream].sendMax.ackLatestOrLoss(sent.num, fate)
+ case frameTypeNewConnectionID:
+ seq := int64(sent.nextInt())
+ c.connIDState.ackOrLossNewConnectionID(sent.num, seq, fate)
+ case frameTypeRetireConnectionID:
+ seq := int64(sent.nextInt())
+ c.connIDState.ackOrLossRetireConnectionID(sent.num, seq, fate)
+ case frameTypeHandshakeDone:
+ c.handshakeConfirmed.ackOrLoss(sent.num, fate)
+ }
+ }
+}
diff --git a/internal/quic/conn_loss_test.go b/internal/quic/conn_loss_test.go
new file mode 100644
index 0000000000..9b88462518
--- /dev/null
+++ b/internal/quic/conn_loss_test.go
@@ -0,0 +1,685 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build go1.21
+
+package quic
+
+import (
+ "context"
+ "crypto/tls"
+ "fmt"
+ "testing"
+)
+
+// Frames may be retransmitted either when the packet containing the frame is lost, or on PTO.
+// lostFrameTest runs a test in both configurations.
+func lostFrameTest(t *testing.T, f func(t *testing.T, pto bool)) {
+ t.Run("lost", func(t *testing.T) {
+ f(t, false)
+ })
+ t.Run("pto", func(t *testing.T) {
+ f(t, true)
+ })
+}
+
+// triggerLossOrPTO causes the conn to declare the last sent packet lost,
+// or advances to the PTO timer.
+func (tc *testConn) triggerLossOrPTO(ptype packetType, pto bool) {
+ tc.t.Helper()
+ if pto {
+ if !tc.conn.loss.ptoTimerArmed {
+ tc.t.Fatalf("PTO timer not armed, expected it to be")
+ }
+ if *testVV {
+ tc.t.Logf("advancing to PTO timer")
+ }
+ tc.advanceTo(tc.conn.loss.timer)
+ return
+ }
+ if *testVV {
+ *testVV = false
+ defer func() {
+ tc.t.Logf("cause conn to declare last packet lost")
+ *testVV = true
+ }()
+ }
+ defer func(ignoreFrames map[byte]bool) {
+ tc.ignoreFrames = ignoreFrames
+ }(tc.ignoreFrames)
+ tc.ignoreFrames = map[byte]bool{
+ frameTypeAck: true,
+ frameTypePadding: true,
+ }
+ // Send three packets containing PINGs, and then respond with an ACK for the
+ // last one. This puts the last packet before the PINGs outside the packet
+ // reordering threshold, and it will be declared lost.
+ const lossThreshold = 3
+ var num packetNumber
+ for i := 0; i < lossThreshold; i++ {
+ tc.conn.ping(spaceForPacketType(ptype))
+ d := tc.readDatagram()
+ if d == nil {
+ tc.t.Fatalf("conn is idle; want PING frame")
+ }
+ if d.packets[0].ptype != ptype {
+ tc.t.Fatalf("conn sent %v packet; want %v", d.packets[0].ptype, ptype)
+ }
+ num = d.packets[0].num
+ }
+ tc.writeFrames(ptype, debugFrameAck{
+ ranges: []i64range[packetNumber]{
+ {num, num + 1},
+ },
+ })
+}
+
+func TestLostResetStreamFrame(t *testing.T) {
+ // "Cancellation of stream transmission, as carried in a RESET_STREAM frame,
+ // is sent until acknowledged or until all stream data is acknowledged by the peer [...]"
+ // https://www.rfc-editor.org/rfc/rfc9000.html#section-13.3-3.4
+ lostFrameTest(t, func(t *testing.T, pto bool) {
+ tc, s := newTestConnAndLocalStream(t, serverSide, uniStream, permissiveTransportParameters)
+ tc.ignoreFrame(frameTypeAck)
+
+ s.Reset(1)
+ tc.wantFrame("reset stream",
+ packetType1RTT, debugFrameResetStream{
+ id: s.id,
+ code: 1,
+ })
+
+ tc.triggerLossOrPTO(packetType1RTT, pto)
+ tc.wantFrame("resent RESET_STREAM frame",
+ packetType1RTT, debugFrameResetStream{
+ id: s.id,
+ code: 1,
+ })
+ })
+}
+
+func TestLostStopSendingFrame(t *testing.T) {
+ // "[...] a request to cancel stream transmission, as encoded in a STOP_SENDING frame,
+ // is sent until the receiving part of the stream enters either a "Data Recvd" or
+ // "Reset Recvd" state [...]"
+ // https://www.rfc-editor.org/rfc/rfc9000.html#section-13.3-3.5
+ //
+ // Technically, we can stop sending a STOP_SENDING frame if the peer sends
+ // us all the data for the stream or resets it. We don't bother tracking this,
+ // however, so we'll keep sending the frame until it is acked. This is harmless.
+ lostFrameTest(t, func(t *testing.T, pto bool) {
+ tc, s := newTestConnAndRemoteStream(t, serverSide, uniStream, permissiveTransportParameters)
+ tc.ignoreFrame(frameTypeAck)
+
+ s.CloseRead()
+ tc.wantFrame("stream is read-closed",
+ packetType1RTT, debugFrameStopSending{
+ id: s.id,
+ })
+
+ tc.triggerLossOrPTO(packetType1RTT, pto)
+ tc.wantFrame("resent STOP_SENDING frame",
+ packetType1RTT, debugFrameStopSending{
+ id: s.id,
+ })
+ })
+}
+
+func TestLostCryptoFrame(t *testing.T) {
+ // "Data sent in CRYPTO frames is retransmitted [...] until all data has been acknowledged."
+ // https://www.rfc-editor.org/rfc/rfc9000.html#section-13.3-3.1
+ lostFrameTest(t, func(t *testing.T, pto bool) {
+ tc := newTestConn(t, clientSide)
+ tc.ignoreFrame(frameTypeAck)
+
+ tc.wantFrame("client sends Initial CRYPTO frame",
+ packetTypeInitial, debugFrameCrypto{
+ data: tc.cryptoDataOut[tls.QUICEncryptionLevelInitial],
+ })
+ tc.triggerLossOrPTO(packetTypeInitial, pto)
+ tc.wantFrame("client resends Initial CRYPTO frame",
+ packetTypeInitial, debugFrameCrypto{
+ data: tc.cryptoDataOut[tls.QUICEncryptionLevelInitial],
+ })
+
+ tc.writeFrames(packetTypeInitial,
+ debugFrameCrypto{
+ data: tc.cryptoDataIn[tls.QUICEncryptionLevelInitial],
+ })
+ tc.writeFrames(packetTypeHandshake,
+ debugFrameCrypto{
+ data: tc.cryptoDataIn[tls.QUICEncryptionLevelHandshake],
+ })
+
+ tc.wantFrame("client sends Handshake CRYPTO frame",
+ packetTypeHandshake, debugFrameCrypto{
+ data: tc.cryptoDataOut[tls.QUICEncryptionLevelHandshake],
+ })
+ tc.wantFrame("client provides server with an additional connection ID",
+ packetType1RTT, debugFrameNewConnectionID{
+ seq: 1,
+ connID: testLocalConnID(1),
+ })
+ tc.triggerLossOrPTO(packetTypeHandshake, pto)
+ tc.wantFrame("client resends Handshake CRYPTO frame",
+ packetTypeHandshake, debugFrameCrypto{
+ data: tc.cryptoDataOut[tls.QUICEncryptionLevelHandshake],
+ })
+ })
+}
+
+func TestLostStreamFrameEmpty(t *testing.T) {
+ // A STREAM frame opening a stream, but containing no stream data, should
+ // be retransmitted if lost.
+ lostFrameTest(t, func(t *testing.T, pto bool) {
+ ctx := canceledContext()
+ tc := newTestConn(t, clientSide, permissiveTransportParameters)
+ tc.handshake()
+ tc.ignoreFrame(frameTypeAck)
+
+ c, err := tc.conn.NewStream(ctx)
+ if err != nil {
+ t.Fatalf("NewStream: %v", err)
+ }
+ c.Write(nil) // open the stream
+ tc.wantFrame("created bidirectional stream 0",
+ packetType1RTT, debugFrameStream{
+ id: newStreamID(clientSide, bidiStream, 0),
+ data: []byte{},
+ })
+
+ tc.triggerLossOrPTO(packetType1RTT, pto)
+ tc.wantFrame("resent stream frame",
+ packetType1RTT, debugFrameStream{
+ id: newStreamID(clientSide, bidiStream, 0),
+ data: []byte{},
+ })
+ })
+}
+
+func TestLostStreamWithData(t *testing.T) {
+ // "Application data sent in STREAM frames is retransmitted in new STREAM
+ // frames unless the endpoint has sent a RESET_STREAM for that stream."
+ // https://www.rfc-editor.org/rfc/rfc9000#section-13.3-3.2
+ //
+ // TODO: Lost stream frame after RESET_STREAM
+ lostFrameTest(t, func(t *testing.T, pto bool) {
+ data := []byte{0, 1, 2, 3, 4, 5, 6, 7}
+ tc, s := newTestConnAndLocalStream(t, serverSide, uniStream, func(p *transportParameters) {
+ p.initialMaxStreamsUni = 1
+ p.initialMaxData = 1 << 20
+ p.initialMaxStreamDataUni = 1 << 20
+ })
+ s.Write(data[:4])
+ tc.wantFrame("send [0,4)",
+ packetType1RTT, debugFrameStream{
+ id: s.id,
+ off: 0,
+ data: data[:4],
+ })
+ s.Write(data[4:8])
+ tc.wantFrame("send [4,8)",
+ packetType1RTT, debugFrameStream{
+ id: s.id,
+ off: 4,
+ data: data[4:8],
+ })
+ s.CloseWrite()
+ tc.wantFrame("send FIN",
+ packetType1RTT, debugFrameStream{
+ id: s.id,
+ off: 8,
+ fin: true,
+ data: []byte{},
+ })
+
+ tc.triggerLossOrPTO(packetType1RTT, pto)
+ tc.wantFrame("resend data",
+ packetType1RTT, debugFrameStream{
+ id: s.id,
+ off: 0,
+ fin: true,
+ data: data[:8],
+ })
+ })
+}
+
+func TestLostStreamPartialLoss(t *testing.T) {
+ // Conn sends four STREAM packets.
+ // ACKs are received for the packets containing bytes 0 and 2.
+ // The remaining packets are declared lost.
+ // The Conn resends only the lost data.
+ //
+ // This test doesn't have a PTO mode, because the ACK for the packet containing byte 2
+ // starts the loss timer for the packet containing byte 1, and the PTO timer is not
+ // armed when the loss timer is.
+ data := []byte{0, 1, 2, 3}
+ tc, s := newTestConnAndLocalStream(t, serverSide, uniStream, func(p *transportParameters) {
+ p.initialMaxStreamsUni = 1
+ p.initialMaxData = 1 << 20
+ p.initialMaxStreamDataUni = 1 << 20
+ })
+ for i := range data {
+ s.Write(data[i : i+1])
+ tc.wantFrame(fmt.Sprintf("send STREAM frame with byte %v", i),
+ packetType1RTT, debugFrameStream{
+ id: s.id,
+ off: int64(i),
+ data: data[i : i+1],
+ })
+ if i%2 == 0 {
+ tc.writeAckForLatest()
+ }
+ }
+ const pto = false
+ tc.triggerLossOrPTO(packetType1RTT, pto)
+ tc.wantFrame("resend byte 1",
+ packetType1RTT, debugFrameStream{
+ id: s.id,
+ off: 1,
+ data: data[1:2],
+ })
+ tc.wantFrame("resend byte 3",
+ packetType1RTT, debugFrameStream{
+ id: s.id,
+ off: 3,
+ data: data[3:4],
+ })
+ tc.wantIdle("no more frames sent after packet loss")
+}
+
+func TestLostMaxDataFrame(t *testing.T) {
+ // "An updated value is sent in a MAX_DATA frame if the packet
+ // containing the most recently sent MAX_DATA frame is declared lost [...]"
+ // https://www.rfc-editor.org/rfc/rfc9000#section-13.3-3.7
+ lostFrameTest(t, func(t *testing.T, pto bool) {
+ const maxWindowSize = 32
+ buf := make([]byte, maxWindowSize)
+ tc, s := newTestConnAndRemoteStream(t, serverSide, uniStream, func(c *Config) {
+ c.MaxConnReadBufferSize = 32
+ })
+
+ // We send MAX_DATA = 63.
+ tc.writeFrames(packetType1RTT, debugFrameStream{
+ id: s.id,
+ off: 0,
+ data: make([]byte, maxWindowSize),
+ })
+ if n, err := s.Read(buf[:maxWindowSize-1]); err != nil || n != maxWindowSize-1 {
+ t.Fatalf("Read() = %v, %v; want %v, nil", n, err, maxWindowSize-1)
+ }
+ tc.wantFrame("conn window is extended after reading data",
+ packetType1RTT, debugFrameMaxData{
+ max: (maxWindowSize * 2) - 1,
+ })
+
+ // MAX_DATA = 64, which is only one more byte, so we don't send the frame.
+ if n, err := s.Read(buf); err != nil || n != 1 {
+ t.Fatalf("Read() = %v, %v; want %v, nil", n, err, 1)
+ }
+ tc.wantIdle("read doesn't extend window enough to send another MAX_DATA")
+
+ // The MAX_DATA = 63 packet was lost, so we send 64.
+ tc.triggerLossOrPTO(packetType1RTT, pto)
+ tc.wantFrame("resent MAX_DATA includes most current value",
+ packetType1RTT, debugFrameMaxData{
+ max: maxWindowSize * 2,
+ })
+ })
+}
+
+func TestLostMaxStreamDataFrame(t *testing.T) {
+ // "[...] an updated value is sent when the packet containing
+ // the most recent MAX_STREAM_DATA frame for a stream is lost"
+ // https://www.rfc-editor.org/rfc/rfc9000#section-13.3-3.8
+ lostFrameTest(t, func(t *testing.T, pto bool) {
+ const maxWindowSize = 32
+ buf := make([]byte, maxWindowSize)
+ tc, s := newTestConnAndRemoteStream(t, serverSide, uniStream, func(c *Config) {
+ c.MaxStreamReadBufferSize = maxWindowSize
+ })
+
+ // We send MAX_STREAM_DATA = 63.
+ tc.writeFrames(packetType1RTT, debugFrameStream{
+ id: s.id,
+ off: 0,
+ data: make([]byte, maxWindowSize),
+ })
+ if n, err := s.Read(buf[:maxWindowSize-1]); err != nil || n != maxWindowSize-1 {
+ t.Fatalf("Read() = %v, %v; want %v, nil", n, err, maxWindowSize-1)
+ }
+ tc.wantFrame("stream window is extended after reading data",
+ packetType1RTT, debugFrameMaxStreamData{
+ id: s.id,
+ max: (maxWindowSize * 2) - 1,
+ })
+
+ // MAX_STREAM_DATA = 64, which is only one more byte, so we don't send the frame.
+ if n, err := s.Read(buf); err != nil || n != 1 {
+ t.Fatalf("Read() = %v, %v; want %v, nil", n, err, 1)
+ }
+ tc.wantIdle("read doesn't extend window enough to send another MAX_STREAM_DATA")
+
+ // The MAX_STREAM_DATA = 63 packet was lost, so we send 64.
+ tc.triggerLossOrPTO(packetType1RTT, pto)
+ tc.wantFrame("resent MAX_STREAM_DATA includes most current value",
+ packetType1RTT, debugFrameMaxStreamData{
+ id: s.id,
+ max: maxWindowSize * 2,
+ })
+ })
+}
+
+func TestLostMaxStreamDataFrameAfterStreamFinReceived(t *testing.T) {
+ // "An endpoint SHOULD stop sending MAX_STREAM_DATA frames when
+ // the receiving part of the stream enters a "Size Known" or "Reset Recvd" state."
+ // https://www.rfc-editor.org/rfc/rfc9000#section-13.3-3.8
+ lostFrameTest(t, func(t *testing.T, pto bool) {
+ const maxWindowSize = 10
+ buf := make([]byte, maxWindowSize)
+ tc, s := newTestConnAndRemoteStream(t, serverSide, uniStream, func(c *Config) {
+ c.MaxStreamReadBufferSize = maxWindowSize
+ })
+
+ tc.writeFrames(packetType1RTT, debugFrameStream{
+ id: s.id,
+ off: 0,
+ data: make([]byte, maxWindowSize),
+ })
+ if n, err := s.Read(buf); err != nil || n != maxWindowSize {
+ t.Fatalf("Read() = %v, %v; want %v, nil", n, err, maxWindowSize)
+ }
+ tc.wantFrame("stream window is extended after reading data",
+ packetType1RTT, debugFrameMaxStreamData{
+ id: s.id,
+ max: 2 * maxWindowSize,
+ })
+
+ tc.writeFrames(packetType1RTT, debugFrameStream{
+ id: s.id,
+ off: maxWindowSize,
+ fin: true,
+ })
+
+ tc.ignoreFrame(frameTypePing)
+ tc.triggerLossOrPTO(packetType1RTT, pto)
+ tc.wantIdle("lost MAX_STREAM_DATA not resent for stream in 'size known'")
+ })
+}
+
+func TestLostMaxStreamsFrameMostRecent(t *testing.T) {
+ // "[...] an updated value is sent when a packet containing the
+ // most recent MAX_STREAMS for a stream type frame is declared lost [...]"
+ // https://www.rfc-editor.org/rfc/rfc9000#section-13.3-3.9
+ testStreamTypes(t, "", func(t *testing.T, styp streamType) {
+ lostFrameTest(t, func(t *testing.T, pto bool) {
+ ctx := canceledContext()
+ tc := newTestConn(t, serverSide, func(c *Config) {
+ c.MaxUniRemoteStreams = 1
+ c.MaxBidiRemoteStreams = 1
+ })
+ tc.handshake()
+ tc.ignoreFrame(frameTypeAck)
+ tc.writeFrames(packetType1RTT, debugFrameStream{
+ id: newStreamID(clientSide, styp, 0),
+ fin: true,
+ })
+ s, err := tc.conn.AcceptStream(ctx)
+ if err != nil {
+ t.Fatalf("AcceptStream() = %v", err)
+ }
+ s.CloseContext(ctx)
+ if styp == bidiStream {
+ tc.wantFrame("stream is closed",
+ packetType1RTT, debugFrameStream{
+ id: s.id,
+ data: []byte{},
+ fin: true,
+ })
+ tc.writeAckForAll()
+ }
+ tc.wantFrame("closing stream updates peer's MAX_STREAMS",
+ packetType1RTT, debugFrameMaxStreams{
+ streamType: styp,
+ max: 2,
+ })
+
+ tc.triggerLossOrPTO(packetType1RTT, pto)
+ tc.wantFrame("lost MAX_STREAMS is resent",
+ packetType1RTT, debugFrameMaxStreams{
+ streamType: styp,
+ max: 2,
+ })
+ })
+ })
+}
+
+func TestLostMaxStreamsFrameNotMostRecent(t *testing.T) {
+ // Send two MAX_STREAMS frames, lose the first one.
+ //
+ // No PTO mode for this test: The ack that causes the first frame
+ // to be lost arms the loss timer for the second, so the PTO timer is not armed.
+ const pto = false
+ ctx := canceledContext()
+ tc := newTestConn(t, serverSide, func(c *Config) {
+ c.MaxUniRemoteStreams = 2
+ })
+ tc.handshake()
+ tc.ignoreFrame(frameTypeAck)
+ for i := int64(0); i < 2; i++ {
+ tc.writeFrames(packetType1RTT, debugFrameStream{
+ id: newStreamID(clientSide, uniStream, i),
+ fin: true,
+ })
+ s, err := tc.conn.AcceptStream(ctx)
+ if err != nil {
+ t.Fatalf("AcceptStream() = %v", err)
+ }
+ if err := s.CloseContext(ctx); err != nil {
+ t.Fatalf("stream.Close() = %v", err)
+ }
+ tc.wantFrame("closing stream updates peer's MAX_STREAMS",
+ packetType1RTT, debugFrameMaxStreams{
+ streamType: uniStream,
+ max: 3 + i,
+ })
+ }
+
+ // The second MAX_STREAMS frame is acked.
+ tc.writeAckForLatest()
+
+ // The first MAX_STREAMS frame is lost.
+ tc.conn.ping(appDataSpace)
+ tc.wantFrame("connection should send a PING frame",
+ packetType1RTT, debugFramePing{})
+ tc.triggerLossOrPTO(packetType1RTT, pto)
+ tc.wantIdle("superseded MAX_DATA is not resent on loss")
+}
+
+func TestLostStreamDataBlockedFrame(t *testing.T) {
+ // "A new [STREAM_DATA_BLOCKED] frame is sent if a packet containing
+ // the most recent frame for a scope is lost [...]"
+ // https://www.rfc-editor.org/rfc/rfc9000#section-13.3-3.10
+ lostFrameTest(t, func(t *testing.T, pto bool) {
+ tc, s := newTestConnAndLocalStream(t, serverSide, uniStream, func(p *transportParameters) {
+ p.initialMaxStreamsUni = 1
+ p.initialMaxData = 1 << 20
+ })
+
+ w := runAsync(tc, func(ctx context.Context) (int, error) {
+ return s.WriteContext(ctx, []byte{0, 1, 2, 3})
+ })
+ defer w.cancel()
+ tc.wantFrame("write is blocked by flow control",
+ packetType1RTT, debugFrameStreamDataBlocked{
+ id: s.id,
+ max: 0,
+ })
+
+ tc.writeFrames(packetType1RTT, debugFrameMaxStreamData{
+ id: s.id,
+ max: 1,
+ })
+ tc.wantFrame("write makes some progress, but is still blocked by flow control",
+ packetType1RTT, debugFrameStreamDataBlocked{
+ id: s.id,
+ max: 1,
+ })
+ tc.wantFrame("write consuming available window",
+ packetType1RTT, debugFrameStream{
+ id: s.id,
+ off: 0,
+ data: []byte{0},
+ })
+
+ tc.triggerLossOrPTO(packetType1RTT, pto)
+ tc.wantFrame("STREAM_DATA_BLOCKED is resent",
+ packetType1RTT, debugFrameStreamDataBlocked{
+ id: s.id,
+ max: 1,
+ })
+ tc.wantFrame("STREAM is resent as well",
+ packetType1RTT, debugFrameStream{
+ id: s.id,
+ off: 0,
+ data: []byte{0},
+ })
+ })
+}
+
+func TestLostStreamDataBlockedFrameAfterStreamUnblocked(t *testing.T) {
+ // "A new [STREAM_DATA_BLOCKED] frame is sent [...] only while
+ // the endpoint is blocked on the corresponding limit."
+ // https://www.rfc-editor.org/rfc/rfc9000#section-13.3-3.10
+ lostFrameTest(t, func(t *testing.T, pto bool) {
+ tc, s := newTestConnAndLocalStream(t, serverSide, uniStream, func(p *transportParameters) {
+ p.initialMaxStreamsUni = 1
+ p.initialMaxData = 1 << 20
+ })
+
+ data := []byte{0, 1, 2, 3}
+ w := runAsync(tc, func(ctx context.Context) (int, error) {
+ return s.WriteContext(ctx, data)
+ })
+ defer w.cancel()
+ tc.wantFrame("write is blocked by flow control",
+ packetType1RTT, debugFrameStreamDataBlocked{
+ id: s.id,
+ max: 0,
+ })
+
+ tc.writeFrames(packetType1RTT, debugFrameMaxStreamData{
+ id: s.id,
+ max: 10,
+ })
+ tc.wantFrame("write completes after flow control available",
+ packetType1RTT, debugFrameStream{
+ id: s.id,
+ off: 0,
+ data: data,
+ })
+
+ tc.triggerLossOrPTO(packetType1RTT, pto)
+ tc.wantFrame("STREAM data is resent",
+ packetType1RTT, debugFrameStream{
+ id: s.id,
+ off: 0,
+ data: data,
+ })
+ tc.wantIdle("STREAM_DATA_BLOCKED is not resent, since the stream is not blocked")
+ })
+}
+
+func TestLostNewConnectionIDFrame(t *testing.T) {
+ // "New connection IDs are [...] retransmitted if the packet containing them is lost."
+ // https://www.rfc-editor.org/rfc/rfc9000#section-13.3-3.13
+ lostFrameTest(t, func(t *testing.T, pto bool) {
+ tc := newTestConn(t, serverSide)
+ tc.handshake()
+ tc.ignoreFrame(frameTypeAck)
+
+ tc.writeFrames(packetType1RTT,
+ debugFrameRetireConnectionID{
+ seq: 1,
+ })
+ tc.wantFrame("provide a new connection ID after peer retires old one",
+ packetType1RTT, debugFrameNewConnectionID{
+ seq: 2,
+ connID: testLocalConnID(2),
+ })
+
+ tc.triggerLossOrPTO(packetType1RTT, pto)
+ tc.wantFrame("resend new connection ID",
+ packetType1RTT, debugFrameNewConnectionID{
+ seq: 2,
+ connID: testLocalConnID(2),
+ })
+ })
+}
+
+func TestLostRetireConnectionIDFrame(t *testing.T) {
+ // "[...] retired connection IDs are [...] retransmitted
+ // if the packet containing them is lost."
+ // https://www.rfc-editor.org/rfc/rfc9000#section-13.3-3.13
+ lostFrameTest(t, func(t *testing.T, pto bool) {
+ tc := newTestConn(t, clientSide)
+ tc.handshake()
+ tc.ignoreFrame(frameTypeAck)
+
+ tc.writeFrames(packetType1RTT,
+ debugFrameNewConnectionID{
+ seq: 2,
+ retirePriorTo: 1,
+ connID: testPeerConnID(2),
+ })
+ tc.wantFrame("peer requested connection id be retired",
+ packetType1RTT, debugFrameRetireConnectionID{
+ seq: 0,
+ })
+
+ tc.triggerLossOrPTO(packetType1RTT, pto)
+ tc.wantFrame("resend RETIRE_CONNECTION_ID",
+ packetType1RTT, debugFrameRetireConnectionID{
+ seq: 0,
+ })
+ })
+}
+
+func TestLostHandshakeDoneFrame(t *testing.T) {
+ // "The HANDSHAKE_DONE frame MUST be retransmitted until it is acknowledged."
+ // https://www.rfc-editor.org/rfc/rfc9000.html#section-13.3-3.16
+ lostFrameTest(t, func(t *testing.T, pto bool) {
+ tc := newTestConn(t, serverSide)
+ tc.ignoreFrame(frameTypeAck)
+
+ tc.writeFrames(packetTypeInitial,
+ debugFrameCrypto{
+ data: tc.cryptoDataIn[tls.QUICEncryptionLevelInitial],
+ })
+ tc.wantFrame("server sends Initial CRYPTO frame",
+ packetTypeInitial, debugFrameCrypto{
+ data: tc.cryptoDataOut[tls.QUICEncryptionLevelInitial],
+ })
+ tc.wantFrame("server sends Handshake CRYPTO frame",
+ packetTypeHandshake, debugFrameCrypto{
+ data: tc.cryptoDataOut[tls.QUICEncryptionLevelHandshake],
+ })
+ tc.wantFrame("server provides an additional connection ID",
+ packetType1RTT, debugFrameNewConnectionID{
+ seq: 1,
+ connID: testLocalConnID(1),
+ })
+ tc.writeFrames(packetTypeHandshake,
+ debugFrameCrypto{
+ data: tc.cryptoDataIn[tls.QUICEncryptionLevelHandshake],
+ })
+
+ tc.wantFrame("server sends HANDSHAKE_DONE after handshake completes",
+ packetType1RTT, debugFrameHandshakeDone{})
+
+ tc.triggerLossOrPTO(packetType1RTT, pto)
+ tc.wantFrame("server resends HANDSHAKE_DONE",
+ packetType1RTT, debugFrameHandshakeDone{})
+ })
+}
diff --git a/internal/quic/conn_recv.go b/internal/quic/conn_recv.go
new file mode 100644
index 0000000000..9b1ba1ae10
--- /dev/null
+++ b/internal/quic/conn_recv.go
@@ -0,0 +1,478 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build go1.21
+
+package quic
+
+import (
+ "bytes"
+ "encoding/binary"
+ "errors"
+ "time"
+)
+
+func (c *Conn) handleDatagram(now time.Time, dgram *datagram) {
+ buf := dgram.b
+ c.loss.datagramReceived(now, len(buf))
+ if c.isDraining() {
+ return
+ }
+ for len(buf) > 0 {
+ var n int
+ ptype := getPacketType(buf)
+ switch ptype {
+ case packetTypeInitial:
+ if c.side == serverSide && len(dgram.b) < minimumClientInitialDatagramSize {
+ // Discard client-sent Initial packets in too-short datagrams.
+ // https://www.rfc-editor.org/rfc/rfc9000#section-14.1-4
+ return
+ }
+ n = c.handleLongHeader(now, ptype, initialSpace, c.keysInitial.r, buf)
+ case packetTypeHandshake:
+ n = c.handleLongHeader(now, ptype, handshakeSpace, c.keysHandshake.r, buf)
+ case packetType1RTT:
+ n = c.handle1RTT(now, buf)
+ case packetTypeVersionNegotiation:
+ c.handleVersionNegotiation(now, buf)
+ return
+ default:
+ return
+ }
+ if n <= 0 {
+ // Invalid data at the end of a datagram is ignored.
+ break
+ }
+ c.idleTimeout = now.Add(c.maxIdleTimeout)
+ buf = buf[n:]
+ }
+}
+
+func (c *Conn) handleLongHeader(now time.Time, ptype packetType, space numberSpace, k fixedKeys, buf []byte) int {
+ if !k.isSet() {
+ return skipLongHeaderPacket(buf)
+ }
+
+ pnumMax := c.acks[space].largestSeen()
+ p, n := parseLongHeaderPacket(buf, k, pnumMax)
+ if n < 0 {
+ return -1
+ }
+ if buf[0]&reservedLongBits != 0 {
+ // Reserved header bits must be 0.
+ // https://www.rfc-editor.org/rfc/rfc9000#section-17.2-8.2.1
+ c.abort(now, localTransportError(errProtocolViolation))
+ return -1
+ }
+ if p.version != quicVersion1 {
+ // The peer has changed versions on us mid-handshake?
+ c.abort(now, localTransportError(errProtocolViolation))
+ return -1
+ }
+
+ if !c.acks[space].shouldProcess(p.num) {
+ return n
+ }
+
+ if logPackets {
+ logInboundLongPacket(c, p)
+ }
+ c.connIDState.handlePacket(c, p.ptype, p.srcConnID)
+ ackEliciting := c.handleFrames(now, ptype, space, p.payload)
+ c.acks[space].receive(now, space, p.num, ackEliciting)
+ if p.ptype == packetTypeHandshake && c.side == serverSide {
+ c.loss.validateClientAddress()
+
+ // "[...] a server MUST discard Initial keys when it first successfully
+ // processes a Handshake packet [...]"
+ // https://www.rfc-editor.org/rfc/rfc9001#section-4.9.1-2
+ c.discardKeys(now, initialSpace)
+ }
+ return n
+}
+
+func (c *Conn) handle1RTT(now time.Time, buf []byte) int {
+ if !c.keysAppData.canRead() {
+ // 1-RTT packets extend to the end of the datagram,
+ // so skip the remainder of the datagram if we can't parse this.
+ return len(buf)
+ }
+
+ pnumMax := c.acks[appDataSpace].largestSeen()
+ p, err := parse1RTTPacket(buf, &c.keysAppData, connIDLen, pnumMax)
+ if err != nil {
+ // A localTransportError terminates the connection.
+ // Other errors indicate an unparseable packet, but otherwise may be ignored.
+ if _, ok := err.(localTransportError); ok {
+ c.abort(now, err)
+ }
+ return -1
+ }
+ if buf[0]&reserved1RTTBits != 0 {
+ // Reserved header bits must be 0.
+ // https://www.rfc-editor.org/rfc/rfc9000#section-17.3.1-4.8.1
+ c.abort(now, localTransportError(errProtocolViolation))
+ return -1
+ }
+
+ if !c.acks[appDataSpace].shouldProcess(p.num) {
+ return len(buf)
+ }
+
+ if logPackets {
+ logInboundShortPacket(c, p)
+ }
+ ackEliciting := c.handleFrames(now, packetType1RTT, appDataSpace, p.payload)
+ c.acks[appDataSpace].receive(now, appDataSpace, p.num, ackEliciting)
+ return len(buf)
+}
+
+var errVersionNegotiation = errors.New("server does not support QUIC version 1")
+
+func (c *Conn) handleVersionNegotiation(now time.Time, pkt []byte) {
+ if c.side != clientSide {
+ return // servers don't handle Version Negotiation packets
+ }
+ // "A client MUST discard any Version Negotiation packet if it has
+ // received and successfully processed any other packet [...]"
+ // https://www.rfc-editor.org/rfc/rfc9000#section-6.2-2
+ if !c.keysInitial.canRead() {
+ return // discarded Initial keys, connection is already established
+ }
+ if c.acks[initialSpace].seen.numRanges() != 0 {
+ return // processed at least one packet
+ }
+ _, srcConnID, versions := parseVersionNegotiation(pkt)
+ if len(c.connIDState.remote) < 1 || !bytes.Equal(c.connIDState.remote[0].cid, srcConnID) {
+ return // Source Connection ID doesn't match what we sent
+ }
+ for len(versions) >= 4 {
+ ver := binary.BigEndian.Uint32(versions)
+ if ver == 1 {
+ // "A client MUST discard a Version Negotiation packet that lists
+ // the QUIC version selected by the client."
+ // https://www.rfc-editor.org/rfc/rfc9000#section-6.2-2
+ return
+ }
+ versions = versions[4:]
+ }
+ // "A client that supports only this version of QUIC MUST
+ // abandon the current connection attempt if it receives
+ // a Version Negotiation packet, [with the two exceptions handled above]."
+ // https://www.rfc-editor.org/rfc/rfc9000#section-6.2-2
+ c.abortImmediately(now, errVersionNegotiation)
+}
+
+func (c *Conn) handleFrames(now time.Time, ptype packetType, space numberSpace, payload []byte) (ackEliciting bool) {
+ if len(payload) == 0 {
+ // "An endpoint MUST treat receipt of a packet containing no frames
+ // as a connection error of type PROTOCOL_VIOLATION."
+ // https://www.rfc-editor.org/rfc/rfc9000#section-12.4-3
+ c.abort(now, localTransportError(errProtocolViolation))
+ return false
+ }
+ // frameOK verifies that ptype is one of the packets in mask.
+ frameOK := func(c *Conn, ptype, mask packetType) (ok bool) {
+ if ptype&mask == 0 {
+ // "An endpoint MUST treat receipt of a frame in a packet type
+ // that is not permitted as a connection error of type
+ // PROTOCOL_VIOLATION."
+ // https://www.rfc-editor.org/rfc/rfc9000#section-12.4-3
+ c.abort(now, localTransportError(errProtocolViolation))
+ return false
+ }
+ return true
+ }
+ // Packet masks from RFC 9000 Table 3.
+ // https://www.rfc-editor.org/rfc/rfc9000#table-3
+ const (
+ IH_1 = packetTypeInitial | packetTypeHandshake | packetType1RTT
+ __01 = packetType0RTT | packetType1RTT
+ ___1 = packetType1RTT
+ )
+ for len(payload) > 0 {
+ switch payload[0] {
+ case frameTypePadding, frameTypeAck, frameTypeAckECN,
+ frameTypeConnectionCloseTransport, frameTypeConnectionCloseApplication:
+ default:
+ ackEliciting = true
+ }
+ n := -1
+ switch payload[0] {
+ case frameTypePadding:
+ // PADDING is OK in all spaces.
+ n = 1
+ case frameTypePing:
+ // PING is OK in all spaces.
+ //
+ // A PING frame causes us to respond with an ACK by virtue of being
+ // an ack-eliciting frame, but requires no other action.
+ n = 1
+ case frameTypeAck, frameTypeAckECN:
+ if !frameOK(c, ptype, IH_1) {
+ return
+ }
+ n = c.handleAckFrame(now, space, payload)
+ case frameTypeResetStream:
+ if !frameOK(c, ptype, __01) {
+ return
+ }
+ n = c.handleResetStreamFrame(now, space, payload)
+ case frameTypeStopSending:
+ if !frameOK(c, ptype, __01) {
+ return
+ }
+ n = c.handleStopSendingFrame(now, space, payload)
+ case frameTypeCrypto:
+ if !frameOK(c, ptype, IH_1) {
+ return
+ }
+ n = c.handleCryptoFrame(now, space, payload)
+ case frameTypeNewToken:
+ if !frameOK(c, ptype, ___1) {
+ return
+ }
+ _, n = consumeNewTokenFrame(payload)
+ case 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f: // STREAM
+ if !frameOK(c, ptype, __01) {
+ return
+ }
+ n = c.handleStreamFrame(now, space, payload)
+ case frameTypeMaxData:
+ if !frameOK(c, ptype, __01) {
+ return
+ }
+ n = c.handleMaxDataFrame(now, payload)
+ case frameTypeMaxStreamData:
+ if !frameOK(c, ptype, __01) {
+ return
+ }
+ n = c.handleMaxStreamDataFrame(now, payload)
+ case frameTypeMaxStreamsBidi, frameTypeMaxStreamsUni:
+ if !frameOK(c, ptype, __01) {
+ return
+ }
+ n = c.handleMaxStreamsFrame(now, payload)
+ case frameTypeDataBlocked:
+ if !frameOK(c, ptype, __01) {
+ return
+ }
+ _, n = consumeDataBlockedFrame(payload)
+ case frameTypeStreamsBlockedBidi, frameTypeStreamsBlockedUni:
+ if !frameOK(c, ptype, __01) {
+ return
+ }
+ _, _, n = consumeStreamsBlockedFrame(payload)
+ case frameTypeStreamDataBlocked:
+ if !frameOK(c, ptype, __01) {
+ return
+ }
+ _, _, n = consumeStreamDataBlockedFrame(payload)
+ case frameTypeNewConnectionID:
+ if !frameOK(c, ptype, __01) {
+ return
+ }
+ n = c.handleNewConnectionIDFrame(now, space, payload)
+ case frameTypeRetireConnectionID:
+ if !frameOK(c, ptype, __01) {
+ return
+ }
+ n = c.handleRetireConnectionIDFrame(now, space, payload)
+ case frameTypeConnectionCloseTransport:
+ // Transport CONNECTION_CLOSE is OK in all spaces.
+ n = c.handleConnectionCloseTransportFrame(now, payload)
+ case frameTypeConnectionCloseApplication:
+ if !frameOK(c, ptype, __01) {
+ return
+ }
+ n = c.handleConnectionCloseApplicationFrame(now, payload)
+ case frameTypeHandshakeDone:
+ if !frameOK(c, ptype, ___1) {
+ return
+ }
+ n = c.handleHandshakeDoneFrame(now, space, payload)
+ }
+ if n < 0 {
+ c.abort(now, localTransportError(errFrameEncoding))
+ return false
+ }
+ payload = payload[n:]
+ }
+ return ackEliciting
+}
+
+func (c *Conn) handleAckFrame(now time.Time, space numberSpace, payload []byte) int {
+ c.loss.receiveAckStart()
+ largest, ackDelay, n := consumeAckFrame(payload, func(rangeIndex int, start, end packetNumber) {
+ if end > c.loss.nextNumber(space) {
+ // Acknowledgement of a packet we never sent.
+ c.abort(now, localTransportError(errProtocolViolation))
+ return
+ }
+ c.loss.receiveAckRange(now, space, rangeIndex, start, end, c.handleAckOrLoss)
+ })
+ // Prior to receiving the peer's transport parameters, we cannot
+ // interpret the ACK Delay field because we don't know the ack_delay_exponent
+ // to apply.
+ //
+ // For servers, we should always know the ack_delay_exponent because the
+ // client's transport parameters are carried in its Initial packets and we
+ // won't send an ack-eliciting Initial packet until after receiving the last
+ // client Initial packet.
+ //
+ // For clients, we won't receive the server's transport parameters until handling
+ // its Handshake flight, which will probably happen after reading its ACK for our
+ // Initial packet(s). However, the peer's acknowledgement delay cannot reduce our
+ // adjusted RTT sample below min_rtt, and min_rtt is generally going to be set
+ // by the packet containing the ACK for our Initial flight. Therefore, the
+ // ACK Delay for an ACK in the Initial space is likely to be ignored anyway.
+ //
+ // Long story short, setting the delay to 0 prior to reading transport parameters
+ // is usually going to have no effect, will have only a minor effect in the rare
+ // cases when it happens, and there aren't any good alternatives anyway since we
+ // can't interpret the ACK Delay field without knowing the exponent.
+ var delay time.Duration
+ if c.peerAckDelayExponent >= 0 {
+ delay = ackDelay.Duration(uint8(c.peerAckDelayExponent))
+ }
+ c.loss.receiveAckEnd(now, space, delay, c.handleAckOrLoss)
+ if space == appDataSpace {
+ c.keysAppData.handleAckFor(largest)
+ }
+ return n
+}
+
+func (c *Conn) handleMaxDataFrame(now time.Time, payload []byte) int {
+ maxData, n := consumeMaxDataFrame(payload)
+ if n < 0 {
+ return -1
+ }
+ c.streams.outflow.setMaxData(maxData)
+ return n
+}
+
+func (c *Conn) handleMaxStreamDataFrame(now time.Time, payload []byte) int {
+ id, maxStreamData, n := consumeMaxStreamDataFrame(payload)
+ if n < 0 {
+ return -1
+ }
+ if s := c.streamForFrame(now, id, sendStream); s != nil {
+ if err := s.handleMaxStreamData(maxStreamData); err != nil {
+ c.abort(now, err)
+ return -1
+ }
+ }
+ return n
+}
+
+func (c *Conn) handleMaxStreamsFrame(now time.Time, payload []byte) int {
+ styp, max, n := consumeMaxStreamsFrame(payload)
+ if n < 0 {
+ return -1
+ }
+ c.streams.localLimit[styp].setMax(max)
+ return n
+}
+
+func (c *Conn) handleResetStreamFrame(now time.Time, space numberSpace, payload []byte) int {
+ id, code, finalSize, n := consumeResetStreamFrame(payload)
+ if n < 0 {
+ return -1
+ }
+ if s := c.streamForFrame(now, id, recvStream); s != nil {
+ if err := s.handleReset(code, finalSize); err != nil {
+ c.abort(now, err)
+ }
+ }
+ return n
+}
+
+func (c *Conn) handleStopSendingFrame(now time.Time, space numberSpace, payload []byte) int {
+ id, code, n := consumeStopSendingFrame(payload)
+ if n < 0 {
+ return -1
+ }
+ if s := c.streamForFrame(now, id, sendStream); s != nil {
+ if err := s.handleStopSending(code); err != nil {
+ c.abort(now, err)
+ }
+ }
+ return n
+}
+
+func (c *Conn) handleCryptoFrame(now time.Time, space numberSpace, payload []byte) int {
+ off, data, n := consumeCryptoFrame(payload)
+ err := c.handleCrypto(now, space, off, data)
+ if err != nil {
+ c.abort(now, err)
+ return -1
+ }
+ return n
+}
+
+func (c *Conn) handleStreamFrame(now time.Time, space numberSpace, payload []byte) int {
+ id, off, fin, b, n := consumeStreamFrame(payload)
+ if n < 0 {
+ return -1
+ }
+ if s := c.streamForFrame(now, id, recvStream); s != nil {
+ if err := s.handleData(off, b, fin); err != nil {
+ c.abort(now, err)
+ }
+ }
+ return n
+}
+
+func (c *Conn) handleNewConnectionIDFrame(now time.Time, space numberSpace, payload []byte) int {
+ seq, retire, connID, resetToken, n := consumeNewConnectionIDFrame(payload)
+ if n < 0 {
+ return -1
+ }
+ if err := c.connIDState.handleNewConnID(seq, retire, connID, resetToken); err != nil {
+ c.abort(now, err)
+ }
+ return n
+}
+
+func (c *Conn) handleRetireConnectionIDFrame(now time.Time, space numberSpace, payload []byte) int {
+ seq, n := consumeRetireConnectionIDFrame(payload)
+ if n < 0 {
+ return -1
+ }
+ if err := c.connIDState.handleRetireConnID(c, seq); err != nil {
+ c.abort(now, err)
+ }
+ return n
+}
+
+func (c *Conn) handleConnectionCloseTransportFrame(now time.Time, payload []byte) int {
+ code, _, reason, n := consumeConnectionCloseTransportFrame(payload)
+ if n < 0 {
+ return -1
+ }
+ c.enterDraining(peerTransportError{code: code, reason: reason})
+ return n
+}
+
+func (c *Conn) handleConnectionCloseApplicationFrame(now time.Time, payload []byte) int {
+ code, reason, n := consumeConnectionCloseApplicationFrame(payload)
+ if n < 0 {
+ return -1
+ }
+ c.enterDraining(&ApplicationError{Code: code, Reason: reason})
+ return n
+}
+
+func (c *Conn) handleHandshakeDoneFrame(now time.Time, space numberSpace, payload []byte) int {
+ if c.side == serverSide {
+ // Clients should never send HANDSHAKE_DONE.
+ // https://www.rfc-editor.org/rfc/rfc9000#section-19.20-4
+ c.abort(now, localTransportError(errProtocolViolation))
+ return -1
+ }
+ if !c.isClosingOrDraining() {
+ c.confirmHandshake(now)
+ }
+ return 1
+}
diff --git a/internal/quic/conn_send.go b/internal/quic/conn_send.go
new file mode 100644
index 0000000000..00b02c2a31
--- /dev/null
+++ b/internal/quic/conn_send.go
@@ -0,0 +1,351 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build go1.21
+
+package quic
+
+import (
+ "crypto/tls"
+ "errors"
+ "time"
+)
+
+// maybeSend sends datagrams, if possible.
+//
+// If sending is blocked by pacing, it returns the next time
+// a datagram may be sent.
+//
+// If sending is blocked indefinitely, it returns the zero Time.
+func (c *Conn) maybeSend(now time.Time) (next time.Time) {
+ // Assumption: The congestion window is not underutilized.
+ // If congestion control, pacing, and anti-amplification all permit sending,
+ // but we have no packet to send, then we will declare the window underutilized.
+ c.loss.cc.setUnderutilized(false)
+
+ // Send one datagram on each iteration of this loop,
+ // until we hit a limit or run out of data to send.
+ //
+ // For each number space where we have write keys,
+ // attempt to construct a packet in that space.
+ // If the packet contains no frames (we have no data in need of sending),
+ // abandon the packet.
+ //
+ // Speculatively constructing packets means we don't need
+ // separate code paths for "do we have data to send?" and
+ // "send the data" that need to be kept in sync.
+ for {
+ limit, next := c.loss.sendLimit(now)
+ if limit == ccBlocked {
+ // If anti-amplification blocks sending, then no packet can be sent.
+ return next
+ }
+ if !c.sendOK(now) {
+ return time.Time{}
+ }
+ // We may still send ACKs, even if congestion control or pacing limit sending.
+
+ // Prepare to write a datagram of at most maxSendSize bytes.
+ c.w.reset(c.loss.maxSendSize())
+
+ dstConnID, ok := c.connIDState.dstConnID()
+ if !ok {
+ // It is currently not possible for us to end up without a connection ID,
+ // but handle the case anyway.
+ return time.Time{}
+ }
+
+ // Initial packet.
+ pad := false
+ var sentInitial *sentPacket
+ if c.keysInitial.canWrite() {
+ pnumMaxAcked := c.acks[initialSpace].largestSeen()
+ pnum := c.loss.nextNumber(initialSpace)
+ p := longPacket{
+ ptype: packetTypeInitial,
+ version: quicVersion1,
+ num: pnum,
+ dstConnID: dstConnID,
+ srcConnID: c.connIDState.srcConnID(),
+ }
+ c.w.startProtectedLongHeaderPacket(pnumMaxAcked, p)
+ c.appendFrames(now, initialSpace, pnum, limit)
+ if logPackets {
+ logSentPacket(c, packetTypeInitial, pnum, p.srcConnID, p.dstConnID, c.w.payload())
+ }
+ sentInitial = c.w.finishProtectedLongHeaderPacket(pnumMaxAcked, c.keysInitial.w, p)
+ if sentInitial != nil {
+ // Client initial packets need to be sent in a datagram padded to
+ // at least 1200 bytes. We can't add the padding yet, however,
+ // since we may want to coalesce additional packets with this one.
+ if c.side == clientSide {
+ pad = true
+ }
+ }
+ }
+
+ // Handshake packet.
+ if c.keysHandshake.canWrite() {
+ pnumMaxAcked := c.acks[handshakeSpace].largestSeen()
+ pnum := c.loss.nextNumber(handshakeSpace)
+ p := longPacket{
+ ptype: packetTypeHandshake,
+ version: quicVersion1,
+ num: pnum,
+ dstConnID: dstConnID,
+ srcConnID: c.connIDState.srcConnID(),
+ }
+ c.w.startProtectedLongHeaderPacket(pnumMaxAcked, p)
+ c.appendFrames(now, handshakeSpace, pnum, limit)
+ if logPackets {
+ logSentPacket(c, packetTypeHandshake, pnum, p.srcConnID, p.dstConnID, c.w.payload())
+ }
+ if sent := c.w.finishProtectedLongHeaderPacket(pnumMaxAcked, c.keysHandshake.w, p); sent != nil {
+ c.loss.packetSent(now, handshakeSpace, sent)
+ if c.side == clientSide {
+ // "[...] a client MUST discard Initial keys when it first
+ // sends a Handshake packet [...]"
+ // https://www.rfc-editor.org/rfc/rfc9001.html#section-4.9.1-2
+ c.discardKeys(now, initialSpace)
+ }
+ }
+ }
+
+ // 1-RTT packet.
+ if c.keysAppData.canWrite() {
+ pnumMaxAcked := c.acks[appDataSpace].largestSeen()
+ pnum := c.loss.nextNumber(appDataSpace)
+ c.w.start1RTTPacket(pnum, pnumMaxAcked, dstConnID)
+ c.appendFrames(now, appDataSpace, pnum, limit)
+ if pad && len(c.w.payload()) > 0 {
+ // 1-RTT packets have no length field and extend to the end
+ // of the datagram, so if we're sending a datagram that needs
+ // padding we need to add it inside the 1-RTT packet.
+ c.w.appendPaddingTo(minimumClientInitialDatagramSize)
+ pad = false
+ }
+ if logPackets {
+ logSentPacket(c, packetType1RTT, pnum, nil, dstConnID, c.w.payload())
+ }
+ if sent := c.w.finish1RTTPacket(pnum, pnumMaxAcked, dstConnID, &c.keysAppData); sent != nil {
+ c.loss.packetSent(now, appDataSpace, sent)
+ }
+ }
+
+ buf := c.w.datagram()
+ if len(buf) == 0 {
+ if limit == ccOK {
+ // We have nothing to send, and congestion control does not
+ // block sending. The congestion window is underutilized.
+ c.loss.cc.setUnderutilized(true)
+ }
+ return next
+ }
+
+ if sentInitial != nil {
+ if pad {
+ // Pad out the datagram with zeros, coalescing the Initial
+ // packet with invalid packets that will be ignored by the peer.
+ // https://www.rfc-editor.org/rfc/rfc9000.html#section-14.1-1
+ for len(buf) < minimumClientInitialDatagramSize {
+ buf = append(buf, 0)
+ // Technically this padding isn't in any packet, but
+ // account it to the Initial packet in this datagram
+ // for purposes of flow control and loss recovery.
+ sentInitial.size++
+ sentInitial.inFlight = true
+ }
+ }
+ // If we're a client and this Initial packet is coalesced
+ // with a Handshake packet, then we've discarded Initial keys
+ // since constructing the packet and shouldn't record it as in-flight.
+ if c.keysInitial.canWrite() {
+ c.loss.packetSent(now, initialSpace, sentInitial)
+ }
+ }
+
+ c.listener.sendDatagram(buf, c.peerAddr)
+ }
+}
+
+func (c *Conn) appendFrames(now time.Time, space numberSpace, pnum packetNumber, limit ccLimit) {
+ if c.lifetime.localErr != nil {
+ c.appendConnectionCloseFrame(now, space, c.lifetime.localErr)
+ return
+ }
+
+ shouldSendAck := c.acks[space].shouldSendAck(now)
+ if limit != ccOK {
+ // ACKs are not limited by congestion control.
+ if shouldSendAck && c.appendAckFrame(now, space) {
+ c.acks[space].sentAck()
+ }
+ return
+ }
+ // We want to send an ACK frame if the ack controller wants to send a frame now,
+ // OR if we are sending a packet anyway and have ack-eliciting packets which we
+ // have not yet acked.
+ //
+ // We speculatively add ACK frames here, to put them at the front of the packet
+ // to avoid truncation.
+ //
+ // After adding all frames, if we don't need to send an ACK frame and have not
+ // added any other frames, we abandon the packet.
+ if c.appendAckFrame(now, space) {
+ defer func() {
+ // All frames other than ACK and PADDING are ack-eliciting,
+ // so if the packet is ack-eliciting we've added additional
+ // frames to it.
+ if !shouldSendAck && !c.w.sent.ackEliciting {
+ // There's nothing in this packet but ACK frames, and
+ // we don't want to send an ACK-only packet at this time.
+ // Abandoning the packet means we wrote an ACK frame for
+ // nothing, but constructing the frame is cheap.
+ c.w.abandonPacket()
+ return
+ }
+ // Either we are willing to send an ACK-only packet,
+ // or we've added additional frames.
+ c.acks[space].sentAck()
+ if !c.w.sent.ackEliciting && c.keysAppData.needAckEliciting() {
+ // The peer has initiated a key update.
+ // We haven't sent them any packets yet in the new phase.
+ // Make this an ack-eliciting packet.
+ // Their ack of this packet will complete the key update.
+ c.w.appendPingFrame()
+ }
+ }()
+ }
+ if limit != ccOK {
+ return
+ }
+ pto := c.loss.ptoExpired
+
+ // TODO: Add all the other frames we can send.
+
+ // CRYPTO
+ c.crypto[space].dataToSend(pto, func(off, size int64) int64 {
+ b, _ := c.w.appendCryptoFrame(off, int(size))
+ c.crypto[space].sendData(off, b)
+ return int64(len(b))
+ })
+
+ // Test-only PING frames.
+ if space == c.testSendPingSpace && c.testSendPing.shouldSendPTO(pto) {
+ if !c.w.appendPingFrame() {
+ return
+ }
+ c.testSendPing.setSent(pnum)
+ }
+
+ if space == appDataSpace {
+ // HANDSHAKE_DONE
+ if c.handshakeConfirmed.shouldSendPTO(pto) {
+ if !c.w.appendHandshakeDoneFrame() {
+ return
+ }
+ c.handshakeConfirmed.setSent(pnum)
+ }
+
+ // NEW_CONNECTION_ID, RETIRE_CONNECTION_ID
+ if !c.connIDState.appendFrames(&c.w, pnum, pto) {
+ return
+ }
+
+ // All stream-related frames. This should come last in the packet,
+ // so large amounts of STREAM data don't crowd out other frames
+ // we may need to send.
+ if !c.appendStreamFrames(&c.w, pnum, pto) {
+ return
+ }
+ }
+
+ // If this is a PTO probe and we haven't added an ack-eliciting frame yet,
+ // add a PING to make this an ack-eliciting probe.
+ //
+ // Technically, there are separate PTO timers for each number space.
+ // When a PTO timer expires, we MUST send an ack-eliciting packet in the
+ // timer's space. We SHOULD send ack-eliciting packets in every other space
+ // with in-flight data. (RFC 9002, section 6.2.4)
+ //
+ // What we actually do is send a single datagram containing an ack-eliciting packet
+ // for every space for which we have keys.
+ //
+ // We fill the PTO probe packets with new or unacknowledged data. For example,
+ // a PTO probe sent for the Initial space will generally retransmit previously
+ // sent but unacknowledged CRYPTO data.
+ //
+ // When sending a PTO probe datagram containing multiple packets, it is
+ // possible that an earlier packet will fill up the datagram, leaving no
+ // space for the remaining probe packet(s). This is not a problem in practice.
+ //
+ // A client discards Initial keys when it first sends a Handshake packet
+ // (RFC 9001 Section 4.9.1). Handshake keys are discarded when the handshake
+ // is confirmed (RFC 9001 Section 4.9.2). The PTO timer is not set for the
+ // Application Data packet number space until the handshake is confirmed
+ // (RFC 9002 Section 6.2.1). Therefore, the only times a PTO probe can fire
+ // while data for multiple spaces is in flight are:
+ //
+ // - a server's Initial or Handshake timers can fire while Initial and Handshake
+ // data is in flight; and
+ //
+ // - a client's Handshake timer can fire while Handshake and Application Data
+ // data is in flight.
+ //
+ // It is theoretically possible for a server's Initial CRYPTO data to overflow
+ // the maximum datagram size, but unlikely in practice; this space contains
+ // only the ServerHello TLS message, which is small. It's also unlikely that
+ // the Handshake PTO probe will fire while Initial data is in flight (this
+ // requires not just that the Initial CRYPTO data completely fill a datagram,
+ // but a quite specific arrangement of lost and retransmitted packets.)
+ // We don't bother worrying about this case here, since the worst case is
+ // that we send a PTO probe for the in-flight Initial data and drop the
+ // Handshake probe.
+ //
+ // If a client's Handshake PTO timer fires while Application Data data is in
+ // flight, it is possible that the resent Handshake CRYPTO data will crowd
+ // out the probe for the Application Data space. However, since this probe is
+ // optional (recall that the Application Data PTO timer is never set until
+ // after Handshake keys have been discarded), dropping it is acceptable.
+ if pto && !c.w.sent.ackEliciting {
+ c.w.appendPingFrame()
+ }
+}
+
+func (c *Conn) appendAckFrame(now time.Time, space numberSpace) bool {
+ seen, delay := c.acks[space].acksToSend(now)
+ if len(seen) == 0 {
+ return false
+ }
+ d := unscaledAckDelayFromDuration(delay, ackDelayExponent)
+ return c.w.appendAckFrame(seen, d)
+}
+
+func (c *Conn) appendConnectionCloseFrame(now time.Time, space numberSpace, err error) {
+ c.lifetime.connCloseSentTime = now
+ switch e := err.(type) {
+ case localTransportError:
+ c.w.appendConnectionCloseTransportFrame(transportError(e), 0, "")
+ case *ApplicationError:
+ if space != appDataSpace {
+ // "CONNECTION_CLOSE frames signaling application errors (type 0x1d)
+ // MUST only appear in the application data packet number space."
+ // https://www.rfc-editor.org/rfc/rfc9000#section-12.5-2.2
+ c.w.appendConnectionCloseTransportFrame(errApplicationError, 0, "")
+ } else {
+ c.w.appendConnectionCloseApplicationFrame(e.Code, e.Reason)
+ }
+ default:
+ // TLS alerts are sent using error codes [0x0100,0x01ff).
+ // https://www.rfc-editor.org/rfc/rfc9000#section-20.1-2.36.1
+ var alert tls.AlertError
+ if errors.As(err, &alert) {
+ // tls.AlertError is a uint8, so this can't exceed 0x01ff.
+ code := errTLSBase + transportError(alert)
+ c.w.appendConnectionCloseTransportFrame(code, 0, "")
+ } else {
+ c.w.appendConnectionCloseTransportFrame(errInternal, 0, "")
+ }
+ }
+}
diff --git a/internal/quic/conn_streams.go b/internal/quic/conn_streams.go
new file mode 100644
index 0000000000..a0793297e1
--- /dev/null
+++ b/internal/quic/conn_streams.go
@@ -0,0 +1,441 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build go1.21
+
+package quic
+
+import (
+ "context"
+ "sync"
+ "sync/atomic"
+ "time"
+)
+
+type streamsState struct {
+ queue queue[*Stream] // new, peer-created streams
+
+ streamsMu sync.Mutex
+ streams map[streamID]*Stream
+
+ // Limits on the number of streams, indexed by streamType.
+ localLimit [streamTypeCount]localStreamLimits
+ remoteLimit [streamTypeCount]remoteStreamLimits
+
+ // Peer configuration provided in transport parameters.
+ peerInitialMaxStreamDataRemote [streamTypeCount]int64 // streams opened by us
+ peerInitialMaxStreamDataBidiLocal int64 // streams opened by them
+
+ // Connection-level flow control.
+ inflow connInflow
+ outflow connOutflow
+
+ // Streams with frames to send are stored in one of two circular linked lists,
+ // depending on whether they require connection-level flow control.
+ needSend atomic.Bool
+ sendMu sync.Mutex
+ queueMeta streamRing // streams with any non-flow-controlled frames
+ queueData streamRing // streams with only flow-controlled frames
+}
+
+func (c *Conn) streamsInit() {
+ c.streams.streams = make(map[streamID]*Stream)
+ c.streams.queue = newQueue[*Stream]()
+ c.streams.localLimit[bidiStream].init()
+ c.streams.localLimit[uniStream].init()
+ c.streams.remoteLimit[bidiStream].init(c.config.maxBidiRemoteStreams())
+ c.streams.remoteLimit[uniStream].init(c.config.maxUniRemoteStreams())
+ c.inflowInit()
+}
+
+// AcceptStream waits for and returns the next stream created by the peer.
+func (c *Conn) AcceptStream(ctx context.Context) (*Stream, error) {
+ return c.streams.queue.get(ctx, c.testHooks)
+}
+
+// NewStream creates a stream.
+//
+// If the peer's maximum stream limit for the connection has been reached,
+// NewStream blocks until the limit is increased or the context expires.
+func (c *Conn) NewStream(ctx context.Context) (*Stream, error) {
+ return c.newLocalStream(ctx, bidiStream)
+}
+
+// NewSendOnlyStream creates a unidirectional, send-only stream.
+//
+// If the peer's maximum stream limit for the connection has been reached,
+// NewSendOnlyStream blocks until the limit is increased or the context expires.
+func (c *Conn) NewSendOnlyStream(ctx context.Context) (*Stream, error) {
+ return c.newLocalStream(ctx, uniStream)
+}
+
+func (c *Conn) newLocalStream(ctx context.Context, styp streamType) (*Stream, error) {
+ c.streams.streamsMu.Lock()
+ defer c.streams.streamsMu.Unlock()
+
+ num, err := c.streams.localLimit[styp].open(ctx, c)
+ if err != nil {
+ return nil, err
+ }
+
+ s := newStream(c, newStreamID(c.side, styp, num))
+ s.outmaxbuf = c.config.maxStreamWriteBufferSize()
+ s.outwin = c.streams.peerInitialMaxStreamDataRemote[styp]
+ if styp == bidiStream {
+ s.inmaxbuf = c.config.maxStreamReadBufferSize()
+ s.inwin = c.config.maxStreamReadBufferSize()
+ }
+ s.inUnlock()
+ s.outUnlock()
+
+ c.streams.streams[s.id] = s
+ return s, nil
+}
+
+// streamFrameType identifies which direction of a stream,
+// from the local perspective, a frame is associated with.
+//
+// For example, STREAM is a recvStream frame,
+// because it carries data from the peer to us.
+type streamFrameType uint8
+
+const (
+ sendStream = streamFrameType(iota) // for example, MAX_DATA
+ recvStream // for example, STREAM_DATA_BLOCKED
+)
+
+// streamForID returns the stream with the given id.
+// If the stream does not exist, it returns nil.
+func (c *Conn) streamForID(id streamID) *Stream {
+ c.streams.streamsMu.Lock()
+ defer c.streams.streamsMu.Unlock()
+ return c.streams.streams[id]
+}
+
+// streamForFrame returns the stream with the given id.
+// If the stream does not exist, it may be created.
+//
+// streamForFrame aborts the connection if the stream id, state, and frame type don't align.
+// For example, it aborts the connection with a STREAM_STATE error if a MAX_DATA frame
+// is received for a receive-only stream, or if the peer attempts to create a stream that
+// should be originated locally.
+//
+// streamForFrame returns nil if the stream no longer exists or if an error occurred.
+func (c *Conn) streamForFrame(now time.Time, id streamID, ftype streamFrameType) *Stream {
+ if id.streamType() == uniStream {
+ if (id.initiator() == c.side) != (ftype == sendStream) {
+ // Received an invalid frame for unidirectional stream.
+ // For example, a RESET_STREAM frame for a send-only stream.
+ c.abort(now, localTransportError(errStreamState))
+ return nil
+ }
+ }
+
+ c.streams.streamsMu.Lock()
+ defer c.streams.streamsMu.Unlock()
+ s, isOpen := c.streams.streams[id]
+ if s != nil {
+ return s
+ }
+
+ num := id.num()
+ styp := id.streamType()
+ if id.initiator() == c.side {
+ if num < c.streams.localLimit[styp].opened {
+ // This stream was created by us, and has been closed.
+ return nil
+ }
+ // Received a frame for a stream that should be originated by us,
+ // but which we never created.
+ c.abort(now, localTransportError(errStreamState))
+ return nil
+ } else {
+ // if isOpen, this is a stream that was implicitly opened by a
+ // previous frame for a larger-numbered stream, but we haven't
+ // actually created it yet.
+ if !isOpen && num < c.streams.remoteLimit[styp].opened {
+ // This stream was created by the peer, and has been closed.
+ return nil
+ }
+ }
+
+ prevOpened := c.streams.remoteLimit[styp].opened
+ if err := c.streams.remoteLimit[styp].open(id); err != nil {
+ c.abort(now, err)
+ return nil
+ }
+
+ // Receiving a frame for a stream implicitly creates all streams
+ // with the same initiator and type and a lower number.
+ // Add a nil entry to the streams map for each implicitly created stream.
+ for n := newStreamID(id.initiator(), id.streamType(), prevOpened); n < id; n += 4 {
+ c.streams.streams[n] = nil
+ }
+
+ s = newStream(c, id)
+ s.inmaxbuf = c.config.maxStreamReadBufferSize()
+ s.inwin = c.config.maxStreamReadBufferSize()
+ if id.streamType() == bidiStream {
+ s.outmaxbuf = c.config.maxStreamWriteBufferSize()
+ s.outwin = c.streams.peerInitialMaxStreamDataBidiLocal
+ }
+ s.inUnlock()
+ s.outUnlock()
+
+ c.streams.streams[id] = s
+ c.streams.queue.put(s)
+ return s
+}
+
+// maybeQueueStreamForSend marks a stream as containing frames that need sending.
+func (c *Conn) maybeQueueStreamForSend(s *Stream, state streamState) {
+ if state.wantQueue() == state.inQueue() {
+ return // already on the right queue
+ }
+ c.streams.sendMu.Lock()
+ defer c.streams.sendMu.Unlock()
+ state = s.state.load() // may have changed while waiting
+ c.queueStreamForSendLocked(s, state)
+
+ c.streams.needSend.Store(true)
+ c.wake()
+}
+
+// queueStreamForSendLocked moves a stream to the correct send queue,
+// or removes it from all queues.
+//
+// state is the last known stream state.
+func (c *Conn) queueStreamForSendLocked(s *Stream, state streamState) {
+ for {
+ wantQueue := state.wantQueue()
+ inQueue := state.inQueue()
+ if inQueue == wantQueue {
+ return // already on the right queue
+ }
+
+ switch inQueue {
+ case metaQueue:
+ c.streams.queueMeta.remove(s)
+ case dataQueue:
+ c.streams.queueData.remove(s)
+ }
+
+ switch wantQueue {
+ case metaQueue:
+ c.streams.queueMeta.append(s)
+ state = s.state.set(streamQueueMeta, streamQueueMeta|streamQueueData)
+ case dataQueue:
+ c.streams.queueData.append(s)
+ state = s.state.set(streamQueueData, streamQueueMeta|streamQueueData)
+ case noQueue:
+ state = s.state.set(0, streamQueueMeta|streamQueueData)
+ }
+
+ // If the stream state changed while we were moving the stream,
+ // we might now be on the wrong queue.
+ //
+ // For example:
+ // - stream has data to send: streamOutSendData|streamQueueData
+ // - appendStreamFrames sends all the data: streamQueueData
+ // - concurrently, more data is written: streamOutSendData|streamQueueData
+ // - appendStreamFrames calls us with the last state it observed
+ // (streamQueueData).
+ // - We remove the stream from the queue and observe the updated state:
+ // streamOutSendData
+ // - We realize that the stream needs to go back on the data queue.
+ //
+ // Go back around the loop to confirm we're on the correct queue.
+ }
+}
+
+// appendStreamFrames writes stream-related frames to the current packet.
+//
+// It returns true if no more frames need appending,
+// false if not everything fit in the current packet.
+func (c *Conn) appendStreamFrames(w *packetWriter, pnum packetNumber, pto bool) bool {
+ // MAX_DATA
+ if !c.appendMaxDataFrame(w, pnum, pto) {
+ return false
+ }
+
+ // MAX_STREAM_DATA
+ if !c.streams.remoteLimit[uniStream].appendFrame(w, uniStream, pnum, pto) {
+ return false
+ }
+ if !c.streams.remoteLimit[bidiStream].appendFrame(w, bidiStream, pnum, pto) {
+ return false
+ }
+
+ if pto {
+ return c.appendStreamFramesPTO(w, pnum)
+ }
+ if !c.streams.needSend.Load() {
+ return true
+ }
+ c.streams.sendMu.Lock()
+ defer c.streams.sendMu.Unlock()
+ // queueMeta contains streams with non-flow-controlled frames to send.
+ for c.streams.queueMeta.head != nil {
+ s := c.streams.queueMeta.head
+ state := s.state.load()
+ if state&(streamQueueMeta|streamConnRemoved) != streamQueueMeta {
+ panic("BUG: queueMeta stream is not streamQueueMeta")
+ }
+ if state&streamInSendMeta != 0 {
+ s.ingate.lock()
+ ok := s.appendInFramesLocked(w, pnum, pto)
+ state = s.inUnlockNoQueue()
+ if !ok {
+ return false
+ }
+ if state&streamInSendMeta != 0 {
+ panic("BUG: streamInSendMeta set after successfully appending frames")
+ }
+ }
+ if state&streamOutSendMeta != 0 {
+ s.outgate.lock()
+ // This might also append flow-controlled frames if we have any
+ // and available conn-level quota. That's fine.
+ ok := s.appendOutFramesLocked(w, pnum, pto)
+ state = s.outUnlockNoQueue()
+ // We're checking both ok and state, because appendOutFramesLocked
+ // might have filled up the packet with flow-controlled data.
+ // If so, we want to move the stream to queueData for any remaining frames.
+ if !ok && state&streamOutSendMeta != 0 {
+ return false
+ }
+ if state&streamOutSendMeta != 0 {
+ panic("BUG: streamOutSendMeta set after successfully appending frames")
+ }
+ }
+ // We've sent all frames for this stream, so remove it from the send queue.
+ c.streams.queueMeta.remove(s)
+ if state&(streamInDone|streamOutDone) == streamInDone|streamOutDone {
+ // Stream is finished, remove it from the conn.
+ state = s.state.set(streamConnRemoved, streamQueueMeta|streamConnRemoved)
+ delete(c.streams.streams, s.id)
+
+ // Record finalization of remote streams, to know when
+ // to extend the peer's stream limit.
+ if s.id.initiator() != c.side {
+ c.streams.remoteLimit[s.id.streamType()].close()
+ }
+ } else {
+ state = s.state.set(0, streamQueueMeta|streamConnRemoved)
+ }
+ // The stream may have flow-controlled data to send,
+ // or something might have added non-flow-controlled frames after we
+ // unlocked the stream.
+ // If so, put the stream back on a queue.
+ c.queueStreamForSendLocked(s, state)
+ }
+ // queueData contains streams with flow-controlled frames.
+ for c.streams.queueData.head != nil {
+ avail := c.streams.outflow.avail()
+ if avail == 0 {
+ break // no flow control quota available
+ }
+ s := c.streams.queueData.head
+ s.outgate.lock()
+ ok := s.appendOutFramesLocked(w, pnum, pto)
+ state := s.outUnlockNoQueue()
+ if !ok {
+ // We've sent some data for this stream, but it still has more to send.
+ // If the stream got a reasonable chance to put data in a packet,
+ // advance sendHead to the next stream in line, to avoid starvation.
+ // We'll come back to this stream after going through the others.
+ //
+ // If the packet was already mostly out of space, leave sendHead alone
+ // and come back to this stream again on the next packet.
+ if avail > 512 {
+ c.streams.queueData.head = s.next
+ }
+ return false
+ }
+ if state&streamQueueData == 0 {
+ panic("BUG: queueData stream is not streamQueueData")
+ }
+ if state&streamOutSendData != 0 {
+ // We must have run out of connection-level flow control:
+ // appendOutFramesLocked says it wrote all it can, but there's
+ // still data to send.
+ //
+ // Advance sendHead to the next stream in line to avoid starvation.
+ if c.streams.outflow.avail() != 0 {
+ panic("BUG: streamOutSendData set and flow control available after send")
+ }
+ c.streams.queueData.head = s.next
+ return true
+ }
+ c.streams.queueData.remove(s)
+ state = s.state.set(0, streamQueueData)
+ c.queueStreamForSendLocked(s, state)
+ }
+ if c.streams.queueMeta.head == nil && c.streams.queueData.head == nil {
+ c.streams.needSend.Store(false)
+ }
+ return true
+}
+
+// appendStreamFramesPTO writes stream-related frames to the current packet
+// for a PTO probe.
+//
+// It returns true if no more frames need appending,
+// false if not everything fit in the current packet.
+func (c *Conn) appendStreamFramesPTO(w *packetWriter, pnum packetNumber) bool {
+ c.streams.sendMu.Lock()
+ defer c.streams.sendMu.Unlock()
+ const pto = true
+ for _, s := range c.streams.streams {
+ const pto = true
+ s.ingate.lock()
+ inOK := s.appendInFramesLocked(w, pnum, pto)
+ s.inUnlockNoQueue()
+ if !inOK {
+ return false
+ }
+
+ s.outgate.lock()
+ outOK := s.appendOutFramesLocked(w, pnum, pto)
+ s.outUnlockNoQueue()
+ if !outOK {
+ return false
+ }
+ }
+ return true
+}
+
+// A streamRing is a circular linked list of streams.
+type streamRing struct {
+ head *Stream
+}
+
+// remove removes s from the ring.
+// s must be on the ring.
+func (r *streamRing) remove(s *Stream) {
+ if s.next == s {
+ r.head = nil // s was the last stream in the ring
+ } else {
+ s.prev.next = s.next
+ s.next.prev = s.prev
+ if r.head == s {
+ r.head = s.next
+ }
+ }
+}
+
+// append places s at the last position in the ring.
+// s must not be attached to any ring.
+func (r *streamRing) append(s *Stream) {
+ if r.head == nil {
+ r.head = s
+ s.next = s
+ s.prev = s
+ } else {
+ s.prev = r.head.prev
+ s.next = r.head
+ s.prev.next = s
+ s.next.prev = s
+ }
+}
diff --git a/internal/quic/conn_streams_test.go b/internal/quic/conn_streams_test.go
new file mode 100644
index 0000000000..69f982c3a6
--- /dev/null
+++ b/internal/quic/conn_streams_test.go
@@ -0,0 +1,480 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build go1.21
+
+package quic
+
+import (
+ "context"
+ "fmt"
+ "io"
+ "math"
+ "testing"
+)
+
+func TestStreamsCreate(t *testing.T) {
+ ctx := canceledContext()
+ tc := newTestConn(t, clientSide, permissiveTransportParameters)
+ tc.handshake()
+
+ c, err := tc.conn.NewStream(ctx)
+ if err != nil {
+ t.Fatalf("NewStream: %v", err)
+ }
+ c.Write(nil) // open the stream
+ tc.wantFrame("created bidirectional stream 0",
+ packetType1RTT, debugFrameStream{
+ id: 0, // client-initiated, bidi, number 0
+ data: []byte{},
+ })
+
+ c, err = tc.conn.NewSendOnlyStream(ctx)
+ if err != nil {
+ t.Fatalf("NewStream: %v", err)
+ }
+ c.Write(nil) // open the stream
+ tc.wantFrame("created unidirectional stream 0",
+ packetType1RTT, debugFrameStream{
+ id: 2, // client-initiated, uni, number 0
+ data: []byte{},
+ })
+
+ c, err = tc.conn.NewStream(ctx)
+ if err != nil {
+ t.Fatalf("NewStream: %v", err)
+ }
+ c.Write(nil) // open the stream
+ tc.wantFrame("created bidirectional stream 1",
+ packetType1RTT, debugFrameStream{
+ id: 4, // client-initiated, uni, number 4
+ data: []byte{},
+ })
+}
+
+func TestStreamsAccept(t *testing.T) {
+ ctx := canceledContext()
+ tc := newTestConn(t, serverSide)
+ tc.handshake()
+
+ tc.writeFrames(packetType1RTT,
+ debugFrameStream{
+ id: 0, // client-initiated, bidi, number 0
+ },
+ debugFrameStream{
+ id: 2, // client-initiated, uni, number 0
+ },
+ debugFrameStream{
+ id: 4, // client-initiated, bidi, number 1
+ })
+
+ for _, accept := range []struct {
+ id streamID
+ readOnly bool
+ }{
+ {0, false},
+ {2, true},
+ {4, false},
+ } {
+ s, err := tc.conn.AcceptStream(ctx)
+ if err != nil {
+ t.Fatalf("conn.AcceptStream() = %v, want stream %v", err, accept.id)
+ }
+ if got, want := s.id, accept.id; got != want {
+ t.Fatalf("conn.AcceptStream() = stream %v, want %v", got, want)
+ }
+ if got, want := s.IsReadOnly(), accept.readOnly; got != want {
+ t.Fatalf("stream %v: s.IsReadOnly() = %v, want %v", accept.id, got, want)
+ }
+ }
+
+ _, err := tc.conn.AcceptStream(ctx)
+ if err != context.Canceled {
+ t.Fatalf("conn.AcceptStream() = %v, want context.Canceled", err)
+ }
+}
+
+func TestStreamsBlockingAccept(t *testing.T) {
+ tc := newTestConn(t, serverSide)
+ tc.handshake()
+
+ a := runAsync(tc, func(ctx context.Context) (*Stream, error) {
+ return tc.conn.AcceptStream(ctx)
+ })
+ if _, err := a.result(); err != errNotDone {
+ tc.t.Fatalf("AcceptStream() = _, %v; want errNotDone", err)
+ }
+
+ sid := newStreamID(clientSide, bidiStream, 0)
+ tc.writeFrames(packetType1RTT,
+ debugFrameStream{
+ id: sid,
+ })
+
+ s, err := a.result()
+ if err != nil {
+ t.Fatalf("conn.AcceptStream() = _, %v, want stream", err)
+ }
+ if got, want := s.id, sid; got != want {
+ t.Fatalf("conn.AcceptStream() = stream %v, want %v", got, want)
+ }
+ if got, want := s.IsReadOnly(), false; got != want {
+ t.Fatalf("s.IsReadOnly() = %v, want %v", got, want)
+ }
+}
+
+func TestStreamsLocalStreamNotCreated(t *testing.T) {
+ // "An endpoint MUST terminate the connection with error STREAM_STATE_ERROR
+ // if it receives a STREAM frame for a locally initiated stream that has
+ // not yet been created [...]"
+ // https://www.rfc-editor.org/rfc/rfc9000.html#section-19.8-3
+ tc := newTestConn(t, serverSide)
+ tc.handshake()
+
+ tc.writeFrames(packetType1RTT,
+ debugFrameStream{
+ id: 1, // server-initiated, bidi, number 0
+ })
+ tc.wantFrame("peer sent STREAM frame for an uncreated local stream",
+ packetType1RTT, debugFrameConnectionCloseTransport{
+ code: errStreamState,
+ })
+}
+
+func TestStreamsLocalStreamClosed(t *testing.T) {
+ tc, s := newTestConnAndLocalStream(t, clientSide, uniStream, permissiveTransportParameters)
+ s.CloseWrite()
+ tc.wantFrame("FIN for closed stream",
+ packetType1RTT, debugFrameStream{
+ id: newStreamID(clientSide, uniStream, 0),
+ fin: true,
+ data: []byte{},
+ })
+ tc.writeAckForAll()
+
+ tc.writeFrames(packetType1RTT, debugFrameStopSending{
+ id: newStreamID(clientSide, uniStream, 0),
+ })
+ tc.wantIdle("frame for finalized stream is ignored")
+
+ // ACKing the last stream packet should have cleaned up the stream.
+ // Check that we don't have any state left.
+ if got := len(tc.conn.streams.streams); got != 0 {
+ t.Fatalf("after close, len(tc.conn.streams.streams) = %v, want 0", got)
+ }
+ if tc.conn.streams.queueMeta.head != nil {
+ t.Fatalf("after close, stream send queue is not empty; should be")
+ }
+}
+
+func TestStreamsStreamSendOnly(t *testing.T) {
+ // "An endpoint MUST terminate the connection with error STREAM_STATE_ERROR
+ // if it receives a STREAM frame for a locally initiated stream that has
+ // not yet been created [...]"
+ // https://www.rfc-editor.org/rfc/rfc9000.html#section-19.8-3
+ ctx := canceledContext()
+ tc := newTestConn(t, serverSide, permissiveTransportParameters)
+ tc.handshake()
+
+ c, err := tc.conn.NewSendOnlyStream(ctx)
+ if err != nil {
+ t.Fatalf("NewStream: %v", err)
+ }
+ c.Write(nil) // open the stream
+ tc.wantFrame("created unidirectional stream 0",
+ packetType1RTT, debugFrameStream{
+ id: 3, // server-initiated, uni, number 0
+ data: []byte{},
+ })
+
+ tc.writeFrames(packetType1RTT,
+ debugFrameStream{
+ id: 3, // server-initiated, bidi, number 0
+ })
+ tc.wantFrame("peer sent STREAM frame for a send-only stream",
+ packetType1RTT, debugFrameConnectionCloseTransport{
+ code: errStreamState,
+ })
+}
+
+func TestStreamsWriteQueueFairness(t *testing.T) {
+ ctx := canceledContext()
+ const dataLen = 1 << 20
+ const numStreams = 3
+ tc := newTestConn(t, clientSide, func(p *transportParameters) {
+ p.initialMaxStreamsBidi = numStreams
+ p.initialMaxData = 1<<62 - 1
+ p.initialMaxStreamDataBidiRemote = dataLen
+ }, func(c *Config) {
+ c.MaxStreamWriteBufferSize = dataLen
+ })
+ tc.handshake()
+ tc.ignoreFrame(frameTypeAck)
+
+ // Create a number of streams, and write a bunch of data to them.
+ // The streams are not limited by flow control.
+ //
+ // The first stream we create is going to immediately consume all
+ // available congestion window.
+ //
+ // Once we've created all the remaining streams,
+ // we start sending acks back to open up the congestion window.
+ // We verify that all streams can make progress.
+ data := make([]byte, dataLen)
+ var streams []*Stream
+ for i := 0; i < numStreams; i++ {
+ s, err := tc.conn.NewStream(ctx)
+ if err != nil {
+ t.Fatal(err)
+ }
+ streams = append(streams, s)
+ if n, err := s.WriteContext(ctx, data); n != len(data) || err != nil {
+ t.Fatalf("s.WriteContext() = %v, %v; want %v, nil", n, err, len(data))
+ }
+ // Wait for the stream to finish writing whatever frames it can before
+ // congestion control blocks it.
+ tc.wait()
+ }
+
+ sent := make([]int64, len(streams))
+ for {
+ p := tc.readPacket()
+ if p == nil {
+ break
+ }
+ tc.writeFrames(packetType1RTT, debugFrameAck{
+ ranges: []i64range[packetNumber]{{0, p.num}},
+ })
+ for _, f := range p.frames {
+ sf, ok := f.(debugFrameStream)
+ if !ok {
+ t.Fatalf("got unexpected frame (want STREAM): %v", sf)
+ }
+ if got, want := sf.off, sent[sf.id.num()]; got != want {
+ t.Fatalf("got frame: %v\nwant offset: %v", sf, want)
+ }
+ sent[sf.id.num()] = sf.off + int64(len(sf.data))
+ // Look at the amount of data sent by all streams, excluding the first one.
+ // (The first stream got a head start when it consumed the initial window.)
+ //
+ // We expect that difference between the streams making the most and least progress
+ // so far will be less than the maximum datagram size.
+ minSent := sent[1]
+ maxSent := sent[1]
+ for _, s := range sent[2:] {
+ minSent = min(minSent, s)
+ maxSent = max(maxSent, s)
+ }
+ const maxDelta = maxUDPPayloadSize
+ if d := maxSent - minSent; d > maxDelta {
+ t.Fatalf("stream data sent: %v; delta=%v, want delta <= %v", sent, d, maxDelta)
+ }
+ }
+ }
+ // Final check that every stream sent the full amount of data expected.
+ for num, s := range sent {
+ if s != dataLen {
+ t.Errorf("stream %v sent %v bytes, want %v", num, s, dataLen)
+ }
+ }
+}
+
+func TestStreamsShutdown(t *testing.T) {
+ // These tests verify that a stream is removed from the Conn's map of live streams
+ // after it is fully shut down.
+ //
+ // Each case consists of a setup step, after which one stream should exist,
+ // and a shutdown step, after which no streams should remain in the Conn.
+ for _, test := range []struct {
+ name string
+ side streamSide
+ styp streamType
+ setup func(*testing.T, *testConn, *Stream)
+ shutdown func(*testing.T, *testConn, *Stream)
+ }{{
+ name: "closed",
+ side: localStream,
+ styp: uniStream,
+ setup: func(t *testing.T, tc *testConn, s *Stream) {
+ s.CloseContext(canceledContext())
+ },
+ shutdown: func(t *testing.T, tc *testConn, s *Stream) {
+ tc.writeAckForAll()
+ },
+ }, {
+ name: "local close",
+ side: localStream,
+ styp: bidiStream,
+ setup: func(t *testing.T, tc *testConn, s *Stream) {
+ tc.writeFrames(packetType1RTT, debugFrameResetStream{
+ id: s.id,
+ })
+ s.CloseContext(canceledContext())
+ },
+ shutdown: func(t *testing.T, tc *testConn, s *Stream) {
+ tc.writeAckForAll()
+ },
+ }, {
+ name: "remote reset",
+ side: localStream,
+ styp: bidiStream,
+ setup: func(t *testing.T, tc *testConn, s *Stream) {
+ s.CloseContext(canceledContext())
+ tc.wantIdle("all frames after CloseContext are ignored")
+ tc.writeAckForAll()
+ },
+ shutdown: func(t *testing.T, tc *testConn, s *Stream) {
+ tc.writeFrames(packetType1RTT, debugFrameResetStream{
+ id: s.id,
+ })
+ },
+ }, {
+ name: "local close",
+ side: remoteStream,
+ styp: uniStream,
+ setup: func(t *testing.T, tc *testConn, s *Stream) {
+ ctx := canceledContext()
+ tc.writeFrames(packetType1RTT, debugFrameStream{
+ id: s.id,
+ fin: true,
+ })
+ if n, err := s.ReadContext(ctx, make([]byte, 16)); n != 0 || err != io.EOF {
+ t.Errorf("ReadContext() = %v, %v; want 0, io.EOF", n, err)
+ }
+ },
+ shutdown: func(t *testing.T, tc *testConn, s *Stream) {
+ s.CloseRead()
+ },
+ }} {
+ name := fmt.Sprintf("%v/%v/%v", test.side, test.styp, test.name)
+ t.Run(name, func(t *testing.T) {
+ tc, s := newTestConnAndStream(t, serverSide, test.side, test.styp,
+ permissiveTransportParameters)
+ tc.ignoreFrame(frameTypeStreamBase)
+ tc.ignoreFrame(frameTypeStopSending)
+ test.setup(t, tc, s)
+ tc.wantIdle("conn should be idle after setup")
+ if got, want := len(tc.conn.streams.streams), 1; got != want {
+ t.Fatalf("after setup: %v streams in Conn's map; want %v", got, want)
+ }
+ test.shutdown(t, tc, s)
+ tc.wantIdle("conn should be idle after shutdown")
+ if got, want := len(tc.conn.streams.streams), 0; got != want {
+ t.Fatalf("after shutdown: %v streams in Conn's map; want %v", got, want)
+ }
+ })
+ }
+}
+
+func TestStreamsCreateAndCloseRemote(t *testing.T) {
+ // This test exercises creating new streams in response to frames
+ // from the peer, and cleaning up after streams are fully closed.
+ //
+ // It's overfitted to the current implementation, but works through
+ // a number of corner cases in that implementation.
+ //
+ // Disable verbose logging in this test: It sends a lot of packets,
+ // and they're not especially interesting on their own.
+ defer func(vv bool) {
+ *testVV = vv
+ }(*testVV)
+ *testVV = false
+ ctx := canceledContext()
+ tc := newTestConn(t, serverSide, permissiveTransportParameters)
+ tc.handshake()
+ tc.ignoreFrame(frameTypeAck)
+ type op struct {
+ id streamID
+ }
+ type streamOp op
+ type resetOp op
+ type acceptOp op
+ const noStream = math.MaxInt64
+ stringID := func(id streamID) string {
+ return fmt.Sprintf("%v/%v", id.streamType(), id.num())
+ }
+ for _, op := range []any{
+ "opening bidi/5 implicitly opens bidi/0-4",
+ streamOp{newStreamID(clientSide, bidiStream, 5)},
+ acceptOp{newStreamID(clientSide, bidiStream, 5)},
+ "bidi/3 was implicitly opened",
+ streamOp{newStreamID(clientSide, bidiStream, 3)},
+ acceptOp{newStreamID(clientSide, bidiStream, 3)},
+ resetOp{newStreamID(clientSide, bidiStream, 3)},
+ "bidi/3 is done, frames for it are discarded",
+ streamOp{newStreamID(clientSide, bidiStream, 3)},
+ "open and close some uni streams as well",
+ streamOp{newStreamID(clientSide, uniStream, 0)},
+ acceptOp{newStreamID(clientSide, uniStream, 0)},
+ streamOp{newStreamID(clientSide, uniStream, 1)},
+ acceptOp{newStreamID(clientSide, uniStream, 1)},
+ streamOp{newStreamID(clientSide, uniStream, 2)},
+ acceptOp{newStreamID(clientSide, uniStream, 2)},
+ resetOp{newStreamID(clientSide, uniStream, 1)},
+ resetOp{newStreamID(clientSide, uniStream, 0)},
+ resetOp{newStreamID(clientSide, uniStream, 2)},
+ "closing an implicitly opened stream causes us to accept it",
+ resetOp{newStreamID(clientSide, bidiStream, 0)},
+ acceptOp{newStreamID(clientSide, bidiStream, 0)},
+ resetOp{newStreamID(clientSide, bidiStream, 1)},
+ acceptOp{newStreamID(clientSide, bidiStream, 1)},
+ resetOp{newStreamID(clientSide, bidiStream, 2)},
+ acceptOp{newStreamID(clientSide, bidiStream, 2)},
+ "stream bidi/3 was reset previously",
+ resetOp{newStreamID(clientSide, bidiStream, 3)},
+ resetOp{newStreamID(clientSide, bidiStream, 4)},
+ acceptOp{newStreamID(clientSide, bidiStream, 4)},
+ "stream bidi/5 was reset previously",
+ resetOp{newStreamID(clientSide, bidiStream, 5)},
+ "stream bidi/6 was not implicitly opened",
+ resetOp{newStreamID(clientSide, bidiStream, 6)},
+ acceptOp{newStreamID(clientSide, bidiStream, 6)},
+ } {
+ if _, ok := op.(acceptOp); !ok {
+ if s, err := tc.conn.AcceptStream(ctx); err == nil {
+ t.Fatalf("accepted stream %v, want none", stringID(s.id))
+ }
+ }
+ switch op := op.(type) {
+ case string:
+ t.Log("# " + op)
+ case streamOp:
+ t.Logf("open stream %v", stringID(op.id))
+ tc.writeFrames(packetType1RTT, debugFrameStream{
+ id: streamID(op.id),
+ })
+ case resetOp:
+ t.Logf("reset stream %v", stringID(op.id))
+ tc.writeFrames(packetType1RTT, debugFrameResetStream{
+ id: op.id,
+ })
+ case acceptOp:
+ s, err := tc.conn.AcceptStream(ctx)
+ if err != nil {
+ t.Fatalf("AcceptStream() = %q; want stream %v", err, stringID(op.id))
+ }
+ if s.id != op.id {
+ t.Fatalf("accepted stram %v; want stream %v", err, stringID(op.id))
+ }
+ t.Logf("accepted stream %v", stringID(op.id))
+ // Immediately close the stream, so the stream becomes done when the
+ // peer closes its end.
+ s.CloseContext(ctx)
+ }
+ p := tc.readPacket()
+ if p != nil {
+ tc.writeFrames(p.ptype, debugFrameAck{
+ ranges: []i64range[packetNumber]{{0, p.num + 1}},
+ })
+ }
+ }
+ // Every stream should be fully closed now.
+ // Check that we don't have any state left.
+ if got := len(tc.conn.streams.streams); got != 0 {
+ t.Fatalf("after test, len(tc.conn.streams.streams) = %v, want 0", got)
+ }
+ if tc.conn.streams.queueMeta.head != nil {
+ t.Fatalf("after test, stream send queue is not empty; should be")
+ }
+}
diff --git a/internal/quic/conn_test.go b/internal/quic/conn_test.go
new file mode 100644
index 0000000000..6a359e89a1
--- /dev/null
+++ b/internal/quic/conn_test.go
@@ -0,0 +1,996 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build go1.21
+
+package quic
+
+import (
+ "bytes"
+ "context"
+ "crypto/tls"
+ "errors"
+ "flag"
+ "fmt"
+ "math"
+ "net/netip"
+ "reflect"
+ "strings"
+ "testing"
+ "time"
+)
+
+var testVV = flag.Bool("vv", false, "even more verbose test output")
+
+func TestConnTestConn(t *testing.T) {
+ tc := newTestConn(t, serverSide)
+ if got, want := tc.timeUntilEvent(), defaultMaxIdleTimeout; got != want {
+ t.Errorf("new conn timeout=%v, want %v (max_idle_timeout)", got, want)
+ }
+
+ var ranAt time.Time
+ tc.conn.runOnLoop(func(now time.Time, c *Conn) {
+ ranAt = now
+ })
+ if !ranAt.Equal(tc.now) {
+ t.Errorf("func ran on loop at %v, want %v", ranAt, tc.now)
+ }
+ tc.wait()
+
+ nextTime := tc.now.Add(defaultMaxIdleTimeout / 2)
+ tc.advanceTo(nextTime)
+ tc.conn.runOnLoop(func(now time.Time, c *Conn) {
+ ranAt = now
+ })
+ if !ranAt.Equal(nextTime) {
+ t.Errorf("func ran on loop at %v, want %v", ranAt, nextTime)
+ }
+ tc.wait()
+
+ tc.advanceToTimer()
+ if !tc.conn.exited {
+ t.Errorf("after advancing to idle timeout, exited = false, want true")
+ }
+}
+
+type testDatagram struct {
+ packets []*testPacket
+ paddedSize int
+}
+
+func (d testDatagram) String() string {
+ var b strings.Builder
+ fmt.Fprintf(&b, "datagram with %v packets", len(d.packets))
+ if d.paddedSize > 0 {
+ fmt.Fprintf(&b, " (padded to %v bytes)", d.paddedSize)
+ }
+ b.WriteString(":")
+ for _, p := range d.packets {
+ b.WriteString("\n")
+ b.WriteString(p.String())
+ }
+ return b.String()
+}
+
+type testPacket struct {
+ ptype packetType
+ version uint32
+ num packetNumber
+ keyPhaseBit bool
+ keyNumber int
+ dstConnID []byte
+ srcConnID []byte
+ frames []debugFrame
+}
+
+func (p testPacket) String() string {
+ var b strings.Builder
+ fmt.Fprintf(&b, " %v %v", p.ptype, p.num)
+ if p.version != 0 {
+ fmt.Fprintf(&b, " version=%v", p.version)
+ }
+ if p.srcConnID != nil {
+ fmt.Fprintf(&b, " src={%x}", p.srcConnID)
+ }
+ if p.dstConnID != nil {
+ fmt.Fprintf(&b, " dst={%x}", p.dstConnID)
+ }
+ for _, f := range p.frames {
+ fmt.Fprintf(&b, "\n %v", f)
+ }
+ return b.String()
+}
+
+// maxTestKeyPhases is the maximum number of 1-RTT keys we'll generate in a test.
+const maxTestKeyPhases = 3
+
+// A testConn is a Conn whose external interactions (sending and receiving packets,
+// setting timers) can be manipulated in tests.
+type testConn struct {
+ t *testing.T
+ conn *Conn
+ listener *testListener
+ now time.Time
+ timer time.Time
+ timerLastFired time.Time
+ idlec chan struct{} // only accessed on the conn's loop
+
+ // Keys are distinct from the conn's keys,
+ // because the test may know about keys before the conn does.
+ // For example, when sending a datagram with coalesced
+ // Initial and Handshake packets to a client conn,
+ // we use Handshake keys to encrypt the packet.
+ // The client only acquires those keys when it processes
+ // the Initial packet.
+ keysInitial fixedKeyPair
+ keysHandshake fixedKeyPair
+ rkeyAppData test1RTTKeys
+ wkeyAppData test1RTTKeys
+ rsecrets [numberSpaceCount]keySecret
+ wsecrets [numberSpaceCount]keySecret
+
+ // testConn uses a test hook to snoop on the conn's TLS events.
+ // CRYPTO data produced by the conn's QUICConn is placed in
+ // cryptoDataOut.
+ //
+ // The peerTLSConn is is a QUICConn representing the peer.
+ // CRYPTO data produced by the conn is written to peerTLSConn,
+ // and data produced by peerTLSConn is placed in cryptoDataIn.
+ cryptoDataOut map[tls.QUICEncryptionLevel][]byte
+ cryptoDataIn map[tls.QUICEncryptionLevel][]byte
+ peerTLSConn *tls.QUICConn
+
+ // Information about the conn's (fake) peer.
+ peerConnID []byte // source conn id of peer's packets
+ peerNextPacketNum [numberSpaceCount]packetNumber // next packet number to use
+
+ // Datagrams, packets, and frames sent by the conn,
+ // but not yet processed by the test.
+ sentDatagrams [][]byte
+ sentPackets []*testPacket
+ sentFrames []debugFrame
+ lastPacket *testPacket
+
+ recvDatagram chan *datagram
+
+ // Transport parameters sent by the conn.
+ sentTransportParameters *transportParameters
+
+ // Frame types to ignore in tests.
+ ignoreFrames map[byte]bool
+
+ // Values to set in packets sent to the conn.
+ sendKeyNumber int
+ sendKeyPhaseBit bool
+
+ asyncTestState
+}
+
+type test1RTTKeys struct {
+ hdr headerKey
+ pkt [maxTestKeyPhases]packetKey
+}
+
+type keySecret struct {
+ suite uint16
+ secret []byte
+}
+
+// newTestConn creates a Conn for testing.
+//
+// The Conn's event loop is controlled by the test,
+// allowing test code to access Conn state directly
+// by first ensuring the loop goroutine is idle.
+func newTestConn(t *testing.T, side connSide, opts ...any) *testConn {
+ t.Helper()
+ tc := &testConn{
+ t: t,
+ now: time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC),
+ peerConnID: testPeerConnID(0),
+ ignoreFrames: map[byte]bool{
+ frameTypePadding: true, // ignore PADDING by default
+ },
+ cryptoDataOut: make(map[tls.QUICEncryptionLevel][]byte),
+ cryptoDataIn: make(map[tls.QUICEncryptionLevel][]byte),
+ recvDatagram: make(chan *datagram),
+ }
+ t.Cleanup(tc.cleanup)
+
+ config := &Config{
+ TLSConfig: newTestTLSConfig(side),
+ }
+ peerProvidedParams := defaultTransportParameters()
+ peerProvidedParams.initialSrcConnID = testPeerConnID(0)
+ if side == clientSide {
+ peerProvidedParams.originalDstConnID = testLocalConnID(-1)
+ }
+ for _, o := range opts {
+ switch o := o.(type) {
+ case func(*Config):
+ o(config)
+ case func(*tls.Config):
+ o(config.TLSConfig)
+ case func(p *transportParameters):
+ o(&peerProvidedParams)
+ default:
+ t.Fatalf("unknown newTestConn option %T", o)
+ }
+ }
+
+ var initialConnID []byte
+ if side == serverSide {
+ // The initial connection ID for the server is chosen by the client.
+ initialConnID = testPeerConnID(-1)
+ }
+
+ peerQUICConfig := &tls.QUICConfig{TLSConfig: newTestTLSConfig(side.peer())}
+ if side == clientSide {
+ tc.peerTLSConn = tls.QUICServer(peerQUICConfig)
+ } else {
+ tc.peerTLSConn = tls.QUICClient(peerQUICConfig)
+ }
+ tc.peerTLSConn.SetTransportParameters(marshalTransportParameters(peerProvidedParams))
+ tc.peerTLSConn.Start(context.Background())
+
+ tc.listener = newTestListener(t, config, (*testConnHooks)(tc))
+ conn, err := tc.listener.l.newConn(
+ tc.now,
+ side,
+ initialConnID,
+ netip.MustParseAddrPort("127.0.0.1:443"))
+ if err != nil {
+ tc.t.Fatal(err)
+ }
+ tc.conn = conn
+
+ conn.keysAppData.updateAfter = maxPacketNumber // disable key updates
+ tc.keysInitial.r = conn.keysInitial.w
+ tc.keysInitial.w = conn.keysInitial.r
+
+ tc.wait()
+ return tc
+}
+
+// advance causes time to pass.
+func (tc *testConn) advance(d time.Duration) {
+ tc.t.Helper()
+ tc.advanceTo(tc.now.Add(d))
+}
+
+// advanceTo sets the current time.
+func (tc *testConn) advanceTo(now time.Time) {
+ tc.t.Helper()
+ if tc.now.After(now) {
+ tc.t.Fatalf("time moved backwards: %v -> %v", tc.now, now)
+ }
+ tc.now = now
+ if tc.timer.After(tc.now) {
+ return
+ }
+ tc.conn.sendMsg(timerEvent{})
+ tc.wait()
+}
+
+// advanceToTimer sets the current time to the time of the Conn's next timer event.
+func (tc *testConn) advanceToTimer() {
+ if tc.timer.IsZero() {
+ tc.t.Fatalf("advancing to timer, but timer is not set")
+ }
+ tc.advanceTo(tc.timer)
+}
+
+func (tc *testConn) timerDelay() time.Duration {
+ if tc.timer.IsZero() {
+ return math.MaxInt64 // infinite
+ }
+ if tc.timer.Before(tc.now) {
+ return 0
+ }
+ return tc.timer.Sub(tc.now)
+}
+
+const infiniteDuration = time.Duration(math.MaxInt64)
+
+// timeUntilEvent returns the amount of time until the next connection event.
+func (tc *testConn) timeUntilEvent() time.Duration {
+ if tc.timer.IsZero() {
+ return infiniteDuration
+ }
+ if tc.timer.Before(tc.now) {
+ return 0
+ }
+ return tc.timer.Sub(tc.now)
+}
+
+// wait blocks until the conn becomes idle.
+// The conn is idle when it is blocked waiting for a packet to arrive or a timer to expire.
+// Tests shouldn't need to call wait directly.
+// testConn methods that wake the Conn event loop will call wait for them.
+func (tc *testConn) wait() {
+ tc.t.Helper()
+ idlec := make(chan struct{})
+ fail := false
+ tc.conn.sendMsg(func(now time.Time, c *Conn) {
+ if tc.idlec != nil {
+ tc.t.Errorf("testConn.wait called concurrently")
+ fail = true
+ close(idlec)
+ } else {
+ // nextMessage will close idlec.
+ tc.idlec = idlec
+ }
+ })
+ select {
+ case <-idlec:
+ case <-tc.conn.donec:
+ // We may have async ops that can proceed now that the conn is done.
+ tc.wakeAsync()
+ }
+ if fail {
+ panic(fail)
+ }
+}
+
+func (tc *testConn) cleanup() {
+ if tc.conn == nil {
+ return
+ }
+ tc.conn.exit()
+ <-tc.conn.donec
+}
+
+func (tc *testConn) logDatagram(text string, d *testDatagram) {
+ tc.t.Helper()
+ if !*testVV {
+ return
+ }
+ pad := ""
+ if d.paddedSize > 0 {
+ pad = fmt.Sprintf(" (padded to %v)", d.paddedSize)
+ }
+ tc.t.Logf("%v datagram%v", text, pad)
+ for _, p := range d.packets {
+ var s string
+ switch p.ptype {
+ case packetType1RTT:
+ s = fmt.Sprintf(" %v pnum=%v", p.ptype, p.num)
+ default:
+ s = fmt.Sprintf(" %v pnum=%v ver=%v dst={%x} src={%x}", p.ptype, p.num, p.version, p.dstConnID, p.srcConnID)
+ }
+ if p.keyPhaseBit {
+ s += fmt.Sprintf(" KeyPhase")
+ }
+ if p.keyNumber != 0 {
+ s += fmt.Sprintf(" keynum=%v", p.keyNumber)
+ }
+ tc.t.Log(s)
+ for _, f := range p.frames {
+ tc.t.Logf(" %v", f)
+ }
+ }
+}
+
+// write sends the Conn a datagram.
+func (tc *testConn) write(d *testDatagram) {
+ tc.t.Helper()
+ var buf []byte
+ tc.logDatagram("<- conn under test receives", d)
+ for _, p := range d.packets {
+ space := spaceForPacketType(p.ptype)
+ if p.num >= tc.peerNextPacketNum[space] {
+ tc.peerNextPacketNum[space] = p.num + 1
+ }
+ pad := 0
+ if p.ptype == packetType1RTT {
+ pad = d.paddedSize
+ }
+ buf = append(buf, tc.encodeTestPacket(p, pad)...)
+ }
+ for len(buf) < d.paddedSize {
+ buf = append(buf, 0)
+ }
+ // TODO: This should use tc.listener.write.
+ tc.conn.sendMsg(&datagram{
+ b: buf,
+ })
+ tc.wait()
+}
+
+// writeFrame sends the Conn a datagram containing the given frames.
+func (tc *testConn) writeFrames(ptype packetType, frames ...debugFrame) {
+ tc.t.Helper()
+ space := spaceForPacketType(ptype)
+ dstConnID := tc.conn.connIDState.local[0].cid
+ if tc.conn.connIDState.local[0].seq == -1 && ptype != packetTypeInitial {
+ // Only use the transient connection ID in Initial packets.
+ dstConnID = tc.conn.connIDState.local[1].cid
+ }
+ d := &testDatagram{
+ packets: []*testPacket{{
+ ptype: ptype,
+ num: tc.peerNextPacketNum[space],
+ keyNumber: tc.sendKeyNumber,
+ keyPhaseBit: tc.sendKeyPhaseBit,
+ frames: frames,
+ version: quicVersion1,
+ dstConnID: dstConnID,
+ srcConnID: tc.peerConnID,
+ }},
+ }
+ if ptype == packetTypeInitial && tc.conn.side == serverSide {
+ d.paddedSize = 1200
+ }
+ tc.write(d)
+}
+
+// writeAckForAll sends the Conn a datagram containing an ack for all packets up to the
+// last one received.
+func (tc *testConn) writeAckForAll() {
+ tc.t.Helper()
+ if tc.lastPacket == nil {
+ return
+ }
+ tc.writeFrames(tc.lastPacket.ptype, debugFrameAck{
+ ranges: []i64range[packetNumber]{{0, tc.lastPacket.num + 1}},
+ })
+}
+
+// writeAckForLatest sends the Conn a datagram containing an ack for the
+// most recent packet received.
+func (tc *testConn) writeAckForLatest() {
+ tc.t.Helper()
+ if tc.lastPacket == nil {
+ return
+ }
+ tc.writeFrames(tc.lastPacket.ptype, debugFrameAck{
+ ranges: []i64range[packetNumber]{{tc.lastPacket.num, tc.lastPacket.num + 1}},
+ })
+}
+
+// ignoreFrame hides frames of the given type sent by the Conn.
+func (tc *testConn) ignoreFrame(frameType byte) {
+ tc.ignoreFrames[frameType] = true
+}
+
+// readDatagram reads the next datagram sent by the Conn.
+// It returns nil if the Conn has no more datagrams to send at this time.
+func (tc *testConn) readDatagram() *testDatagram {
+ tc.t.Helper()
+ tc.wait()
+ tc.sentPackets = nil
+ tc.sentFrames = nil
+ buf := tc.listener.read()
+ if buf == nil {
+ return nil
+ }
+ d := tc.parseTestDatagram(buf)
+ // Log the datagram before removing ignored frames.
+ // When things go wrong, it's useful to see all the frames.
+ tc.logDatagram("-> conn under test sends", d)
+ typeForFrame := func(f debugFrame) byte {
+ // This is very clunky, and points at a problem
+ // in how we specify what frames to ignore in tests.
+ //
+ // We mark frames to ignore using the frame type,
+ // but we've got a debugFrame data structure here.
+ // Perhaps we should be ignoring frames by debugFrame
+ // type instead: tc.ignoreFrame[debugFrameAck]().
+ switch f := f.(type) {
+ case debugFramePadding:
+ return frameTypePadding
+ case debugFramePing:
+ return frameTypePing
+ case debugFrameAck:
+ return frameTypeAck
+ case debugFrameResetStream:
+ return frameTypeResetStream
+ case debugFrameStopSending:
+ return frameTypeStopSending
+ case debugFrameCrypto:
+ return frameTypeCrypto
+ case debugFrameNewToken:
+ return frameTypeNewToken
+ case debugFrameStream:
+ return frameTypeStreamBase
+ case debugFrameMaxData:
+ return frameTypeMaxData
+ case debugFrameMaxStreamData:
+ return frameTypeMaxStreamData
+ case debugFrameMaxStreams:
+ if f.streamType == bidiStream {
+ return frameTypeMaxStreamsBidi
+ } else {
+ return frameTypeMaxStreamsUni
+ }
+ case debugFrameDataBlocked:
+ return frameTypeDataBlocked
+ case debugFrameStreamDataBlocked:
+ return frameTypeStreamDataBlocked
+ case debugFrameStreamsBlocked:
+ if f.streamType == bidiStream {
+ return frameTypeStreamsBlockedBidi
+ } else {
+ return frameTypeStreamsBlockedUni
+ }
+ case debugFrameNewConnectionID:
+ return frameTypeNewConnectionID
+ case debugFrameRetireConnectionID:
+ return frameTypeRetireConnectionID
+ case debugFramePathChallenge:
+ return frameTypePathChallenge
+ case debugFramePathResponse:
+ return frameTypePathResponse
+ case debugFrameConnectionCloseTransport:
+ return frameTypeConnectionCloseTransport
+ case debugFrameConnectionCloseApplication:
+ return frameTypeConnectionCloseApplication
+ case debugFrameHandshakeDone:
+ return frameTypeHandshakeDone
+ }
+ panic(fmt.Errorf("unhandled frame type %T", f))
+ }
+ for _, p := range d.packets {
+ var frames []debugFrame
+ for _, f := range p.frames {
+ if !tc.ignoreFrames[typeForFrame(f)] {
+ frames = append(frames, f)
+ }
+ }
+ p.frames = frames
+ }
+ return d
+}
+
+// readPacket reads the next packet sent by the Conn.
+// It returns nil if the Conn has no more packets to send at this time.
+func (tc *testConn) readPacket() *testPacket {
+ tc.t.Helper()
+ for len(tc.sentPackets) == 0 {
+ d := tc.readDatagram()
+ if d == nil {
+ return nil
+ }
+ tc.sentPackets = d.packets
+ }
+ p := tc.sentPackets[0]
+ tc.sentPackets = tc.sentPackets[1:]
+ tc.lastPacket = p
+ return p
+}
+
+// readFrame reads the next frame sent by the Conn.
+// It returns nil if the Conn has no more frames to send at this time.
+func (tc *testConn) readFrame() (debugFrame, packetType) {
+ tc.t.Helper()
+ for len(tc.sentFrames) == 0 {
+ p := tc.readPacket()
+ if p == nil {
+ return nil, packetTypeInvalid
+ }
+ tc.sentFrames = p.frames
+ }
+ f := tc.sentFrames[0]
+ tc.sentFrames = tc.sentFrames[1:]
+ return f, tc.lastPacket.ptype
+}
+
+// wantDatagram indicates that we expect the Conn to send a datagram.
+func (tc *testConn) wantDatagram(expectation string, want *testDatagram) {
+ tc.t.Helper()
+ got := tc.readDatagram()
+ if !reflect.DeepEqual(got, want) {
+ tc.t.Fatalf("%v:\ngot datagram: %v\nwant datagram: %v", expectation, got, want)
+ }
+}
+
+// wantPacket indicates that we expect the Conn to send a packet.
+func (tc *testConn) wantPacket(expectation string, want *testPacket) {
+ tc.t.Helper()
+ got := tc.readPacket()
+ if !reflect.DeepEqual(got, want) {
+ tc.t.Fatalf("%v:\ngot packet: %v\nwant packet: %v", expectation, got, want)
+ }
+}
+
+// wantFrame indicates that we expect the Conn to send a frame.
+func (tc *testConn) wantFrame(expectation string, wantType packetType, want debugFrame) {
+ tc.t.Helper()
+ got, gotType := tc.readFrame()
+ if got == nil {
+ tc.t.Fatalf("%v:\nconnection is idle\nwant %v frame: %v", expectation, wantType, want)
+ }
+ if gotType != wantType {
+ tc.t.Fatalf("%v:\ngot %v packet, want %v\ngot frame: %v", expectation, gotType, wantType, got)
+ }
+ if !reflect.DeepEqual(got, want) {
+ tc.t.Fatalf("%v:\ngot frame: %v\nwant frame: %v", expectation, got, want)
+ }
+}
+
+// wantFrameType indicates that we expect the Conn to send a frame,
+// although we don't care about the contents.
+func (tc *testConn) wantFrameType(expectation string, wantType packetType, want debugFrame) {
+ tc.t.Helper()
+ got, gotType := tc.readFrame()
+ if got == nil {
+ tc.t.Fatalf("%v:\nconnection is idle\nwant %v frame: %v", expectation, wantType, want)
+ }
+ if gotType != wantType {
+ tc.t.Fatalf("%v:\ngot %v packet, want %v\ngot frame: %v", expectation, gotType, wantType, got)
+ }
+ if reflect.TypeOf(got) != reflect.TypeOf(want) {
+ tc.t.Fatalf("%v:\ngot frame: %v\nwant frame of type: %v", expectation, got, want)
+ }
+}
+
+// wantIdle indicates that we expect the Conn to not send any more frames.
+func (tc *testConn) wantIdle(expectation string) {
+ tc.t.Helper()
+ switch {
+ case len(tc.sentFrames) > 0:
+ tc.t.Fatalf("expect: %v\nunexpectedly got: %v", expectation, tc.sentFrames[0])
+ case len(tc.sentPackets) > 0:
+ tc.t.Fatalf("expect: %v\nunexpectedly got: %v", expectation, tc.sentPackets[0])
+ }
+ if f, _ := tc.readFrame(); f != nil {
+ tc.t.Fatalf("expect: %v\nunexpectedly got: %v", expectation, f)
+ }
+}
+
+func (tc *testConn) encodeTestPacket(p *testPacket, pad int) []byte {
+ tc.t.Helper()
+ var w packetWriter
+ w.reset(1200)
+ var pnumMaxAcked packetNumber
+ if p.ptype != packetType1RTT {
+ w.startProtectedLongHeaderPacket(pnumMaxAcked, longPacket{
+ ptype: p.ptype,
+ version: p.version,
+ num: p.num,
+ dstConnID: p.dstConnID,
+ srcConnID: p.srcConnID,
+ })
+ } else {
+ w.start1RTTPacket(p.num, pnumMaxAcked, p.dstConnID)
+ }
+ for _, f := range p.frames {
+ f.write(&w)
+ }
+ w.appendPaddingTo(pad)
+ if p.ptype != packetType1RTT {
+ var k fixedKeys
+ switch p.ptype {
+ case packetTypeInitial:
+ k = tc.keysInitial.w
+ case packetTypeHandshake:
+ k = tc.keysHandshake.w
+ }
+ if !k.isSet() {
+ tc.t.Fatalf("sending %v packet with no write key", p.ptype)
+ }
+ w.finishProtectedLongHeaderPacket(pnumMaxAcked, k, longPacket{
+ ptype: p.ptype,
+ version: p.version,
+ num: p.num,
+ dstConnID: p.dstConnID,
+ srcConnID: p.srcConnID,
+ })
+ } else {
+ if !tc.wkeyAppData.hdr.isSet() {
+ tc.t.Fatalf("sending 1-RTT packet with no write key")
+ }
+ // Somewhat hackish: Generate a temporary updatingKeyPair that will
+ // always use our desired key phase.
+ k := &updatingKeyPair{
+ w: updatingKeys{
+ hdr: tc.wkeyAppData.hdr,
+ pkt: [2]packetKey{
+ tc.wkeyAppData.pkt[p.keyNumber],
+ tc.wkeyAppData.pkt[p.keyNumber],
+ },
+ },
+ updateAfter: maxPacketNumber,
+ }
+ if p.keyPhaseBit {
+ k.phase |= keyPhaseBit
+ }
+ w.finish1RTTPacket(p.num, pnumMaxAcked, p.dstConnID, k)
+ }
+ return w.datagram()
+}
+
+func (tc *testConn) parseTestDatagram(buf []byte) *testDatagram {
+ tc.t.Helper()
+ bufSize := len(buf)
+ d := &testDatagram{}
+ size := len(buf)
+ for len(buf) > 0 {
+ if buf[0] == 0 {
+ d.paddedSize = bufSize
+ break
+ }
+ ptype := getPacketType(buf)
+ if isLongHeader(buf[0]) {
+ var k fixedKeyPair
+ switch ptype {
+ case packetTypeInitial:
+ k = tc.keysInitial
+ case packetTypeHandshake:
+ k = tc.keysHandshake
+ }
+ if !k.canRead() {
+ tc.t.Fatalf("reading %v packet with no read key", ptype)
+ }
+ var pnumMax packetNumber // TODO: Track packet numbers.
+ p, n := parseLongHeaderPacket(buf, k.r, pnumMax)
+ if n < 0 {
+ tc.t.Fatalf("packet parse error")
+ }
+ frames, err := tc.parseTestFrames(p.payload)
+ if err != nil {
+ tc.t.Fatal(err)
+ }
+ d.packets = append(d.packets, &testPacket{
+ ptype: p.ptype,
+ version: p.version,
+ num: p.num,
+ dstConnID: p.dstConnID,
+ srcConnID: p.srcConnID,
+ frames: frames,
+ })
+ buf = buf[n:]
+ } else {
+ if !tc.rkeyAppData.hdr.isSet() {
+ tc.t.Fatalf("reading 1-RTT packet with no read key")
+ }
+ var pnumMax packetNumber // TODO: Track packet numbers.
+ pnumOff := 1 + len(tc.peerConnID)
+ // Try unprotecting the packet with the first maxTestKeyPhases keys.
+ var phase int
+ var pnum packetNumber
+ var hdr []byte
+ var pay []byte
+ var err error
+ for phase = 0; phase < maxTestKeyPhases; phase++ {
+ b := append([]byte{}, buf...)
+ hdr, pay, pnum, err = tc.rkeyAppData.hdr.unprotect(b, pnumOff, pnumMax)
+ if err != nil {
+ tc.t.Fatalf("1-RTT packet header parse error")
+ }
+ k := tc.rkeyAppData.pkt[phase]
+ pay, err = k.unprotect(hdr, pay, pnum)
+ if err == nil {
+ break
+ }
+ }
+ if err != nil {
+ tc.t.Fatalf("1-RTT packet payload parse error")
+ }
+ frames, err := tc.parseTestFrames(pay)
+ if err != nil {
+ tc.t.Fatal(err)
+ }
+ d.packets = append(d.packets, &testPacket{
+ ptype: packetType1RTT,
+ num: pnum,
+ dstConnID: hdr[1:][:len(tc.peerConnID)],
+ keyPhaseBit: hdr[0]&keyPhaseBit != 0,
+ keyNumber: phase,
+ frames: frames,
+ })
+ buf = buf[len(buf):]
+ }
+ }
+ // This is rather hackish: If the last frame in the last packet
+ // in the datagram is PADDING, then remove it and record
+ // the padded size in the testDatagram.paddedSize.
+ //
+ // This makes it easier to write a test that expects a datagram
+ // padded to 1200 bytes.
+ if len(d.packets) > 0 && len(d.packets[len(d.packets)-1].frames) > 0 {
+ p := d.packets[len(d.packets)-1]
+ f := p.frames[len(p.frames)-1]
+ if _, ok := f.(debugFramePadding); ok {
+ p.frames = p.frames[:len(p.frames)-1]
+ d.paddedSize = size
+ }
+ }
+ return d
+}
+
+func (tc *testConn) parseTestFrames(payload []byte) ([]debugFrame, error) {
+ tc.t.Helper()
+ var frames []debugFrame
+ for len(payload) > 0 {
+ f, n := parseDebugFrame(payload)
+ if n < 0 {
+ return nil, errors.New("error parsing frames")
+ }
+ frames = append(frames, f)
+ payload = payload[n:]
+ }
+ return frames, nil
+}
+
+func spaceForPacketType(ptype packetType) numberSpace {
+ switch ptype {
+ case packetTypeInitial:
+ return initialSpace
+ case packetType0RTT:
+ panic("TODO: packetType0RTT")
+ case packetTypeHandshake:
+ return handshakeSpace
+ case packetTypeRetry:
+ panic("TODO: packetTypeRetry")
+ case packetType1RTT:
+ return appDataSpace
+ }
+ panic("unknown packet type")
+}
+
+// testConnHooks implements connTestHooks.
+type testConnHooks testConn
+
+// handleTLSEvent processes TLS events generated by
+// the connection under test's tls.QUICConn.
+//
+// We maintain a second tls.QUICConn representing the peer,
+// and feed the TLS handshake data into it.
+//
+// We stash TLS handshake data from both sides in the testConn,
+// where it can be used by tests.
+//
+// We snoop packet protection keys out of the tls.QUICConns,
+// and verify that both sides of the connection are getting
+// matching keys.
+func (tc *testConnHooks) handleTLSEvent(e tls.QUICEvent) {
+ checkKey := func(typ string, secrets *[numberSpaceCount]keySecret, e tls.QUICEvent) {
+ var space numberSpace
+ switch {
+ case e.Level == tls.QUICEncryptionLevelHandshake:
+ space = handshakeSpace
+ case e.Level == tls.QUICEncryptionLevelApplication:
+ space = appDataSpace
+ default:
+ tc.t.Errorf("unexpected encryption level %v", e.Level)
+ return
+ }
+ if secrets[space].secret == nil {
+ secrets[space].suite = e.Suite
+ secrets[space].secret = append([]byte{}, e.Data...)
+ } else if secrets[space].suite != e.Suite || !bytes.Equal(secrets[space].secret, e.Data) {
+ tc.t.Errorf("%v key mismatch for level for level %v", typ, e.Level)
+ }
+ }
+ setAppDataKey := func(suite uint16, secret []byte, k *test1RTTKeys) {
+ k.hdr.init(suite, secret)
+ for i := 0; i < len(k.pkt); i++ {
+ k.pkt[i].init(suite, secret)
+ secret = updateSecret(suite, secret)
+ }
+ }
+ switch e.Kind {
+ case tls.QUICSetReadSecret:
+ checkKey("write", &tc.wsecrets, e)
+ switch e.Level {
+ case tls.QUICEncryptionLevelHandshake:
+ tc.keysHandshake.w.init(e.Suite, e.Data)
+ case tls.QUICEncryptionLevelApplication:
+ setAppDataKey(e.Suite, e.Data, &tc.wkeyAppData)
+ }
+ case tls.QUICSetWriteSecret:
+ checkKey("read", &tc.rsecrets, e)
+ switch e.Level {
+ case tls.QUICEncryptionLevelHandshake:
+ tc.keysHandshake.r.init(e.Suite, e.Data)
+ case tls.QUICEncryptionLevelApplication:
+ setAppDataKey(e.Suite, e.Data, &tc.rkeyAppData)
+ }
+ case tls.QUICWriteData:
+ tc.cryptoDataOut[e.Level] = append(tc.cryptoDataOut[e.Level], e.Data...)
+ tc.peerTLSConn.HandleData(e.Level, e.Data)
+ }
+ for {
+ e := tc.peerTLSConn.NextEvent()
+ switch e.Kind {
+ case tls.QUICNoEvent:
+ return
+ case tls.QUICSetReadSecret:
+ checkKey("write", &tc.rsecrets, e)
+ switch e.Level {
+ case tls.QUICEncryptionLevelHandshake:
+ tc.keysHandshake.r.init(e.Suite, e.Data)
+ case tls.QUICEncryptionLevelApplication:
+ setAppDataKey(e.Suite, e.Data, &tc.rkeyAppData)
+ }
+ case tls.QUICSetWriteSecret:
+ checkKey("read", &tc.wsecrets, e)
+ switch e.Level {
+ case tls.QUICEncryptionLevelHandshake:
+ tc.keysHandshake.w.init(e.Suite, e.Data)
+ case tls.QUICEncryptionLevelApplication:
+ setAppDataKey(e.Suite, e.Data, &tc.wkeyAppData)
+ }
+ case tls.QUICWriteData:
+ tc.cryptoDataIn[e.Level] = append(tc.cryptoDataIn[e.Level], e.Data...)
+ case tls.QUICTransportParameters:
+ p, err := unmarshalTransportParams(e.Data)
+ if err != nil {
+ tc.t.Logf("sent unparseable transport parameters %x %v", e.Data, err)
+ } else {
+ tc.sentTransportParameters = &p
+ }
+ }
+ }
+}
+
+// nextMessage is called by the Conn's event loop to request its next event.
+func (tc *testConnHooks) nextMessage(msgc chan any, timer time.Time) (now time.Time, m any) {
+ tc.timer = timer
+ for {
+ if !timer.IsZero() && !timer.After(tc.now) {
+ if timer.Equal(tc.timerLastFired) {
+ // If the connection timer fires at time T, the Conn should take some
+ // action to advance the timer into the future. If the Conn reschedules
+ // the timer for the same time, it isn't making progress and we have a bug.
+ tc.t.Errorf("connection timer spinning; now=%v timer=%v", tc.now, timer)
+ } else {
+ tc.timerLastFired = timer
+ return tc.now, timerEvent{}
+ }
+ }
+ select {
+ case m := <-msgc:
+ return tc.now, m
+ default:
+ }
+ if !tc.wakeAsync() {
+ break
+ }
+ }
+ // If the message queue is empty, then the conn is idle.
+ if tc.idlec != nil {
+ idlec := tc.idlec
+ tc.idlec = nil
+ close(idlec)
+ }
+ m = <-msgc
+ return tc.now, m
+}
+
+func (tc *testConnHooks) newConnID(seq int64) ([]byte, error) {
+ return testLocalConnID(seq), nil
+}
+
+func (tc *testConnHooks) timeNow() time.Time {
+ return tc.now
+}
+
+// testLocalConnID returns the connection ID with a given sequence number
+// used by a Conn under test.
+func testLocalConnID(seq int64) []byte {
+ cid := make([]byte, connIDLen)
+ copy(cid, []byte{0xc0, 0xff, 0xee})
+ cid[len(cid)-1] = byte(seq)
+ return cid
+}
+
+// testPeerConnID returns the connection ID with a given sequence number
+// used by the fake peer of a Conn under test.
+func testPeerConnID(seq int64) []byte {
+ // Use a different length than we choose for our own conn ids,
+ // to help catch any bad assumptions.
+ return []byte{0xbe, 0xee, 0xff, byte(seq)}
+}
+
+// canceledContext returns a canceled Context.
+//
+// Functions which take a context preference progress over cancelation.
+// For example, a read with a canceled context will return data if any is available.
+// Tests use canceled contexts to perform non-blocking operations.
+func canceledContext() context.Context {
+ ctx, cancel := context.WithCancel(context.Background())
+ cancel()
+ return ctx
+}
diff --git a/internal/quic/crypto_stream.go b/internal/quic/crypto_stream.go
new file mode 100644
index 0000000000..8aa8f7b828
--- /dev/null
+++ b/internal/quic/crypto_stream.go
@@ -0,0 +1,138 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build go1.21
+
+package quic
+
+// "Implementations MUST support buffering at least 4096 bytes of data
+// received in out-of-order CRYPTO frames."
+// https://www.rfc-editor.org/rfc/rfc9000.html#section-7.5-2
+//
+// 4096 is too small for real-world cases, however, so we allow more.
+const cryptoBufferSize = 1 << 20
+
+// A cryptoStream is the stream of data passed in CRYPTO frames.
+// There is one cryptoStream per packet number space.
+type cryptoStream struct {
+ // CRYPTO data received from the peer.
+ in pipe
+ inset rangeset[int64] // bytes received
+
+ // CRYPTO data queued for transmission to the peer.
+ out pipe
+ outunsent rangeset[int64] // bytes in need of sending
+ outacked rangeset[int64] // bytes acked by peer
+}
+
+// handleCrypto processes data received in a CRYPTO frame.
+func (s *cryptoStream) handleCrypto(off int64, b []byte, f func([]byte) error) error {
+ end := off + int64(len(b))
+ if end-s.inset.min() > cryptoBufferSize {
+ return localTransportError(errCryptoBufferExceeded)
+ }
+ s.inset.add(off, end)
+ if off == s.in.start {
+ // Fast path: This is the next chunk of data in the stream,
+ // so just handle it immediately.
+ if err := f(b); err != nil {
+ return err
+ }
+ s.in.discardBefore(end)
+ } else {
+ // This is either data we've already processed,
+ // data we can't process yet, or a mix of both.
+ s.in.writeAt(b, off)
+ }
+ // s.in.start is the next byte in sequence.
+ // If it's in s.inset, we have bytes to provide.
+ // If it isn't, we don't--we're either out of data,
+ // or only have data that comes after the next byte.
+ if !s.inset.contains(s.in.start) {
+ return nil
+ }
+ // size is the size of the first contiguous chunk of bytes
+ // that have not been processed yet.
+ size := int(s.inset[0].end - s.in.start)
+ if size <= 0 {
+ return nil
+ }
+ err := s.in.read(s.in.start, size, f)
+ s.in.discardBefore(s.inset[0].end)
+ return err
+}
+
+// write queues data for sending to the peer.
+// It does not block or limit the amount of buffered data.
+// QUIC connections don't communicate the amount of CRYPTO data they are willing to buffer,
+// so we send what we have and the peer can close the connection if it is too much.
+func (s *cryptoStream) write(b []byte) {
+ start := s.out.end
+ s.out.writeAt(b, start)
+ s.outunsent.add(start, s.out.end)
+}
+
+// ackOrLoss reports that an CRYPTO frame sent by us has been acknowledged by the peer, or lost.
+func (s *cryptoStream) ackOrLoss(start, end int64, fate packetFate) {
+ switch fate {
+ case packetAcked:
+ s.outacked.add(start, end)
+ s.outunsent.sub(start, end)
+ // If this ack is for data at the start of the send buffer, we can now discard it.
+ if s.outacked.contains(s.out.start) {
+ s.out.discardBefore(s.outacked[0].end)
+ }
+ case packetLost:
+ // Mark everything lost, but not previously acked, as needing retransmission.
+ // We do this by adding all the lost bytes to outunsent, and then
+ // removing everything already acked.
+ s.outunsent.add(start, end)
+ for _, a := range s.outacked {
+ s.outunsent.sub(a.start, a.end)
+ }
+ }
+}
+
+// dataToSend reports what data should be sent in CRYPTO frames to the peer.
+// It calls f with each range of data to send.
+// f uses sendData to get the bytes to send, and returns the number of bytes sent.
+// dataToSend calls f until no data is left, or f returns 0.
+//
+// This function is unusually indirect (why not just return a []byte,
+// or implement io.Reader?).
+//
+// Returning a []byte to the caller either requires that we store the
+// data to send contiguously (which we don't), allocate a temporary buffer
+// and copy into it (inefficient), or return less data than we have available
+// (requires complexity to avoid unnecessarily breaking data across frames).
+//
+// Accepting a []byte from the caller (io.Reader) makes packet construction
+// difficult. Since CRYPTO data is encoded with a varint length prefix, the
+// location of the data depends on the length of the data. (We could hardcode
+// a 2-byte length, of course.)
+//
+// Instead, we tell the caller how much data is, the caller figures out where
+// to put it (and possibly decides that it doesn't have space for this data
+// in the packet after all), and the caller then makes a separate call to
+// copy the data it wants into position.
+func (s *cryptoStream) dataToSend(pto bool, f func(off, size int64) (sent int64)) {
+ for {
+ off, size := dataToSend(s.out.start, s.out.end, s.outunsent, s.outacked, pto)
+ if size == 0 {
+ return
+ }
+ n := f(off, size)
+ if n == 0 || pto {
+ return
+ }
+ }
+}
+
+// sendData fills b with data to send to the peer, starting at off,
+// and marks the data as sent. The caller must have already ascertained
+// that there is data to send in this region using dataToSend.
+func (s *cryptoStream) sendData(off int64, b []byte) {
+ s.out.copy(off, b)
+ s.outunsent.sub(off, off+int64(len(b)))
+}
diff --git a/internal/quic/crypto_stream_test.go b/internal/quic/crypto_stream_test.go
new file mode 100644
index 0000000000..a6c1e1b521
--- /dev/null
+++ b/internal/quic/crypto_stream_test.go
@@ -0,0 +1,265 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build go1.21
+
+package quic
+
+import (
+ "crypto/rand"
+ "reflect"
+ "testing"
+)
+
+func TestCryptoStreamReceive(t *testing.T) {
+ data := make([]byte, 1<<20)
+ rand.Read(data) // doesn't need to be crypto/rand, but non-deprecated and harmless
+ type frame struct {
+ start int64
+ end int64
+ want int
+ }
+ for _, test := range []struct {
+ name string
+ frames []frame
+ }{{
+ name: "linear",
+ frames: []frame{{
+ start: 0,
+ end: 1000,
+ want: 1000,
+ }, {
+ start: 1000,
+ end: 2000,
+ want: 2000,
+ }, {
+ // larger than any realistic packet can hold
+ start: 2000,
+ end: 1 << 20,
+ want: 1 << 20,
+ }},
+ }, {
+ name: "out of order",
+ frames: []frame{{
+ start: 1000,
+ end: 2000,
+ }, {
+ start: 2000,
+ end: 3000,
+ }, {
+ start: 0,
+ end: 1000,
+ want: 3000,
+ }},
+ }, {
+ name: "resent",
+ frames: []frame{{
+ start: 0,
+ end: 1000,
+ want: 1000,
+ }, {
+ start: 0,
+ end: 1000,
+ want: 1000,
+ }, {
+ start: 1000,
+ end: 2000,
+ want: 2000,
+ }, {
+ start: 0,
+ end: 1000,
+ want: 2000,
+ }, {
+ start: 1000,
+ end: 2000,
+ want: 2000,
+ }},
+ }, {
+ name: "overlapping",
+ frames: []frame{{
+ start: 0,
+ end: 1000,
+ want: 1000,
+ }, {
+ start: 3000,
+ end: 4000,
+ want: 1000,
+ }, {
+ start: 2000,
+ end: 3000,
+ want: 1000,
+ }, {
+ start: 1000,
+ end: 3000,
+ want: 4000,
+ }},
+ }} {
+ t.Run(test.name, func(t *testing.T) {
+ var s cryptoStream
+ var got []byte
+ for _, f := range test.frames {
+ t.Logf("receive [%v,%v)", f.start, f.end)
+ s.handleCrypto(
+ f.start,
+ data[f.start:f.end],
+ func(b []byte) error {
+ t.Logf("got new bytes [%v,%v)", len(got), len(got)+len(b))
+ got = append(got, b...)
+ return nil
+ },
+ )
+ if len(got) != f.want {
+ t.Fatalf("have bytes [0,%v), want [0,%v)", len(got), f.want)
+ }
+ for i := range got {
+ if got[i] != data[i] {
+ t.Fatalf("byte %v of received data = %v, want %v", i, got[i], data[i])
+ }
+ }
+ }
+ })
+ }
+}
+
+func TestCryptoStreamSends(t *testing.T) {
+ data := make([]byte, 1<<20)
+ rand.Read(data) // doesn't need to be crypto/rand, but non-deprecated and harmless
+ type (
+ sendOp i64range[int64]
+ ackOp i64range[int64]
+ lossOp i64range[int64]
+ )
+ for _, test := range []struct {
+ name string
+ size int64
+ ops []any
+ wantSend []i64range[int64]
+ wantPTOSend []i64range[int64]
+ }{{
+ name: "writes with data remaining",
+ size: 4000,
+ ops: []any{
+ sendOp{0, 1000},
+ sendOp{1000, 2000},
+ sendOp{2000, 3000},
+ },
+ wantSend: []i64range[int64]{
+ {3000, 4000},
+ },
+ wantPTOSend: []i64range[int64]{
+ {0, 4000},
+ },
+ }, {
+ name: "lost data is resent",
+ size: 4000,
+ ops: []any{
+ sendOp{0, 1000},
+ sendOp{1000, 2000},
+ sendOp{2000, 3000},
+ sendOp{3000, 4000},
+ lossOp{1000, 2000},
+ lossOp{3000, 4000},
+ },
+ wantSend: []i64range[int64]{
+ {1000, 2000},
+ {3000, 4000},
+ },
+ wantPTOSend: []i64range[int64]{
+ {0, 4000},
+ },
+ }, {
+ name: "acked data at start of range",
+ size: 4000,
+ ops: []any{
+ sendOp{0, 4000},
+ ackOp{0, 1000},
+ ackOp{1000, 2000},
+ ackOp{2000, 3000},
+ },
+ wantSend: nil,
+ wantPTOSend: []i64range[int64]{
+ {3000, 4000},
+ },
+ }, {
+ name: "acked data is not resent on pto",
+ size: 4000,
+ ops: []any{
+ sendOp{0, 4000},
+ ackOp{1000, 2000},
+ },
+ wantSend: nil,
+ wantPTOSend: []i64range[int64]{
+ {0, 1000},
+ },
+ }, {
+ // This is an unusual, but possible scenario:
+ // Data is sent, resent, one of the two sends is acked, and the other is lost.
+ name: "acked and then lost data is not resent",
+ size: 4000,
+ ops: []any{
+ sendOp{0, 4000},
+ sendOp{1000, 2000}, // resent, no-op
+ ackOp{1000, 2000},
+ lossOp{1000, 2000},
+ },
+ wantSend: nil,
+ wantPTOSend: []i64range[int64]{
+ {0, 1000},
+ },
+ }, {
+ // The opposite of the above scenario: data is marked lost, and then acked
+ // before being resent.
+ name: "lost and then acked data is not resent",
+ size: 4000,
+ ops: []any{
+ sendOp{0, 4000},
+ sendOp{1000, 2000}, // resent, no-op
+ lossOp{1000, 2000},
+ ackOp{1000, 2000},
+ },
+ wantSend: nil,
+ wantPTOSend: []i64range[int64]{
+ {0, 1000},
+ },
+ }} {
+ t.Run(test.name, func(t *testing.T) {
+ var s cryptoStream
+ s.write(data[:test.size])
+ for _, op := range test.ops {
+ switch op := op.(type) {
+ case sendOp:
+ t.Logf("send [%v,%v)", op.start, op.end)
+ b := make([]byte, op.end-op.start)
+ s.sendData(op.start, b)
+ case ackOp:
+ t.Logf("ack [%v,%v)", op.start, op.end)
+ s.ackOrLoss(op.start, op.end, packetAcked)
+ case lossOp:
+ t.Logf("loss [%v,%v)", op.start, op.end)
+ s.ackOrLoss(op.start, op.end, packetLost)
+ default:
+ t.Fatalf("unhandled type %T", op)
+ }
+ }
+ var gotSend []i64range[int64]
+ s.dataToSend(true, func(off, size int64) (wrote int64) {
+ gotSend = append(gotSend, i64range[int64]{off, off + size})
+ return 0
+ })
+ if !reflect.DeepEqual(gotSend, test.wantPTOSend) {
+ t.Fatalf("got data to send on PTO: %v, want %v", gotSend, test.wantPTOSend)
+ }
+ gotSend = nil
+ s.dataToSend(false, func(off, size int64) (wrote int64) {
+ gotSend = append(gotSend, i64range[int64]{off, off + size})
+ b := make([]byte, size)
+ s.sendData(off, b)
+ return int64(len(b))
+ })
+ if !reflect.DeepEqual(gotSend, test.wantSend) {
+ t.Fatalf("got data to send: %v, want %v", gotSend, test.wantSend)
+ }
+ })
+ }
+}
diff --git a/internal/quic/dgram.go b/internal/quic/dgram.go
new file mode 100644
index 0000000000..79e6650fa4
--- /dev/null
+++ b/internal/quic/dgram.go
@@ -0,0 +1,38 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build go1.21
+
+package quic
+
+import (
+ "net/netip"
+ "sync"
+)
+
+type datagram struct {
+ b []byte
+ addr netip.AddrPort
+}
+
+var datagramPool = sync.Pool{
+ New: func() any {
+ return &datagram{
+ b: make([]byte, maxUDPPayloadSize),
+ }
+ },
+}
+
+func newDatagram() *datagram {
+ m := datagramPool.Get().(*datagram)
+ m.b = m.b[:cap(m.b)]
+ return m
+}
+
+func (m *datagram) recycle() {
+ if cap(m.b) != maxUDPPayloadSize {
+ return
+ }
+ datagramPool.Put(m)
+}
diff --git a/internal/quic/errors.go b/internal/quic/errors.go
index a9ebbe4b74..8e01bb7cb7 100644
--- a/internal/quic/errors.go
+++ b/internal/quic/errors.go
@@ -10,7 +10,7 @@ import (
"fmt"
)
-// A transportError is an transport error code from RFC 9000 Section 20.1.
+// A transportError is a transport error code from RFC 9000 Section 20.1.
//
// The transportError type doesn't implement the error interface to ensure we always
// distinguish between errors sent to and received from the peer.
@@ -99,6 +99,14 @@ func (e peerTransportError) Error() string {
return fmt.Sprintf("peer closed connection: %v: %q", e.code, e.reason)
}
+// A StreamErrorCode is an application protocol error code (RFC 9000, Section 20.2)
+// indicating whay a stream is being closed.
+type StreamErrorCode uint64
+
+func (e StreamErrorCode) Error() string {
+ return fmt.Sprintf("stream error code %v", uint64(e))
+}
+
// An ApplicationError is an application protocol error code (RFC 9000, Section 20.2).
// Application protocol errors may be sent when terminating a stream or connection.
type ApplicationError struct {
@@ -106,7 +114,13 @@ type ApplicationError struct {
Reason string
}
-func (e ApplicationError) Error() string {
+func (e *ApplicationError) Error() string {
// TODO: Include the Reason string here, but sanitize it first.
return fmt.Sprintf("AppError %v", e.Code)
}
+
+// Is reports a match if err is an *ApplicationError with a matching Code.
+func (e *ApplicationError) Is(err error) bool {
+ e2, ok := err.(*ApplicationError)
+ return ok && e2.Code == e.Code
+}
diff --git a/internal/quic/frame_debug.go b/internal/quic/frame_debug.go
index 945bb9d1f7..7a5aee57b1 100644
--- a/internal/quic/frame_debug.go
+++ b/internal/quic/frame_debug.go
@@ -120,7 +120,7 @@ type debugFrameAck struct {
func parseDebugFrameAck(b []byte) (f debugFrameAck, n int) {
f.ranges = nil
- _, f.ackDelay, n = consumeAckFrame(b, func(start, end packetNumber) {
+ _, f.ackDelay, n = consumeAckFrame(b, func(_ int, start, end packetNumber) {
f.ranges = append(f.ranges, i64range[packetNumber]{
start: start,
end: end,
@@ -386,10 +386,7 @@ func (f debugFrameNewConnectionID) write(w *packetWriter) bool {
// debugFrameRetireConnectionID is a NEW_CONNECTION_ID frame.
type debugFrameRetireConnectionID struct {
- seq uint64
- retirePriorTo uint64
- connID []byte
- token [16]byte
+ seq int64
}
func parseDebugFrameRetireConnectionID(b []byte) (f debugFrameRetireConnectionID, n int) {
diff --git a/internal/quic/gate.go b/internal/quic/gate.go
new file mode 100644
index 0000000000..a2fb537115
--- /dev/null
+++ b/internal/quic/gate.go
@@ -0,0 +1,91 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build go1.21
+
+package quic
+
+import "context"
+
+// An gate is a monitor (mutex + condition variable) with one bit of state.
+//
+// The condition may be either set or unset.
+// Lock operations may be unconditional, or wait for the condition to be set.
+// Unlock operations record the new state of the condition.
+type gate struct {
+ // When unlocked, exactly one of set or unset contains a value.
+ // When locked, neither chan contains a value.
+ set chan struct{}
+ unset chan struct{}
+}
+
+// newGate returns a new, unlocked gate with the condition unset.
+func newGate() gate {
+ g := newLockedGate()
+ g.unlock(false)
+ return g
+}
+
+// newLocked gate returns a new, locked gate.
+func newLockedGate() gate {
+ return gate{
+ set: make(chan struct{}, 1),
+ unset: make(chan struct{}, 1),
+ }
+}
+
+// lock acquires the gate unconditionally.
+// It reports whether the condition is set.
+func (g *gate) lock() (set bool) {
+ select {
+ case <-g.set:
+ return true
+ case <-g.unset:
+ return false
+ }
+}
+
+// waitAndLock waits until the condition is set before acquiring the gate.
+// If the context expires, waitAndLock returns an error and does not acquire the gate.
+func (g *gate) waitAndLock(ctx context.Context, testHooks connTestHooks) error {
+ if testHooks != nil {
+ return testHooks.waitUntil(ctx, g.lockIfSet)
+ }
+ select {
+ case <-g.set:
+ return nil
+ default:
+ }
+ select {
+ case <-g.set:
+ return nil
+ case <-ctx.Done():
+ return ctx.Err()
+ }
+}
+
+// lockIfSet acquires the gate if and only if the condition is set.
+func (g *gate) lockIfSet() (acquired bool) {
+ select {
+ case <-g.set:
+ return true
+ default:
+ return false
+ }
+}
+
+// unlock sets the condition and releases the gate.
+func (g *gate) unlock(set bool) {
+ if set {
+ g.set <- struct{}{}
+ } else {
+ g.unset <- struct{}{}
+ }
+}
+
+// unlock sets the condition to the result of f and releases the gate.
+// Useful in defers.
+func (g *gate) unlockFunc(f func() bool) {
+ g.unlock(f())
+}
diff --git a/internal/quic/gate_test.go b/internal/quic/gate_test.go
new file mode 100644
index 0000000000..9e84a84bd6
--- /dev/null
+++ b/internal/quic/gate_test.go
@@ -0,0 +1,95 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build go1.21
+
+package quic
+
+import (
+ "context"
+ "testing"
+ "time"
+)
+
+func TestGateLockAndUnlock(t *testing.T) {
+ g := newGate()
+ if set := g.lock(); set {
+ t.Errorf("g.lock() of never-locked gate: true, want false")
+ }
+ unlockedc := make(chan struct{})
+ donec := make(chan struct{})
+ go func() {
+ defer close(donec)
+ set := g.lock()
+ select {
+ case <-unlockedc:
+ default:
+ t.Errorf("g.lock() succeeded while gate was held")
+ }
+ if !set {
+ t.Errorf("g.lock() of set gate: false, want true")
+ }
+ g.unlock(false)
+ }()
+ time.Sleep(1 * time.Millisecond)
+ close(unlockedc)
+ g.unlock(true)
+ <-donec
+ if set := g.lock(); set {
+ t.Errorf("g.lock() of unset gate: true, want false")
+ }
+}
+
+func TestGateWaitAndLockContext(t *testing.T) {
+ g := newGate()
+ // waitAndLock is canceled
+ ctx, cancel := context.WithCancel(context.Background())
+ go func() {
+ time.Sleep(1 * time.Millisecond)
+ cancel()
+ }()
+ if err := g.waitAndLock(ctx, nil); err != context.Canceled {
+ t.Errorf("g.waitAndLock() = %v, want context.Canceled", err)
+ }
+ // waitAndLock succeeds
+ set := false
+ go func() {
+ time.Sleep(1 * time.Millisecond)
+ g.lock()
+ set = true
+ g.unlock(true)
+ }()
+ if err := g.waitAndLock(context.Background(), nil); err != nil {
+ t.Errorf("g.waitAndLock() = %v, want nil", err)
+ }
+ if !set {
+ t.Errorf("g.waitAndLock() returned before gate was set")
+ }
+ g.unlock(true)
+ // waitAndLock succeeds when the gate is set and the context is canceled
+ if err := g.waitAndLock(ctx, nil); err != nil {
+ t.Errorf("g.waitAndLock() = %v, want nil", err)
+ }
+}
+
+func TestGateLockIfSet(t *testing.T) {
+ g := newGate()
+ if locked := g.lockIfSet(); locked {
+ t.Errorf("g.lockIfSet() of unset gate = %v, want false", locked)
+ }
+ g.lock()
+ g.unlock(true)
+ if locked := g.lockIfSet(); !locked {
+ t.Errorf("g.lockIfSet() of set gate = %v, want true", locked)
+ }
+}
+
+func TestGateUnlockFunc(t *testing.T) {
+ g := newGate()
+ go func() {
+ g.lock()
+ defer g.unlockFunc(func() bool { return true })
+ }()
+ g.waitAndLock(context.Background(), nil)
+}
diff --git a/internal/quic/gotraceback_test.go b/internal/quic/gotraceback_test.go
new file mode 100644
index 0000000000..c22702faa4
--- /dev/null
+++ b/internal/quic/gotraceback_test.go
@@ -0,0 +1,26 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build go1.21 && unix
+
+package quic
+
+import (
+ "os"
+ "os/signal"
+ "runtime/debug"
+ "syscall"
+)
+
+// When killed with SIGQUIT (C-\), print stacks with GOTRACEBACK=all rather than system,
+// to reduce irrelevant noise when debugging hung tests.
+func init() {
+ ch := make(chan os.Signal, 1)
+ signal.Notify(ch, syscall.SIGQUIT)
+ go func() {
+ <-ch
+ debug.SetTraceback("all")
+ panic("SIGQUIT")
+ }()
+}
diff --git a/internal/quic/key_update_test.go b/internal/quic/key_update_test.go
new file mode 100644
index 0000000000..4a4d677713
--- /dev/null
+++ b/internal/quic/key_update_test.go
@@ -0,0 +1,234 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build go1.21
+
+package quic
+
+import (
+ "testing"
+)
+
+func TestKeyUpdatePeerUpdates(t *testing.T) {
+ tc := newTestConn(t, serverSide)
+ tc.handshake()
+ tc.ignoreFrames = nil // ignore nothing
+
+ // Peer initiates a key update.
+ tc.sendKeyNumber = 1
+ tc.sendKeyPhaseBit = true
+ tc.writeFrames(packetType1RTT, debugFramePing{})
+
+ // We update to the new key.
+ tc.advanceToTimer()
+ tc.wantFrameType("conn ACKs last packet",
+ packetType1RTT, debugFrameAck{})
+ tc.wantFrame("first packet after a key update is always ack-eliciting",
+ packetType1RTT, debugFramePing{})
+ if got, want := tc.lastPacket.keyNumber, 1; got != want {
+ t.Errorf("after key rotation, conn sent packet with key %v, want %v", got, want)
+ }
+ if !tc.lastPacket.keyPhaseBit {
+ t.Errorf("after key rotation, conn failed to change Key Phase bit")
+ }
+ tc.wantIdle("conn has nothing to send")
+
+ // Peer's ACK of a packet we sent in the new phase completes the update.
+ tc.writeAckForAll()
+
+ // Peer initiates a second key update.
+ tc.sendKeyNumber = 2
+ tc.sendKeyPhaseBit = false
+ tc.writeFrames(packetType1RTT, debugFramePing{})
+
+ // We update to the new key.
+ tc.advanceToTimer()
+ tc.wantFrameType("conn ACKs last packet",
+ packetType1RTT, debugFrameAck{})
+ tc.wantFrame("first packet after a key update is always ack-eliciting",
+ packetType1RTT, debugFramePing{})
+ if got, want := tc.lastPacket.keyNumber, 2; got != want {
+ t.Errorf("after key rotation, conn sent packet with key %v, want %v", got, want)
+ }
+ if tc.lastPacket.keyPhaseBit {
+ t.Errorf("after second key rotation, conn failed to change Key Phase bit")
+ }
+ tc.wantIdle("conn has nothing to send")
+}
+
+func TestKeyUpdateAcceptPreviousPhaseKeys(t *testing.T) {
+ // "An endpoint SHOULD retain old keys for some time after
+ // unprotecting a packet sent using the new keys."
+ // https://www.rfc-editor.org/rfc/rfc9001#section-6.1-8
+ tc := newTestConn(t, serverSide)
+ tc.handshake()
+ tc.ignoreFrames = nil // ignore nothing
+
+ // Peer initiates a key update, skipping one packet number.
+ pnum0 := tc.peerNextPacketNum[appDataSpace]
+ tc.peerNextPacketNum[appDataSpace]++
+ tc.sendKeyNumber = 1
+ tc.sendKeyPhaseBit = true
+ tc.writeFrames(packetType1RTT, debugFramePing{})
+
+ // We update to the new key.
+ // This ACK is not delayed, because we've skipped a packet number.
+ tc.wantFrame("conn ACKs last packet",
+ packetType1RTT, debugFrameAck{
+ ranges: []i64range[packetNumber]{
+ {0, pnum0},
+ {pnum0 + 1, pnum0 + 2},
+ },
+ })
+ tc.wantFrame("first packet after a key update is always ack-eliciting",
+ packetType1RTT, debugFramePing{})
+ if got, want := tc.lastPacket.keyNumber, 1; got != want {
+ t.Errorf("after key rotation, conn sent packet with key %v, want %v", got, want)
+ }
+ if !tc.lastPacket.keyPhaseBit {
+ t.Errorf("after key rotation, conn failed to change Key Phase bit")
+ }
+ tc.wantIdle("conn has nothing to send")
+
+ // We receive the previously-skipped packet in the earlier key phase.
+ tc.peerNextPacketNum[appDataSpace] = pnum0
+ tc.sendKeyNumber = 0
+ tc.sendKeyPhaseBit = false
+ tc.writeFrames(packetType1RTT, debugFramePing{})
+
+ // We ack the reordered packet immediately, still in the new key phase.
+ tc.wantFrame("conn ACKs reordered packet",
+ packetType1RTT, debugFrameAck{
+ ranges: []i64range[packetNumber]{
+ {0, pnum0 + 2},
+ },
+ })
+ tc.wantIdle("packet is not ack-eliciting")
+ if got, want := tc.lastPacket.keyNumber, 1; got != want {
+ t.Errorf("after key rotation, conn sent packet with key %v, want %v", got, want)
+ }
+ if !tc.lastPacket.keyPhaseBit {
+ t.Errorf("after key rotation, conn failed to change Key Phase bit")
+ }
+}
+
+func TestKeyUpdateRejectPacketFromPriorPhase(t *testing.T) {
+ // "Packets with higher packet numbers MUST be protected with either
+ // the same or newer packet protection keys than packets with lower packet numbers."
+ // https://www.rfc-editor.org/rfc/rfc9001#section-6.4-2
+ tc := newTestConn(t, serverSide)
+ tc.handshake()
+ tc.ignoreFrames = nil // ignore nothing
+
+ // Peer initiates a key update.
+ tc.sendKeyNumber = 1
+ tc.sendKeyPhaseBit = true
+ tc.writeFrames(packetType1RTT, debugFramePing{})
+
+ // We update to the new key.
+ tc.advanceToTimer()
+ tc.wantFrameType("conn ACKs last packet",
+ packetType1RTT, debugFrameAck{})
+ tc.wantFrame("first packet after a key update is always ack-eliciting",
+ packetType1RTT, debugFramePing{})
+ if got, want := tc.lastPacket.keyNumber, 1; got != want {
+ t.Errorf("after key rotation, conn sent packet with key %v, want %v", got, want)
+ }
+ if !tc.lastPacket.keyPhaseBit {
+ t.Errorf("after key rotation, conn failed to change Key Phase bit")
+ }
+ tc.wantIdle("conn has nothing to send")
+
+ // Peer sends an ack-eliciting packet using the prior phase keys.
+ // We fail to unprotect the packet and ignore it.
+ skipped := tc.peerNextPacketNum[appDataSpace]
+ tc.sendKeyNumber = 0
+ tc.sendKeyPhaseBit = false
+ tc.writeFrames(packetType1RTT, debugFramePing{})
+
+ // Peer sends an ack-eliciting packet using the current phase keys.
+ tc.sendKeyNumber = 1
+ tc.sendKeyPhaseBit = true
+ tc.writeFrames(packetType1RTT, debugFramePing{})
+
+ // We ack the peer's packets, not including the one sent with the wrong keys.
+ tc.wantFrame("conn ACKs packets, not including packet sent with wrong keys",
+ packetType1RTT, debugFrameAck{
+ ranges: []i64range[packetNumber]{
+ {0, skipped},
+ {skipped + 1, skipped + 2},
+ },
+ })
+}
+
+func TestKeyUpdateLocallyInitiated(t *testing.T) {
+ const updateAfter = 4 // initiate key update after 1-RTT packet 4
+ tc := newTestConn(t, serverSide)
+ tc.conn.keysAppData.updateAfter = updateAfter
+ tc.handshake()
+
+ for {
+ tc.writeFrames(packetType1RTT, debugFramePing{})
+ tc.advanceToTimer()
+ tc.wantFrameType("conn ACKs last packet",
+ packetType1RTT, debugFrameAck{})
+ if tc.lastPacket.num > updateAfter {
+ break
+ }
+ if got, want := tc.lastPacket.keyNumber, 0; got != want {
+ t.Errorf("before key update, conn sent packet with key %v, want %v", got, want)
+ }
+ if tc.lastPacket.keyPhaseBit {
+ t.Errorf("before key update, keyPhaseBit is set, want unset")
+ }
+ }
+ if got, want := tc.lastPacket.keyNumber, 1; got != want {
+ t.Errorf("after key update, conn sent packet with key %v, want %v", got, want)
+ }
+ if !tc.lastPacket.keyPhaseBit {
+ t.Errorf("after key update, keyPhaseBit is unset, want set")
+ }
+ tc.wantFrame("first packet after a key update is always ack-eliciting",
+ packetType1RTT, debugFramePing{})
+ tc.wantIdle("no more frames")
+
+ // Peer sends another packet using the prior phase keys.
+ tc.writeFrames(packetType1RTT, debugFramePing{})
+ tc.advanceToTimer()
+ tc.wantFrameType("conn ACKs packet in prior phase",
+ packetType1RTT, debugFrameAck{})
+ tc.wantIdle("packet is not ack-eliciting")
+ if got, want := tc.lastPacket.keyNumber, 1; got != want {
+ t.Errorf("after key update, conn sent packet with key %v, want %v", got, want)
+ }
+
+ // Peer updates to the next phase.
+ tc.sendKeyNumber = 1
+ tc.sendKeyPhaseBit = true
+ tc.writeAckForAll()
+ tc.writeFrames(packetType1RTT, debugFramePing{})
+ tc.advanceToTimer()
+ tc.wantFrameType("conn ACKs packet in current phase",
+ packetType1RTT, debugFrameAck{})
+ tc.wantIdle("packet is not ack-eliciting")
+ if got, want := tc.lastPacket.keyNumber, 1; got != want {
+ t.Errorf("after key update, conn sent packet with key %v, want %v", got, want)
+ }
+
+ // Peer initiates its own update.
+ tc.sendKeyNumber = 2
+ tc.sendKeyPhaseBit = false
+ tc.writeFrames(packetType1RTT, debugFramePing{})
+ tc.advanceToTimer()
+ tc.wantFrameType("conn ACKs packet in current phase",
+ packetType1RTT, debugFrameAck{})
+ tc.wantFrame("first packet after a key update is always ack-eliciting",
+ packetType1RTT, debugFramePing{})
+ if got, want := tc.lastPacket.keyNumber, 2; got != want {
+ t.Errorf("after peer key update, conn sent packet with key %v, want %v", got, want)
+ }
+ if tc.lastPacket.keyPhaseBit {
+ t.Errorf("after peer key update, keyPhaseBit is unset, want set")
+ }
+}
diff --git a/internal/quic/listener.go b/internal/quic/listener.go
new file mode 100644
index 0000000000..96b1e45934
--- /dev/null
+++ b/internal/quic/listener.go
@@ -0,0 +1,322 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build go1.21
+
+package quic
+
+import (
+ "context"
+ "errors"
+ "net"
+ "net/netip"
+ "sync"
+ "sync/atomic"
+ "time"
+)
+
+// A Listener listens for QUIC traffic on a network address.
+// It can accept inbound connections or create outbound ones.
+//
+// Multiple goroutines may invoke methods on a Listener simultaneously.
+type Listener struct {
+ config *Config
+ udpConn udpConn
+ testHooks connTestHooks
+
+ acceptQueue queue[*Conn] // new inbound connections
+
+ connsMu sync.Mutex
+ conns map[*Conn]struct{}
+ closing bool // set when Close is called
+ closec chan struct{} // closed when the listen loop exits
+
+ // The datagram receive loop keeps a mapping of connection IDs to conns.
+ // When a conn's connection IDs change, we add it to connIDUpdates and set
+ // connIDUpdateNeeded, and the receive loop updates its map.
+ connIDUpdateMu sync.Mutex
+ connIDUpdateNeeded atomic.Bool
+ connIDUpdates []connIDUpdate
+}
+
+// A udpConn is a UDP connection.
+// It is implemented by net.UDPConn.
+type udpConn interface {
+ Close() error
+ LocalAddr() net.Addr
+ ReadMsgUDPAddrPort(b, control []byte) (n, controln, flags int, _ netip.AddrPort, _ error)
+ WriteToUDPAddrPort(b []byte, addr netip.AddrPort) (int, error)
+}
+
+type connIDUpdate struct {
+ conn *Conn
+ retired bool
+ cid []byte
+}
+
+// Listen listens on a local network address.
+// The configuration config must be non-nil.
+func Listen(network, address string, config *Config) (*Listener, error) {
+ if config.TLSConfig == nil {
+ return nil, errors.New("TLSConfig is not set")
+ }
+ a, err := net.ResolveUDPAddr(network, address)
+ if err != nil {
+ return nil, err
+ }
+ udpConn, err := net.ListenUDP(network, a)
+ if err != nil {
+ return nil, err
+ }
+ return newListener(udpConn, config, nil), nil
+}
+
+func newListener(udpConn udpConn, config *Config, hooks connTestHooks) *Listener {
+ l := &Listener{
+ config: config,
+ udpConn: udpConn,
+ testHooks: hooks,
+ conns: make(map[*Conn]struct{}),
+ acceptQueue: newQueue[*Conn](),
+ closec: make(chan struct{}),
+ }
+ go l.listen()
+ return l
+}
+
+// LocalAddr returns the local network address.
+func (l *Listener) LocalAddr() netip.AddrPort {
+ a, _ := l.udpConn.LocalAddr().(*net.UDPAddr)
+ return a.AddrPort()
+}
+
+// Close closes the listener.
+// Any blocked operations on the Listener or associated Conns and Stream will be unblocked
+// and return errors.
+//
+// Close aborts every open connection.
+// Data in stream read and write buffers is discarded.
+// It waits for the peers of any open connection to acknowledge the connection has been closed.
+func (l *Listener) Close(ctx context.Context) error {
+ l.acceptQueue.close(errors.New("listener closed"))
+ l.connsMu.Lock()
+ if !l.closing {
+ l.closing = true
+ for c := range l.conns {
+ c.Abort(errors.New("listener closed"))
+ }
+ if len(l.conns) == 0 {
+ l.udpConn.Close()
+ }
+ }
+ l.connsMu.Unlock()
+ select {
+ case <-l.closec:
+ case <-ctx.Done():
+ l.connsMu.Lock()
+ for c := range l.conns {
+ c.exit()
+ }
+ l.connsMu.Unlock()
+ return ctx.Err()
+ }
+ return nil
+}
+
+// Accept waits for and returns the next connection to the listener.
+func (l *Listener) Accept(ctx context.Context) (*Conn, error) {
+ return l.acceptQueue.get(ctx, nil)
+}
+
+// Dial creates and returns a connection to a network address.
+func (l *Listener) Dial(ctx context.Context, network, address string) (*Conn, error) {
+ u, err := net.ResolveUDPAddr(network, address)
+ if err != nil {
+ return nil, err
+ }
+ addr := u.AddrPort()
+ addr = netip.AddrPortFrom(addr.Addr().Unmap(), addr.Port())
+ c, err := l.newConn(time.Now(), clientSide, nil, addr)
+ if err != nil {
+ return nil, err
+ }
+ if err := c.waitReady(ctx); err != nil {
+ c.Abort(nil)
+ return nil, err
+ }
+ return c, nil
+}
+
+func (l *Listener) newConn(now time.Time, side connSide, initialConnID []byte, peerAddr netip.AddrPort) (*Conn, error) {
+ l.connsMu.Lock()
+ defer l.connsMu.Unlock()
+ if l.closing {
+ return nil, errors.New("listener closed")
+ }
+ c, err := newConn(now, side, initialConnID, peerAddr, l.config, l, l.testHooks)
+ if err != nil {
+ return nil, err
+ }
+ l.conns[c] = struct{}{}
+ return c, nil
+}
+
+// serverConnEstablished is called by a conn when the handshake completes
+// for an inbound (serverSide) connection.
+func (l *Listener) serverConnEstablished(c *Conn) {
+ l.acceptQueue.put(c)
+}
+
+// connDrained is called by a conn when it leaves the draining state,
+// either when the peer acknowledges connection closure or the drain timeout expires.
+func (l *Listener) connDrained(c *Conn) {
+ l.connsMu.Lock()
+ defer l.connsMu.Unlock()
+ delete(l.conns, c)
+ if l.closing && len(l.conns) == 0 {
+ l.udpConn.Close()
+ }
+}
+
+// connIDsChanged is called by a conn when its connection IDs change.
+func (l *Listener) connIDsChanged(c *Conn, retired bool, cids []connID) {
+ l.connIDUpdateMu.Lock()
+ defer l.connIDUpdateMu.Unlock()
+ for _, cid := range cids {
+ l.connIDUpdates = append(l.connIDUpdates, connIDUpdate{
+ conn: c,
+ retired: retired,
+ cid: cid.cid,
+ })
+ }
+ l.connIDUpdateNeeded.Store(true)
+}
+
+// updateConnIDs is called by the datagram receive loop to update its connection ID map.
+func (l *Listener) updateConnIDs(conns map[string]*Conn) {
+ l.connIDUpdateMu.Lock()
+ defer l.connIDUpdateMu.Unlock()
+ for i, u := range l.connIDUpdates {
+ if u.retired {
+ delete(conns, string(u.cid))
+ } else {
+ conns[string(u.cid)] = u.conn
+ }
+ l.connIDUpdates[i] = connIDUpdate{} // drop refs
+ }
+ l.connIDUpdates = l.connIDUpdates[:0]
+ l.connIDUpdateNeeded.Store(false)
+}
+
+func (l *Listener) listen() {
+ defer close(l.closec)
+ conns := map[string]*Conn{}
+ for {
+ m := newDatagram()
+ // TODO: Read and process the ECN (explicit congestion notification) field.
+ // https://tools.ietf.org/html/draft-ietf-quic-transport-32#section-13.4
+ n, _, _, addr, err := l.udpConn.ReadMsgUDPAddrPort(m.b, nil)
+ if err != nil {
+ // The user has probably closed the listener.
+ // We currently don't surface errors from other causes;
+ // we could check to see if the listener has been closed and
+ // record the unexpected error if it has not.
+ return
+ }
+ if n == 0 {
+ continue
+ }
+ if l.connIDUpdateNeeded.Load() {
+ l.updateConnIDs(conns)
+ }
+ m.addr = addr
+ m.b = m.b[:n]
+ l.handleDatagram(m, conns)
+ }
+}
+
+func (l *Listener) handleDatagram(m *datagram, conns map[string]*Conn) {
+ dstConnID, ok := dstConnIDForDatagram(m.b)
+ if !ok {
+ m.recycle()
+ return
+ }
+ c := conns[string(dstConnID)]
+ if c == nil {
+ // TODO: Move this branch into a separate goroutine to avoid blocking
+ // the listener while processing packets.
+ l.handleUnknownDestinationDatagram(m)
+ return
+ }
+
+ // TODO: This can block the listener while waiting for the conn to accept the dgram.
+ // Think about buffering between the receive loop and the conn.
+ c.sendMsg(m)
+}
+
+func (l *Listener) handleUnknownDestinationDatagram(m *datagram) {
+ defer func() {
+ if m != nil {
+ m.recycle()
+ }
+ }()
+ if len(m.b) < minimumClientInitialDatagramSize {
+ return
+ }
+ p, ok := parseGenericLongHeaderPacket(m.b)
+ if !ok {
+ // Not a long header packet, or not parseable.
+ // Short header (1-RTT) packets don't contain enough information
+ // to do anything useful with if we don't recognize the
+ // connection ID.
+ return
+ }
+
+ switch p.version {
+ case quicVersion1:
+ case 0:
+ // Version Negotiation for an unknown connection.
+ return
+ default:
+ // Unknown version.
+ l.sendVersionNegotiation(p, m.addr)
+ return
+ }
+ if getPacketType(m.b) != packetTypeInitial {
+ // This packet isn't trying to create a new connection.
+ // It might be associated with some connection we've lost state for.
+ // TODO: Send a stateless reset when appropriate.
+ // https://www.rfc-editor.org/rfc/rfc9000.html#section-10.3
+ return
+ }
+ var now time.Time
+ if l.testHooks != nil {
+ now = l.testHooks.timeNow()
+ } else {
+ now = time.Now()
+ }
+ var err error
+ c, err := l.newConn(now, serverSide, p.dstConnID, m.addr)
+ if err != nil {
+ // The accept queue is probably full.
+ // We could send a CONNECTION_CLOSE to the peer to reject the connection.
+ // Currently, we just drop the datagram.
+ // https://www.rfc-editor.org/rfc/rfc9000.html#section-5.2.2-5
+ return
+ }
+ c.sendMsg(m)
+ m = nil // don't recycle, sendMsg takes ownership
+}
+
+func (l *Listener) sendVersionNegotiation(p genericLongPacket, addr netip.AddrPort) {
+ m := newDatagram()
+ m.b = appendVersionNegotiation(m.b[:0], p.srcConnID, p.dstConnID, quicVersion1)
+ l.sendDatagram(m.b, addr)
+ m.recycle()
+}
+
+func (l *Listener) sendDatagram(p []byte, addr netip.AddrPort) error {
+ _, err := l.udpConn.WriteToUDPAddrPort(p, addr)
+ return err
+}
diff --git a/internal/quic/listener_test.go b/internal/quic/listener_test.go
new file mode 100644
index 0000000000..9d0f314ecc
--- /dev/null
+++ b/internal/quic/listener_test.go
@@ -0,0 +1,163 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build go1.21
+
+package quic
+
+import (
+ "bytes"
+ "context"
+ "io"
+ "net"
+ "net/netip"
+ "testing"
+)
+
+func TestConnect(t *testing.T) {
+ newLocalConnPair(t, &Config{}, &Config{})
+}
+
+func TestStreamTransfer(t *testing.T) {
+ ctx := context.Background()
+ cli, srv := newLocalConnPair(t, &Config{}, &Config{})
+ data := makeTestData(1 << 20)
+
+ srvdone := make(chan struct{})
+ go func() {
+ defer close(srvdone)
+ s, err := srv.AcceptStream(ctx)
+ if err != nil {
+ t.Errorf("AcceptStream: %v", err)
+ return
+ }
+ b, err := io.ReadAll(s)
+ if err != nil {
+ t.Errorf("io.ReadAll(s): %v", err)
+ return
+ }
+ if !bytes.Equal(b, data) {
+ t.Errorf("read data mismatch (got %v bytes, want %v", len(b), len(data))
+ }
+ if err := s.Close(); err != nil {
+ t.Errorf("s.Close() = %v", err)
+ }
+ }()
+
+ s, err := cli.NewStream(ctx)
+ if err != nil {
+ t.Fatalf("NewStream: %v", err)
+ }
+ n, err := io.Copy(s, bytes.NewBuffer(data))
+ if n != int64(len(data)) || err != nil {
+ t.Fatalf("io.Copy(s, data) = %v, %v; want %v, nil", n, err, len(data))
+ }
+ if err := s.Close(); err != nil {
+ t.Fatalf("s.Close() = %v", err)
+ }
+}
+
+func newLocalConnPair(t *testing.T, conf1, conf2 *Config) (clientConn, serverConn *Conn) {
+ t.Helper()
+ ctx := context.Background()
+ l1 := newLocalListener(t, serverSide, conf1)
+ l2 := newLocalListener(t, clientSide, conf2)
+ c2, err := l2.Dial(ctx, "udp", l1.LocalAddr().String())
+ if err != nil {
+ t.Fatal(err)
+ }
+ c1, err := l1.Accept(ctx)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return c2, c1
+}
+
+func newLocalListener(t *testing.T, side connSide, conf *Config) *Listener {
+ t.Helper()
+ if conf.TLSConfig == nil {
+ conf.TLSConfig = newTestTLSConfig(side)
+ }
+ l, err := Listen("udp", "127.0.0.1:0", conf)
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() {
+ l.Close(context.Background())
+ })
+ return l
+}
+
+type testListener struct {
+ t *testing.T
+ l *Listener
+ recvc chan *datagram
+ idlec chan struct{}
+ sentDatagrams [][]byte
+}
+
+func newTestListener(t *testing.T, config *Config, testHooks connTestHooks) *testListener {
+ tl := &testListener{
+ t: t,
+ recvc: make(chan *datagram),
+ idlec: make(chan struct{}),
+ }
+ tl.l = newListener((*testListenerUDPConn)(tl), config, testHooks)
+ t.Cleanup(tl.cleanup)
+ return tl
+}
+
+func (tl *testListener) cleanup() {
+ tl.l.Close(canceledContext())
+}
+
+func (tl *testListener) wait() {
+ tl.idlec <- struct{}{}
+}
+
+func (tl *testListener) write(d *datagram) {
+ tl.recvc <- d
+ tl.wait()
+}
+
+func (tl *testListener) read() []byte {
+ tl.wait()
+ if len(tl.sentDatagrams) == 0 {
+ return nil
+ }
+ d := tl.sentDatagrams[0]
+ tl.sentDatagrams = tl.sentDatagrams[1:]
+ return d
+}
+
+// testListenerUDPConn implements UDPConn.
+type testListenerUDPConn testListener
+
+func (tl *testListenerUDPConn) Close() error {
+ close(tl.recvc)
+ return nil
+}
+
+func (tl *testListenerUDPConn) LocalAddr() net.Addr {
+ return net.UDPAddrFromAddrPort(netip.MustParseAddrPort("127.0.0.1:443"))
+}
+
+func (tl *testListenerUDPConn) ReadMsgUDPAddrPort(b, control []byte) (n, controln, flags int, _ netip.AddrPort, _ error) {
+ for {
+ select {
+ case d, ok := <-tl.recvc:
+ if !ok {
+ return 0, 0, 0, netip.AddrPort{}, io.EOF
+ }
+ n = copy(b, d.b)
+ return n, 0, 0, d.addr, nil
+ case <-tl.idlec:
+ }
+ }
+}
+
+func (tl *testListenerUDPConn) WriteToUDPAddrPort(b []byte, addr netip.AddrPort) (int, error) {
+ tl.sentDatagrams = append(tl.sentDatagrams, append([]byte(nil), b...))
+ return len(b), nil
+}
diff --git a/internal/quic/log.go b/internal/quic/log.go
new file mode 100644
index 0000000000..d7248343b0
--- /dev/null
+++ b/internal/quic/log.go
@@ -0,0 +1,69 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build go1.21
+
+package quic
+
+import (
+ "fmt"
+ "os"
+ "strings"
+)
+
+var logPackets bool
+
+// Parse GODEBUG settings.
+//
+// GODEBUG=quiclogpackets=1 -- log every packet sent and received.
+func init() {
+ s := os.Getenv("GODEBUG")
+ for len(s) > 0 {
+ var opt string
+ opt, s, _ = strings.Cut(s, ",")
+ switch opt {
+ case "quiclogpackets=1":
+ logPackets = true
+ }
+ }
+}
+
+func logInboundLongPacket(c *Conn, p longPacket) {
+ if !logPackets {
+ return
+ }
+ prefix := c.String()
+ fmt.Printf("%v recv %v %v\n", prefix, p.ptype, p.num)
+ logFrames(prefix+" <- ", p.payload)
+}
+
+func logInboundShortPacket(c *Conn, p shortPacket) {
+ if !logPackets {
+ return
+ }
+ prefix := c.String()
+ fmt.Printf("%v recv 1-RTT %v\n", prefix, p.num)
+ logFrames(prefix+" <- ", p.payload)
+}
+
+func logSentPacket(c *Conn, ptype packetType, pnum packetNumber, src, dst, payload []byte) {
+ if !logPackets || len(payload) == 0 {
+ return
+ }
+ prefix := c.String()
+ fmt.Printf("%v send %v %v\n", prefix, ptype, pnum)
+ logFrames(prefix+" -> ", payload)
+}
+
+func logFrames(prefix string, payload []byte) {
+ for len(payload) > 0 {
+ f, n := parseDebugFrame(payload)
+ if n < 0 {
+ fmt.Printf("%vBAD DATA\n", prefix)
+ break
+ }
+ payload = payload[n:]
+ fmt.Printf("%v%v\n", prefix, f)
+ }
+}
diff --git a/internal/quic/packet.go b/internal/quic/packet.go
index 93a9102e8d..7d69f96d27 100644
--- a/internal/quic/packet.go
+++ b/internal/quic/packet.go
@@ -6,6 +6,11 @@
package quic
+import (
+ "encoding/binary"
+ "fmt"
+)
+
// packetType is a QUIC packet type.
// https://www.rfc-editor.org/rfc/rfc9000.html#section-17
type packetType byte
@@ -20,12 +25,30 @@ const (
packetTypeVersionNegotiation
)
+func (p packetType) String() string {
+ switch p {
+ case packetTypeInitial:
+ return "Initial"
+ case packetType0RTT:
+ return "0-RTT"
+ case packetTypeHandshake:
+ return "Handshake"
+ case packetTypeRetry:
+ return "Retry"
+ case packetType1RTT:
+ return "1-RTT"
+ }
+ return fmt.Sprintf("unknown packet type %v", byte(p))
+}
+
// Bits set in the first byte of a packet.
const (
- headerFormLong = 0x80 // https://www.rfc-editor.org/rfc/rfc9000.html#section-17.2-3.2.1
- headerFormShort = 0x00 // https://www.rfc-editor.org/rfc/rfc9000.html#section-17.3.1-4.2.1
- fixedBit = 0x40 // https://www.rfc-editor.org/rfc/rfc9000.html#section-17.2-3.4.1
- reservedBits = 0x0c // https://www.rfc-editor.org/rfc/rfc9000#section-17.2-8.2.1
+ headerFormLong = 0x80 // https://www.rfc-editor.org/rfc/rfc9000.html#section-17.2-3.2.1
+ headerFormShort = 0x00 // https://www.rfc-editor.org/rfc/rfc9000.html#section-17.3.1-4.2.1
+ fixedBit = 0x40 // https://www.rfc-editor.org/rfc/rfc9000.html#section-17.2-3.4.1
+ reservedLongBits = 0x0c // https://www.rfc-editor.org/rfc/rfc9000#section-17.2-8.2.1
+ reserved1RTTBits = 0x18 // https://www.rfc-editor.org/rfc/rfc9000#section-17.3.1-4.8.1
+ keyPhaseBit = 0x04 // https://www.rfc-editor.org/rfc/rfc9000#section-17.3.1-4.10.1
)
// Long Packet Type bits.
@@ -137,15 +160,41 @@ func dstConnIDForDatagram(pkt []byte) (id []byte, ok bool) {
return b[:n], true
}
+// parseVersionNegotiation parses a Version Negotiation packet.
+// The returned versions is a slice of big-endian uint32s.
+// It returns (nil, nil, nil) for an invalid packet.
+func parseVersionNegotiation(pkt []byte) (dstConnID, srcConnID, versions []byte) {
+ p, ok := parseGenericLongHeaderPacket(pkt)
+ if !ok {
+ return nil, nil, nil
+ }
+ if len(p.data)%4 != 0 {
+ return nil, nil, nil
+ }
+ return p.dstConnID, p.srcConnID, p.data
+}
+
+// appendVersionNegotiation appends a Version Negotiation packet to pkt,
+// returning the result.
+func appendVersionNegotiation(pkt, dstConnID, srcConnID []byte, versions ...uint32) []byte {
+ pkt = append(pkt, headerFormLong|fixedBit) // header byte
+ pkt = append(pkt, 0, 0, 0, 0) // Version (0 for Version Negotiation)
+ pkt = appendUint8Bytes(pkt, dstConnID) // Destination Connection ID
+ pkt = appendUint8Bytes(pkt, srcConnID) // Source Connection ID
+ for _, v := range versions {
+ pkt = binary.BigEndian.AppendUint32(pkt, v) // Supported Version
+ }
+ return pkt
+}
+
// A longPacket is a long header packet.
type longPacket struct {
- ptype packetType
- reservedBits uint8
- version uint32
- num packetNumber
- dstConnID []byte
- srcConnID []byte
- payload []byte
+ ptype packetType
+ version uint32
+ num packetNumber
+ dstConnID []byte
+ srcConnID []byte
+ payload []byte
// The extra data depends on the packet type:
// Initial: Token.
@@ -155,7 +204,45 @@ type longPacket struct {
// A shortPacket is a short header (1-RTT) packet.
type shortPacket struct {
- reservedBits uint8
- num packetNumber
- payload []byte
+ num packetNumber
+ payload []byte
+}
+
+// A genericLongPacket is a long header packet of an arbitrary QUIC version.
+// https://www.rfc-editor.org/rfc/rfc8999#section-5.1
+type genericLongPacket struct {
+ version uint32
+ dstConnID []byte
+ srcConnID []byte
+ data []byte
+}
+
+func parseGenericLongHeaderPacket(b []byte) (p genericLongPacket, ok bool) {
+ if len(b) < 5 || !isLongHeader(b[0]) {
+ return genericLongPacket{}, false
+ }
+ b = b[1:]
+ // Version (32),
+ var n int
+ p.version, n = consumeUint32(b)
+ if n < 0 {
+ return genericLongPacket{}, false
+ }
+ b = b[n:]
+ // Destination Connection ID Length (8),
+ // Destination Connection ID (0..2048),
+ p.dstConnID, n = consumeUint8Bytes(b)
+ if n < 0 || len(p.dstConnID) > 2048/8 {
+ return genericLongPacket{}, false
+ }
+ b = b[n:]
+ // Source Connection ID Length (8),
+ // Source Connection ID (0..2048),
+ p.srcConnID, n = consumeUint8Bytes(b)
+ if n < 0 || len(p.dstConnID) > 2048/8 {
+ return genericLongPacket{}, false
+ }
+ b = b[n:]
+ p.data = b
+ return p, true
}
diff --git a/internal/quic/packet_codec_test.go b/internal/quic/packet_codec_test.go
index 3503d24318..7b01bb00d6 100644
--- a/internal/quic/packet_codec_test.go
+++ b/internal/quic/packet_codec_test.go
@@ -17,7 +17,7 @@ func TestParseLongHeaderPacket(t *testing.T) {
// Example Initial packet from:
// https://www.rfc-editor.org/rfc/rfc9001.html#section-a.3
cid := unhex(`8394c8f03e515708`)
- _, initialServerKeys := initialKeys(cid)
+ initialServerKeys := initialKeys(cid, clientSide).r
pkt := unhex(`
cf000000010008f067a5502a4262b500 4075c0d95a482cd0991cd25b0aac406a
5816b6394100f37a1c69797554780bb3 8cc5a99f5ede4cf73c3ec2493a1839b3
@@ -65,20 +65,21 @@ func TestParseLongHeaderPacket(t *testing.T) {
}
// Parse with the wrong keys.
- _, invalidKeys := initialKeys([]byte{})
+ invalidKeys := initialKeys([]byte{}, clientSide).w
if _, n := parseLongHeaderPacket(pkt, invalidKeys, 0); n != -1 {
t.Fatalf("parse long header packet with wrong keys: n=%v, want -1", n)
}
}
func TestRoundtripEncodeLongPacket(t *testing.T) {
- aes128Keys, _ := newKeys(tls.TLS_AES_128_GCM_SHA256, []byte("secret"))
- aes256Keys, _ := newKeys(tls.TLS_AES_256_GCM_SHA384, []byte("secret"))
- chachaKeys, _ := newKeys(tls.TLS_CHACHA20_POLY1305_SHA256, []byte("secret"))
+ var aes128Keys, aes256Keys, chachaKeys fixedKeys
+ aes128Keys.init(tls.TLS_AES_128_GCM_SHA256, []byte("secret"))
+ aes256Keys.init(tls.TLS_AES_256_GCM_SHA384, []byte("secret"))
+ chachaKeys.init(tls.TLS_CHACHA20_POLY1305_SHA256, []byte("secret"))
for _, test := range []struct {
desc string
p longPacket
- k keys
+ k fixedKeys
}{{
desc: "Initial, 1-byte number, AES128",
p: longPacket{
@@ -145,9 +146,16 @@ func TestRoundtripEncodeLongPacket(t *testing.T) {
}
func TestRoundtripEncodeShortPacket(t *testing.T) {
- aes128Keys, _ := newKeys(tls.TLS_AES_128_GCM_SHA256, []byte("secret"))
- aes256Keys, _ := newKeys(tls.TLS_AES_256_GCM_SHA384, []byte("secret"))
- chachaKeys, _ := newKeys(tls.TLS_CHACHA20_POLY1305_SHA256, []byte("secret"))
+ var aes128Keys, aes256Keys, chachaKeys updatingKeyPair
+ aes128Keys.r.init(tls.TLS_AES_128_GCM_SHA256, []byte("secret"))
+ aes256Keys.r.init(tls.TLS_AES_256_GCM_SHA384, []byte("secret"))
+ chachaKeys.r.init(tls.TLS_CHACHA20_POLY1305_SHA256, []byte("secret"))
+ aes128Keys.w = aes128Keys.r
+ aes256Keys.w = aes256Keys.r
+ chachaKeys.w = chachaKeys.r
+ aes128Keys.updateAfter = maxPacketNumber
+ aes256Keys.updateAfter = maxPacketNumber
+ chachaKeys.updateAfter = maxPacketNumber
connID := make([]byte, connIDLen)
for i := range connID {
connID[i] = byte(i)
@@ -156,7 +164,7 @@ func TestRoundtripEncodeShortPacket(t *testing.T) {
desc string
num packetNumber
payload []byte
- k keys
+ k updatingKeyPair
}{{
desc: "1-byte number, AES128",
num: 0, // 1-byte encoding,
@@ -183,11 +191,11 @@ func TestRoundtripEncodeShortPacket(t *testing.T) {
w.reset(1200)
w.start1RTTPacket(test.num, 0, connID)
w.b = append(w.b, test.payload...)
- w.finish1RTTPacket(test.num, 0, connID, test.k)
+ w.finish1RTTPacket(test.num, 0, connID, &test.k)
pkt := w.datagram()
- p, n := parse1RTTPacket(pkt, test.k, connIDLen, 0)
- if n != len(pkt) {
- t.Errorf("parse1RTTPacket: n=%v, want %v", n, len(pkt))
+ p, err := parse1RTTPacket(pkt, &test.k, connIDLen, 0)
+ if err != nil {
+ t.Errorf("parse1RTTPacket: err=%v, want nil", err)
}
if p.num != test.num || !bytes.Equal(p.payload, test.payload) {
t.Errorf("Round-trip encode/decode did not preserve packet.\nsent: num=%v, payload={%x}\ngot: num=%v, payload={%x}", test.num, test.payload, p.num, p.payload)
@@ -700,7 +708,7 @@ func TestFrameDecodeErrors(t *testing.T) {
func FuzzParseLongHeaderPacket(f *testing.F) {
cid := unhex(`0000000000000000`)
- _, initialServerKeys := initialKeys(cid)
+ initialServerKeys := initialKeys(cid, clientSide).r
f.Fuzz(func(t *testing.T, in []byte) {
parseLongHeaderPacket(in, initialServerKeys, 0)
})
diff --git a/internal/quic/packet_parser.go b/internal/quic/packet_parser.go
index 908a82ed90..ce04339025 100644
--- a/internal/quic/packet_parser.go
+++ b/internal/quic/packet_parser.go
@@ -18,7 +18,7 @@ package quic
// and its length in bytes.
//
// It returns an empty packet and -1 if the packet could not be parsed.
-func parseLongHeaderPacket(pkt []byte, k keys, pnumMax packetNumber) (p longPacket, n int) {
+func parseLongHeaderPacket(pkt []byte, k fixedKeys, pnumMax packetNumber) (p longPacket, n int) {
if len(pkt) < 5 || !isLongHeader(pkt[0]) {
return longPacket{}, -1
}
@@ -91,15 +91,12 @@ func parseLongHeaderPacket(pkt []byte, k keys, pnumMax packetNumber) (p longPack
pnumOff := len(pkt) - len(b)
pkt = pkt[:pnumOff+int(payLen)]
- if k.initialized() {
+ if k.isSet() {
var err error
p.payload, p.num, err = k.unprotect(pkt, pnumOff, pnumMax)
if err != nil {
return longPacket{}, -1
}
- // Reserved bits should always be zero, but this is handled
- // as a protocol-level violation by the caller rather than a parse error.
- p.reservedBits = pkt[0] & reservedBits
}
return p, len(pkt)
}
@@ -146,23 +143,21 @@ func skipLongHeaderPacket(pkt []byte) int {
//
// On input, pkt contains a short header packet, k the decryption keys for the packet,
// and pnumMax the largest packet number seen in the number space of this packet.
-func parse1RTTPacket(pkt []byte, k keys, dstConnIDLen int, pnumMax packetNumber) (p shortPacket, n int) {
- var err error
- p.payload, p.num, err = k.unprotect(pkt, 1+dstConnIDLen, pnumMax)
+func parse1RTTPacket(pkt []byte, k *updatingKeyPair, dstConnIDLen int, pnumMax packetNumber) (p shortPacket, err error) {
+ pay, pnum, err := k.unprotect(pkt, 1+dstConnIDLen, pnumMax)
if err != nil {
- return shortPacket{}, -1
+ return shortPacket{}, err
}
- // Reserved bits should always be zero, but this is handled
- // as a protocol-level violation by the caller rather than a parse error.
- p.reservedBits = pkt[0] & reservedBits
- return p, len(pkt)
+ p.num = pnum
+ p.payload = pay
+ return p, nil
}
// Consume functions return n=-1 on conditions which result in FRAME_ENCODING_ERROR,
// which includes both general parse failures and specific violations of frame
// constraints.
-func consumeAckFrame(frame []byte, f func(start, end packetNumber)) (largest packetNumber, ackDelay unscaledAckDelay, n int) {
+func consumeAckFrame(frame []byte, f func(rangeIndex int, start, end packetNumber)) (largest packetNumber, ackDelay unscaledAckDelay, n int) {
b := frame[1:] // type
largestAck, n := consumeVarint(b)
@@ -195,7 +190,7 @@ func consumeAckFrame(frame []byte, f func(start, end packetNumber)) (largest pac
if rangeMin < 0 || rangeMin > rangeMax {
return 0, 0, -1
}
- f(rangeMin, rangeMax+1)
+ f(int(i), rangeMin, rangeMax+1)
if i == ackRangeCount {
break
@@ -330,6 +325,9 @@ func consumeStreamFrame(b []byte) (id streamID, off int64, fin bool, data []byte
data = b[n:]
n += len(data)
}
+ if off+int64(len(data)) >= 1<<62 {
+ return 0, 0, false, nil, -1
+ }
return streamID(idInt), off, fin, data, n
}
@@ -375,7 +373,7 @@ func consumeMaxStreamsFrame(b []byte) (typ streamType, max int64, n int) {
return 0, 0, -1
}
n += nn
- if v > 1<<60 {
+ if v > maxStreamsLimit {
return 0, 0, -1
}
return typ, int64(v), n
@@ -454,10 +452,10 @@ func consumeNewConnectionIDFrame(b []byte) (seq, retire int64, connID []byte, re
return seq, retire, connID, resetToken, n
}
-func consumeRetireConnectionIDFrame(b []byte) (seq uint64, n int) {
+func consumeRetireConnectionIDFrame(b []byte) (seq int64, n int) {
n = 1
var nn int
- seq, nn = consumeVarint(b[n:])
+ seq, nn = consumeVarintInt64(b[n:])
if nn < 0 {
return 0, -1
}
diff --git a/internal/quic/packet_protection.go b/internal/quic/packet_protection.go
index 1f0a735e8e..7b141ac49e 100644
--- a/internal/quic/packet_protection.go
+++ b/internal/quic/packet_protection.go
@@ -13,7 +13,6 @@ import (
"crypto/sha256"
"crypto/tls"
"errors"
- "fmt"
"hash"
"golang.org/x/crypto/chacha20"
@@ -24,135 +23,183 @@ import (
var errInvalidPacket = errors.New("quic: invalid packet")
-// keys holds the cryptographic material used to protect packets
-// at an encryption level and direction. (e.g., Initial client keys.)
-//
-// keys are not safe for concurrent use.
-type keys struct {
- // AEAD function used for packet protection.
- aead cipher.AEAD
+// headerProtectionSampleSize is the size of the ciphertext sample used for header protection.
+// https://www.rfc-editor.org/rfc/rfc9001#section-5.4.2
+const headerProtectionSampleSize = 16
- // The header_protection function as defined in:
- // https://www.rfc-editor.org/rfc/rfc9001#section-5.4.1
- //
- // This function takes a sample of the packet ciphertext
- // and returns a 5-byte mask which will be applied to the
- // protected portions of the packet header.
- headerProtection func(sample []byte) (mask [5]byte)
+// aeadOverhead is the difference in size between the AEAD output and input.
+// All cipher suites defined for use with QUIC have 16 bytes of overhead.
+const aeadOverhead = 16
- // IV used to construct the AEAD nonce.
- iv []byte
+// A headerKey applies or removes header protection.
+// https://www.rfc-editor.org/rfc/rfc9001#section-5.4
+type headerKey struct {
+ hp headerProtection
}
-// newKeys creates keys for a given cipher suite and secret.
-//
-// It returns an error if the suite is unknown.
-func newKeys(suite uint16, secret []byte) (keys, error) {
+func (k headerKey) isSet() bool {
+ return k.hp != nil
+}
+
+func (k *headerKey) init(suite uint16, secret []byte) {
+ h, keySize := hashForSuite(suite)
+ hpKey := hkdfExpandLabel(h.New, secret, "quic hp", nil, keySize)
switch suite {
- case tls.TLS_AES_128_GCM_SHA256:
- return newAESKeys(secret, crypto.SHA256, 128/8), nil
- case tls.TLS_AES_256_GCM_SHA384:
- return newAESKeys(secret, crypto.SHA384, 256/8), nil
+ case tls.TLS_AES_128_GCM_SHA256, tls.TLS_AES_256_GCM_SHA384:
+ c, err := aes.NewCipher(hpKey)
+ if err != nil {
+ panic(err)
+ }
+ k.hp = &aesHeaderProtection{cipher: c}
case tls.TLS_CHACHA20_POLY1305_SHA256:
- return newChaCha20Keys(secret), nil
+ k.hp = chaCha20HeaderProtection{hpKey}
+ default:
+ panic("BUG: unknown cipher suite")
}
- return keys{}, fmt.Errorf("unknown cipher suite %x", suite)
}
-func newAESKeys(secret []byte, h crypto.Hash, keyBytes int) keys {
- // https://www.rfc-editor.org/rfc/rfc9001#section-5.1
- key := hkdfExpandLabel(h.New, secret, "quic key", nil, keyBytes)
- c, err := aes.NewCipher(key)
- if err != nil {
- panic(err)
+// protect applies header protection.
+// pnumOff is the offset of the packet number in the packet.
+func (k headerKey) protect(hdr []byte, pnumOff int) {
+ // Apply header protection.
+ pnumSize := int(hdr[0]&0x03) + 1
+ sample := hdr[pnumOff+4:][:headerProtectionSampleSize]
+ mask := k.hp.headerProtection(sample)
+ if isLongHeader(hdr[0]) {
+ hdr[0] ^= mask[0] & 0x0f
+ } else {
+ hdr[0] ^= mask[0] & 0x1f
}
- aead, err := cipher.NewGCM(c)
- if err != nil {
- panic(err)
+ for i := 0; i < pnumSize; i++ {
+ hdr[pnumOff+i] ^= mask[1+i]
}
- iv := hkdfExpandLabel(h.New, secret, "quic iv", nil, aead.NonceSize())
- // https://www.rfc-editor.org/rfc/rfc9001#section-5.4.3
- hpKey := hkdfExpandLabel(h.New, secret, "quic hp", nil, keyBytes)
- hp, err := aes.NewCipher(hpKey)
- if err != nil {
- panic(err)
+}
+
+// unprotect removes header protection.
+// pnumOff is the offset of the packet number in the packet.
+// pnumMax is the largest packet number seen in the number space of this packet.
+func (k headerKey) unprotect(pkt []byte, pnumOff int, pnumMax packetNumber) (hdr, pay []byte, pnum packetNumber, _ error) {
+ if len(pkt) < pnumOff+4+headerProtectionSampleSize {
+ return nil, nil, 0, errInvalidPacket
}
- var scratch [aes.BlockSize]byte
- headerProtection := func(sample []byte) (mask [5]byte) {
- hp.Encrypt(scratch[:], sample)
- copy(mask[:], scratch[:])
- return mask
+ numpay := pkt[pnumOff:]
+ sample := numpay[4:][:headerProtectionSampleSize]
+ mask := k.hp.headerProtection(sample)
+ if isLongHeader(pkt[0]) {
+ pkt[0] ^= mask[0] & 0x0f
+ } else {
+ pkt[0] ^= mask[0] & 0x1f
}
- return keys{
- aead: aead,
- iv: iv,
- headerProtection: headerProtection,
+ pnumLen := int(pkt[0]&0x03) + 1
+ pnum = packetNumber(0)
+ for i := 0; i < pnumLen; i++ {
+ numpay[i] ^= mask[1+i]
+ pnum = (pnum << 8) | packetNumber(numpay[i])
}
+ pnum = decodePacketNumber(pnumMax, pnum, pnumLen)
+ hdr = pkt[:pnumOff+pnumLen]
+ pay = numpay[pnumLen:]
+ return hdr, pay, pnum, nil
}
-func newChaCha20Keys(secret []byte) keys {
- // https://www.rfc-editor.org/rfc/rfc9001#section-5.1
- key := hkdfExpandLabel(sha256.New, secret, "quic key", nil, chacha20poly1305.KeySize)
- aead, err := chacha20poly1305.New(key)
+// headerProtection is the header_protection function as defined in:
+// https://www.rfc-editor.org/rfc/rfc9001#section-5.4.1
+//
+// This function takes a sample of the packet ciphertext
+// and returns a 5-byte mask which will be applied to the
+// protected portions of the packet header.
+type headerProtection interface {
+ headerProtection(sample []byte) (mask [5]byte)
+}
+
+// AES-based header protection.
+// https://www.rfc-editor.org/rfc/rfc9001#section-5.4.3
+type aesHeaderProtection struct {
+ cipher cipher.Block
+ scratch [aes.BlockSize]byte
+}
+
+func (hp *aesHeaderProtection) headerProtection(sample []byte) (mask [5]byte) {
+ hp.cipher.Encrypt(hp.scratch[:], sample)
+ copy(mask[:], hp.scratch[:])
+ return mask
+}
+
+// ChaCha20-based header protection.
+// https://www.rfc-editor.org/rfc/rfc9001#section-5.4.4
+type chaCha20HeaderProtection struct {
+ key []byte
+}
+
+func (hp chaCha20HeaderProtection) headerProtection(sample []byte) (mask [5]byte) {
+ counter := uint32(sample[3])<<24 | uint32(sample[2])<<16 | uint32(sample[1])<<8 | uint32(sample[0])
+ nonce := sample[4:16]
+ c, err := chacha20.NewUnauthenticatedCipher(hp.key, nonce)
if err != nil {
panic(err)
}
- iv := hkdfExpandLabel(sha256.New, secret, "quic iv", nil, aead.NonceSize())
- // https://www.rfc-editor.org/rfc/rfc9001#section-5.4.4
- hpKey := hkdfExpandLabel(sha256.New, secret, "quic hp", nil, chacha20.KeySize)
- headerProtection := func(sample []byte) [5]byte {
- counter := uint32(sample[3])<<24 | uint32(sample[2])<<16 | uint32(sample[1])<<8 | uint32(sample[0])
- nonce := sample[4:16]
- c, err := chacha20.NewUnauthenticatedCipher(hpKey, nonce)
- if err != nil {
- panic(err)
- }
- c.SetCounter(counter)
- var mask [5]byte
- c.XORKeyStream(mask[:], mask[:])
- return mask
- }
- return keys{
- aead: aead,
- iv: iv,
- headerProtection: headerProtection,
- }
+ c.SetCounter(counter)
+ c.XORKeyStream(mask[:], mask[:])
+ return mask
}
-// https://www.rfc-editor.org/rfc/rfc9001#section-5.2-2
-var initialSalt = []byte{0x38, 0x76, 0x2c, 0xf7, 0xf5, 0x59, 0x34, 0xb3, 0x4d, 0x17, 0x9a, 0xe6, 0xa4, 0xc8, 0x0c, 0xad, 0xcc, 0xbb, 0x7f, 0x0a}
+// A packetKey applies or removes packet protection.
+// https://www.rfc-editor.org/rfc/rfc9001#section-5.1
+type packetKey struct {
+ aead cipher.AEAD // AEAD function used for packet protection.
+ iv []byte // IV used to construct the AEAD nonce.
+}
-// initialKeys returns the keys used to protect Initial packets.
-//
-// The Initial packet keys are derived from the Destination Connection ID
-// field in the client's first Initial packet.
-//
-// https://www.rfc-editor.org/rfc/rfc9001#section-5.2
-func initialKeys(cid []byte) (clientKeys, serverKeys keys) {
- initialSecret := hkdf.Extract(sha256.New, cid, initialSalt)
- clientInitialSecret := hkdfExpandLabel(sha256.New, initialSecret, "client in", nil, sha256.Size)
- clientKeys, err := newKeys(tls.TLS_AES_128_GCM_SHA256, clientInitialSecret)
+func (k *packetKey) init(suite uint16, secret []byte) {
+ // https://www.rfc-editor.org/rfc/rfc9001#section-5.1
+ h, keySize := hashForSuite(suite)
+ key := hkdfExpandLabel(h.New, secret, "quic key", nil, keySize)
+ switch suite {
+ case tls.TLS_AES_128_GCM_SHA256, tls.TLS_AES_256_GCM_SHA384:
+ k.aead = newAESAEAD(key)
+ case tls.TLS_CHACHA20_POLY1305_SHA256:
+ k.aead = newChaCha20AEAD(key)
+ default:
+ panic("BUG: unknown cipher suite")
+ }
+ k.iv = hkdfExpandLabel(h.New, secret, "quic iv", nil, k.aead.NonceSize())
+}
+
+func newAESAEAD(key []byte) cipher.AEAD {
+ c, err := aes.NewCipher(key)
if err != nil {
panic(err)
}
-
- serverInitialSecret := hkdfExpandLabel(sha256.New, initialSecret, "server in", nil, sha256.Size)
- serverKeys, err = newKeys(tls.TLS_AES_128_GCM_SHA256, serverInitialSecret)
+ aead, err := cipher.NewGCM(c)
if err != nil {
panic(err)
}
+ return aead
+}
- return clientKeys, serverKeys
+func newChaCha20AEAD(key []byte) cipher.AEAD {
+ var err error
+ aead, err := chacha20poly1305.New(key)
+ if err != nil {
+ panic(err)
+ }
+ return aead
}
-const headerProtectionSampleSize = 16
+func (k packetKey) protect(hdr, pay []byte, pnum packetNumber) []byte {
+ k.xorIV(pnum)
+ defer k.xorIV(pnum)
+ return k.aead.Seal(hdr, k.iv, pay, hdr)
+}
-// aeadOverhead is the difference in size between the AEAD output and input.
-// All cipher suites defined for use with QUIC have 16 bytes of overhead.
-const aeadOverhead = 16
+func (k packetKey) unprotect(hdr, pay []byte, pnum packetNumber) (dec []byte, err error) {
+ k.xorIV(pnum)
+ defer k.xorIV(pnum)
+ return k.aead.Open(pay[:0], k.iv, pay, hdr)
+}
// xorIV xors the packet protection IV with the packet number.
-func (k keys) xorIV(pnum packetNumber) {
+func (k packetKey) xorIV(pnum packetNumber) {
k.iv[len(k.iv)-8] ^= uint8(pnum >> 56)
k.iv[len(k.iv)-7] ^= uint8(pnum >> 48)
k.iv[len(k.iv)-6] ^= uint8(pnum >> 40)
@@ -163,17 +210,22 @@ func (k keys) xorIV(pnum packetNumber) {
k.iv[len(k.iv)-1] ^= uint8(pnum)
}
-// initialized returns true if valid keys are available.
-func (k keys) initialized() bool {
- return k.aead != nil
+// A fixedKeys is a header protection key and fixed packet protection key.
+// The packet protection key is fixed (it does not update).
+//
+// Fixed keys are used for Initial and Handshake keys, which do not update.
+type fixedKeys struct {
+ hdr headerKey
+ pkt packetKey
}
-// discard discards the keys (in the sense that we won't use them any more,
-// not that the keys are securely erased).
-//
-// https://www.rfc-editor.org/rfc/rfc9001.html#section-4.9
-func (k *keys) discard() {
- *k = keys{}
+func (k *fixedKeys) init(suite uint16, secret []byte) {
+ k.hdr.init(suite, secret)
+ k.pkt.init(suite, secret)
+}
+
+func (k fixedKeys) isSet() bool {
+ return k.hdr.hp != nil
}
// protect applies packet protection to a packet.
@@ -184,25 +236,10 @@ func (k *keys) discard() {
//
// protect returns the result of appending the encrypted payload to hdr and
// applying header protection.
-func (k keys) protect(hdr, pay []byte, pnumOff int, pnum packetNumber) []byte {
- k.xorIV(pnum)
- hdr = k.aead.Seal(hdr, k.iv, pay, hdr)
- k.xorIV(pnum)
-
- // Apply header protection.
- pnumSize := int(hdr[0]&0x03) + 1
- sample := hdr[pnumOff+4:][:headerProtectionSampleSize]
- mask := k.headerProtection(sample)
- if isLongHeader(hdr[0]) {
- hdr[0] ^= mask[0] & 0x0f
- } else {
- hdr[0] ^= mask[0] & 0x1f
- }
- for i := 0; i < pnumSize; i++ {
- hdr[pnumOff+i] ^= mask[1+i]
- }
-
- return hdr
+func (k fixedKeys) protect(hdr, pay []byte, pnumOff int, pnum packetNumber) []byte {
+ pkt := k.pkt.protect(hdr, pay, pnum)
+ k.hdr.protect(pkt, pnumOff)
+ return pkt
}
// unprotect removes packet protection from a packet.
@@ -213,38 +250,269 @@ func (k keys) protect(hdr, pay []byte, pnumOff int, pnum packetNumber) []byte {
//
// unprotect removes header protection from the header in pkt, and returns
// the unprotected payload and packet number.
-func (k keys) unprotect(pkt []byte, pnumOff int, pnumMax packetNumber) (pay []byte, num packetNumber, err error) {
- if len(pkt) < pnumOff+4+headerProtectionSampleSize {
- return nil, 0, errInvalidPacket
+func (k fixedKeys) unprotect(pkt []byte, pnumOff int, pnumMax packetNumber) (pay []byte, num packetNumber, err error) {
+ hdr, pay, pnum, err := k.hdr.unprotect(pkt, pnumOff, pnumMax)
+ if err != nil {
+ return nil, 0, err
}
- numpay := pkt[pnumOff:]
- sample := numpay[4:][:headerProtectionSampleSize]
- mask := k.headerProtection(sample)
- if isLongHeader(pkt[0]) {
- pkt[0] ^= mask[0] & 0x0f
- } else {
- pkt[0] ^= mask[0] & 0x1f
+ pay, err = k.pkt.unprotect(hdr, pay, pnum)
+ if err != nil {
+ return nil, 0, err
}
- pnumLen := int(pkt[0]&0x03) + 1
- pnum := packetNumber(0)
- for i := 0; i < pnumLen; i++ {
- numpay[i] ^= mask[1+i]
- pnum = (pnum << 8) | packetNumber(numpay[i])
+ return pay, pnum, nil
+}
+
+// A fixedKeyPair is a read/write pair of fixed keys.
+type fixedKeyPair struct {
+ r, w fixedKeys
+}
+
+func (k *fixedKeyPair) discard() {
+ *k = fixedKeyPair{}
+}
+
+func (k *fixedKeyPair) canRead() bool {
+ return k.r.isSet()
+}
+
+func (k *fixedKeyPair) canWrite() bool {
+ return k.w.isSet()
+}
+
+// An updatingKeys is a header protection key and updatable packet protection key.
+// updatingKeys are used for 1-RTT keys, where the packet protection key changes
+// over the lifetime of a connection.
+// https://www.rfc-editor.org/rfc/rfc9001#section-6
+type updatingKeys struct {
+ suite uint16
+ hdr headerKey
+ pkt [2]packetKey // current, next
+ nextSecret []byte // secret used to generate pkt[1]
+}
+
+func (k *updatingKeys) init(suite uint16, secret []byte) {
+ k.suite = suite
+ k.hdr.init(suite, secret)
+ // Initialize pkt[1] with secret_0, and then call update to generate secret_1.
+ k.pkt[1].init(suite, secret)
+ k.nextSecret = secret
+ k.update()
+}
+
+// update performs a key update.
+// The current key in pkt[0] is discarded.
+// The next key in pkt[1] becomes the current key.
+// A new next key is generated in pkt[1].
+func (k *updatingKeys) update() {
+ k.nextSecret = updateSecret(k.suite, k.nextSecret)
+ k.pkt[0] = k.pkt[1]
+ k.pkt[1].init(k.suite, k.nextSecret)
+}
+
+func updateSecret(suite uint16, secret []byte) (nextSecret []byte) {
+ h, _ := hashForSuite(suite)
+ return hkdfExpandLabel(h.New, secret, "quic ku", nil, len(secret))
+}
+
+// An updatingKeyPair is a read/write pair of updating keys.
+//
+// We keep two keys (current and next) in both read and write directions.
+// When an incoming packet's phase matches the current phase bit,
+// we unprotect it using the current keys; otherwise we use the next keys.
+//
+// When updating=false, outgoing packets are protected using the current phase.
+//
+// An update is initiated and updating is set to true when:
+// - we decide to initiate a key update; or
+// - we successfully unprotect a packet using the next keys,
+// indicating the peer has initiated a key update.
+//
+// When updating=true, outgoing packets are protected using the next phase.
+// We do not change the current phase bit or generate new keys yet.
+//
+// The update concludes when we receive an ACK frame for a packet sent
+// with the next keys. At this time, we set updating to false, flip the
+// phase bit, and update the keys. This permits us to handle up to 1-RTT
+// of reordered packets before discarding the previous phase's keys after
+// an update.
+type updatingKeyPair struct {
+ phase uint8 // current key phase (r.pkt[0], w.pkt[0])
+ updating bool
+ authFailures int64 // total packet unprotect failures
+ minSent packetNumber // min packet number sent since entering the updating state
+ minReceived packetNumber // min packet number received in the next phase
+ updateAfter packetNumber // packet number after which to initiate key update
+ r, w updatingKeys
+}
+
+func (k *updatingKeyPair) init() {
+ // 1-RTT packets until the first key update.
+ //
+ // We perform the first key update early in the connection so a peer
+ // which does not support key updates will fail rapidly,
+ // rather than after the connection has been long established.
+ k.updateAfter = 1000
+}
+
+func (k *updatingKeyPair) canRead() bool {
+ return k.r.hdr.hp != nil
+}
+
+func (k *updatingKeyPair) canWrite() bool {
+ return k.w.hdr.hp != nil
+}
+
+// handleAckFor finishes a key update after receiving an ACK for a packet in the next phase.
+func (k *updatingKeyPair) handleAckFor(pnum packetNumber) {
+ if k.updating && pnum >= k.minSent {
+ k.updating = false
+ k.phase ^= keyPhaseBit
+ k.r.update()
+ k.w.update()
}
- pnum = decodePacketNumber(pnumMax, pnum, pnumLen)
+}
- hdr := pkt[:pnumOff+pnumLen]
- pay = numpay[pnumLen:]
- k.xorIV(pnum)
- pay, err = k.aead.Open(pay[:0], k.iv, pay, hdr)
- k.xorIV(pnum)
+// needAckEliciting reports whether we should send an ack-eliciting packet in the next phase.
+// The first packet sent in a phase is ack-eliciting, since the peer must acknowledge a
+// packet in the new phase for us to finish the update.
+func (k *updatingKeyPair) needAckEliciting() bool {
+ return k.updating && k.minSent == maxPacketNumber
+}
+
+// protect applies packet protection to a packet.
+// Parameters and returns are as for fixedKeyPair.protect.
+func (k *updatingKeyPair) protect(hdr, pay []byte, pnumOff int, pnum packetNumber) []byte {
+ var pkt []byte
+ if k.updating {
+ hdr[0] |= k.phase ^ keyPhaseBit
+ pkt = k.w.pkt[1].protect(hdr, pay, pnum)
+ k.minSent = min(pnum, k.minSent)
+ } else {
+ hdr[0] |= k.phase
+ pkt = k.w.pkt[0].protect(hdr, pay, pnum)
+ if pnum >= k.updateAfter {
+ // Initiate a key update, starting with the next packet we send.
+ //
+ // We do this after protecting the current packet
+ // to allow Conn.appendFrames to ensure that the first packet sent
+ // in the new phase is ack-eliciting.
+ k.updating = true
+ k.minSent = maxPacketNumber
+ k.minReceived = maxPacketNumber
+ // The lowest confidentiality limit for a supported AEAD is 2^23 packets.
+ // https://www.rfc-editor.org/rfc/rfc9001#section-6.6-5
+ //
+ // Schedule our next update for half that.
+ k.updateAfter += (1 << 22)
+ }
+ }
+ k.w.hdr.protect(pkt, pnumOff)
+ return pkt
+}
+
+// unprotect removes packet protection from a packet.
+// Parameters and returns are as for fixedKeyPair.unprotect.
+func (k *updatingKeyPair) unprotect(pkt []byte, pnumOff int, pnumMax packetNumber) (pay []byte, pnum packetNumber, err error) {
+ hdr, pay, pnum, err := k.r.hdr.unprotect(pkt, pnumOff, pnumMax)
if err != nil {
return nil, 0, err
}
-
+ // To avoid timing signals that might indicate the key phase bit is invalid,
+ // we always attempt to unprotect the packet with one key.
+ //
+ // If the key phase bit matches and the packet number doesn't come after
+ // the start of an in-progress update, use the current phase.
+ // Otherwise, use the next phase.
+ if hdr[0]&keyPhaseBit == k.phase && (!k.updating || pnum < k.minReceived) {
+ pay, err = k.r.pkt[0].unprotect(hdr, pay, pnum)
+ } else {
+ pay, err = k.r.pkt[1].unprotect(hdr, pay, pnum)
+ if err == nil {
+ if !k.updating {
+ // The peer has initiated a key update.
+ k.updating = true
+ k.minSent = maxPacketNumber
+ k.minReceived = pnum
+ } else {
+ k.minReceived = min(pnum, k.minReceived)
+ }
+ }
+ }
+ if err != nil {
+ k.authFailures++
+ if k.authFailures >= aeadIntegrityLimit(k.r.suite) {
+ return nil, 0, localTransportError(errAEADLimitReached)
+ }
+ return nil, 0, err
+ }
return pay, pnum, nil
}
+// aeadIntegrityLimit returns the integrity limit for an AEAD:
+// The maximum number of received packets that may fail authentication
+// before closing the connection.
+//
+// https://www.rfc-editor.org/rfc/rfc9001#section-6.6-4
+func aeadIntegrityLimit(suite uint16) int64 {
+ switch suite {
+ case tls.TLS_AES_128_GCM_SHA256, tls.TLS_AES_256_GCM_SHA384:
+ return 1 << 52
+ case tls.TLS_CHACHA20_POLY1305_SHA256:
+ return 1 << 36
+ default:
+ panic("BUG: unknown cipher suite")
+ }
+}
+
+// https://www.rfc-editor.org/rfc/rfc9001#section-5.2-2
+var initialSalt = []byte{0x38, 0x76, 0x2c, 0xf7, 0xf5, 0x59, 0x34, 0xb3, 0x4d, 0x17, 0x9a, 0xe6, 0xa4, 0xc8, 0x0c, 0xad, 0xcc, 0xbb, 0x7f, 0x0a}
+
+// initialKeys returns the keys used to protect Initial packets.
+//
+// The Initial packet keys are derived from the Destination Connection ID
+// field in the client's first Initial packet.
+//
+// https://www.rfc-editor.org/rfc/rfc9001#section-5.2
+func initialKeys(cid []byte, side connSide) fixedKeyPair {
+ initialSecret := hkdf.Extract(sha256.New, cid, initialSalt)
+ var clientKeys fixedKeys
+ clientSecret := hkdfExpandLabel(sha256.New, initialSecret, "client in", nil, sha256.Size)
+ clientKeys.init(tls.TLS_AES_128_GCM_SHA256, clientSecret)
+ var serverKeys fixedKeys
+ serverSecret := hkdfExpandLabel(sha256.New, initialSecret, "server in", nil, sha256.Size)
+ serverKeys.init(tls.TLS_AES_128_GCM_SHA256, serverSecret)
+ if side == clientSide {
+ return fixedKeyPair{r: serverKeys, w: clientKeys}
+ } else {
+ return fixedKeyPair{w: serverKeys, r: clientKeys}
+ }
+}
+
+// checkCipherSuite returns an error if suite is not a supported cipher suite.
+func checkCipherSuite(suite uint16) error {
+ switch suite {
+ case tls.TLS_AES_128_GCM_SHA256:
+ case tls.TLS_AES_256_GCM_SHA384:
+ case tls.TLS_CHACHA20_POLY1305_SHA256:
+ default:
+ return errors.New("invalid cipher suite")
+ }
+ return nil
+}
+
+func hashForSuite(suite uint16) (h crypto.Hash, keySize int) {
+ switch suite {
+ case tls.TLS_AES_128_GCM_SHA256:
+ return crypto.SHA256, 128 / 8
+ case tls.TLS_AES_256_GCM_SHA384:
+ return crypto.SHA384, 256 / 8
+ case tls.TLS_CHACHA20_POLY1305_SHA256:
+ return crypto.SHA256, chacha20.KeySize
+ default:
+ panic("BUG: unknown cipher suite")
+ }
+}
+
// hdkfExpandLabel implements HKDF-Expand-Label from RFC 8446, Section 7.1.
//
// Copied from crypto/tls/key_schedule.go.
diff --git a/internal/quic/packet_protection_test.go b/internal/quic/packet_protection_test.go
index 6495360a3b..1fe1307311 100644
--- a/internal/quic/packet_protection_test.go
+++ b/internal/quic/packet_protection_test.go
@@ -16,10 +16,11 @@ func TestPacketProtection(t *testing.T) {
// Test cases from:
// https://www.rfc-editor.org/rfc/rfc9001#section-appendix.a
cid := unhex(`8394c8f03e515708`)
- initialClientKeys, initialServerKeys := initialKeys(cid)
+ k := initialKeys(cid, clientSide)
+ initialClientKeys, initialServerKeys := k.w, k.r
for _, test := range []struct {
name string
- k keys
+ k fixedKeys
pnum packetNumber
hdr []byte
pay []byte
@@ -103,15 +104,13 @@ func TestPacketProtection(t *testing.T) {
`),
}, {
name: "ChaCha20_Poly1305 Short Header",
- k: func() keys {
+ k: func() fixedKeys {
secret := unhex(`
9ac312a7f877468ebe69422748ad00a1
5443f18203a07d6060f688f30f21632b
`)
- k, err := newKeys(tls.TLS_CHACHA20_POLY1305_SHA256, secret)
- if err != nil {
- t.Fatal(err)
- }
+ var k fixedKeys
+ k.init(tls.TLS_CHACHA20_POLY1305_SHA256, secret)
return k
}(),
pnum: 654360564,
diff --git a/internal/quic/packet_test.go b/internal/quic/packet_test.go
index b13a587e54..58c584e162 100644
--- a/internal/quic/packet_test.go
+++ b/internal/quic/packet_test.go
@@ -8,7 +8,9 @@ package quic
import (
"bytes"
+ "encoding/binary"
"encoding/hex"
+ "reflect"
"strings"
"testing"
)
@@ -112,6 +114,124 @@ func TestPacketHeader(t *testing.T) {
}
}
+func TestEncodeDecodeVersionNegotiation(t *testing.T) {
+ dstConnID := []byte("this is a very long destination connection id")
+ srcConnID := []byte("this is a very long source connection id")
+ versions := []uint32{1, 0xffffffff}
+ got := appendVersionNegotiation([]byte{}, dstConnID, srcConnID, versions...)
+ want := bytes.Join([][]byte{{
+ 0b1100_0000, // header byte
+ 0, 0, 0, 0, // Version
+ byte(len(dstConnID)),
+ }, dstConnID, {
+ byte(len(srcConnID)),
+ }, srcConnID, {
+ 0x00, 0x00, 0x00, 0x01,
+ 0xff, 0xff, 0xff, 0xff,
+ }}, nil)
+ if !bytes.Equal(got, want) {
+ t.Fatalf("appendVersionNegotiation(nil, %x, %x, %v):\ngot %x\nwant %x",
+ dstConnID, srcConnID, versions, got, want)
+ }
+ gotDst, gotSrc, gotVersionBytes := parseVersionNegotiation(got)
+ if got, want := gotDst, dstConnID; !bytes.Equal(got, want) {
+ t.Errorf("parseVersionNegotiation: got dstConnID = %x, want %x", got, want)
+ }
+ if got, want := gotSrc, srcConnID; !bytes.Equal(got, want) {
+ t.Errorf("parseVersionNegotiation: got srcConnID = %x, want %x", got, want)
+ }
+ var gotVersions []uint32
+ for len(gotVersionBytes) >= 4 {
+ gotVersions = append(gotVersions, binary.BigEndian.Uint32(gotVersionBytes))
+ gotVersionBytes = gotVersionBytes[4:]
+ }
+ if got, want := gotVersions, versions; !reflect.DeepEqual(got, want) {
+ t.Errorf("parseVersionNegotiation: got versions = %v, want %v", got, want)
+ }
+}
+
+func TestParseGenericLongHeaderPacket(t *testing.T) {
+ for _, test := range []struct {
+ name string
+ packet []byte
+ version uint32
+ dstConnID []byte
+ srcConnID []byte
+ data []byte
+ }{{
+ name: "long header packet",
+ packet: unhex(`
+ 80 01020304 04a1a2a3a4 05b1b2b3b4b5 c1
+ `),
+ version: 0x01020304,
+ dstConnID: unhex(`a1a2a3a4`),
+ srcConnID: unhex(`b1b2b3b4b5`),
+ data: unhex(`c1`),
+ }, {
+ name: "zero everything",
+ packet: unhex(`
+ 80 00000000 00 00
+ `),
+ version: 0,
+ dstConnID: []byte{},
+ srcConnID: []byte{},
+ data: []byte{},
+ }} {
+ t.Run(test.name, func(t *testing.T) {
+ p, ok := parseGenericLongHeaderPacket(test.packet)
+ if !ok {
+ t.Fatalf("parseGenericLongHeaderPacket() = _, false; want true")
+ }
+ if got, want := p.version, test.version; got != want {
+ t.Errorf("version = %v, want %v", got, want)
+ }
+ if got, want := p.dstConnID, test.dstConnID; !bytes.Equal(got, want) {
+ t.Errorf("Destination Connection ID = {%x}, want {%x}", got, want)
+ }
+ if got, want := p.srcConnID, test.srcConnID; !bytes.Equal(got, want) {
+ t.Errorf("Source Connection ID = {%x}, want {%x}", got, want)
+ }
+ if got, want := p.data, test.data; !bytes.Equal(got, want) {
+ t.Errorf("Data = {%x}, want {%x}", got, want)
+ }
+ })
+ }
+}
+
+func TestParseGenericLongHeaderPacketErrors(t *testing.T) {
+ for _, test := range []struct {
+ name string
+ packet []byte
+ }{{
+ name: "short header packet",
+ packet: unhex(`
+ 00 01020304 04a1a2a3a4 05b1b2b3b4b5 c1
+ `),
+ }, {
+ name: "packet too short",
+ packet: unhex(`
+ 80 000000
+ `),
+ }, {
+ name: "destination id too long",
+ packet: unhex(`
+ 80 00000000 02 00
+ `),
+ }, {
+ name: "source id too long",
+ packet: unhex(`
+ 80 00000000 00 01
+ `),
+ }} {
+ t.Run(test.name, func(t *testing.T) {
+ _, ok := parseGenericLongHeaderPacket(test.packet)
+ if ok {
+ t.Fatalf("parseGenericLongHeaderPacket() = _, true; want false")
+ }
+ })
+ }
+}
+
func unhex(s string) []byte {
b, err := hex.DecodeString(strings.Map(func(c rune) rune {
switch c {
diff --git a/internal/quic/packet_writer.go b/internal/quic/packet_writer.go
index 97987e0c2f..0c2b2ee41e 100644
--- a/internal/quic/packet_writer.go
+++ b/internal/quic/packet_writer.go
@@ -100,7 +100,7 @@ func (w *packetWriter) startProtectedLongHeaderPacket(pnumMaxAcked packetNumber,
// finishProtectedLongHeaderPacket finishes writing an Initial, 0-RTT, or Handshake packet,
// canceling the packet if it contains no payload.
// It returns a sentPacket describing the packet, or nil if no packet was written.
-func (w *packetWriter) finishProtectedLongHeaderPacket(pnumMaxAcked packetNumber, k keys, p longPacket) *sentPacket {
+func (w *packetWriter) finishProtectedLongHeaderPacket(pnumMaxAcked packetNumber, k fixedKeys, p longPacket) *sentPacket {
if len(w.b) == w.payOff {
// The payload is empty, so just abandon the packet.
w.b = w.b[:w.pktOff]
@@ -135,7 +135,8 @@ func (w *packetWriter) finishProtectedLongHeaderPacket(pnumMaxAcked packetNumber
pnumOff := len(hdr)
hdr = appendPacketNumber(hdr, p.num, pnumMaxAcked)
- return w.protect(hdr[w.pktOff:], p.num, pnumOff, k)
+ k.protect(hdr[w.pktOff:], w.b[len(hdr):], pnumOff-w.pktOff, p.num)
+ return w.finish(p.num)
}
// start1RTTPacket starts writing a 1-RTT (short header) packet.
@@ -162,14 +163,13 @@ func (w *packetWriter) start1RTTPacket(pnum, pnumMaxAcked packetNumber, dstConnI
// finish1RTTPacket finishes writing a 1-RTT packet,
// canceling the packet if it contains no payload.
// It returns a sentPacket describing the packet, or nil if no packet was written.
-func (w *packetWriter) finish1RTTPacket(pnum, pnumMaxAcked packetNumber, dstConnID []byte, k keys) *sentPacket {
+func (w *packetWriter) finish1RTTPacket(pnum, pnumMaxAcked packetNumber, dstConnID []byte, k *updatingKeyPair) *sentPacket {
if len(w.b) == w.payOff {
// The payload is empty, so just abandon the packet.
w.b = w.b[:w.pktOff]
return nil
}
// TODO: Spin
- // TODO: Key phase
pnumLen := packetNumberLength(pnum, pnumMaxAcked)
hdr := w.b[:w.pktOff]
hdr = append(hdr, 0x40|byte(pnumLen-1))
@@ -177,7 +177,8 @@ func (w *packetWriter) finish1RTTPacket(pnum, pnumMaxAcked packetNumber, dstConn
pnumOff := len(hdr)
hdr = appendPacketNumber(hdr, pnum, pnumMaxAcked)
w.padPacketLength(pnumLen)
- return w.protect(hdr[w.pktOff:], pnum, pnumOff, k)
+ k.protect(hdr[w.pktOff:], w.b[len(hdr):], pnumOff-w.pktOff, pnum)
+ return w.finish(pnum)
}
// padPacketLength pads out the payload of the current packet to the minimum size,
@@ -197,9 +198,8 @@ func (w *packetWriter) padPacketLength(pnumLen int) int {
return plen
}
-// protect applies packet protection and finishes the current packet.
-func (w *packetWriter) protect(hdr []byte, pnum packetNumber, pnumOff int, k keys) *sentPacket {
- k.protect(hdr, w.b[w.pktOff+len(hdr):], pnumOff-w.pktOff, pnum)
+// finish finishes the current packet after protection is applied.
+func (w *packetWriter) finish(pnum packetNumber) *sentPacket {
w.b = w.b[:len(w.b)+aeadOverhead]
w.sent.size = len(w.b) - w.pktOff
w.sent.num = pnum
@@ -237,7 +237,10 @@ func (w *packetWriter) appendPingFrame() (added bool) {
return false
}
w.b = append(w.b, frameTypePing)
- w.sent.appendAckElicitingFrame(frameTypePing)
+ // Mark this packet as ack-eliciting and in-flight,
+ // but there's no need to record the presence of a PING frame in it.
+ w.sent.ackEliciting = true
+ w.sent.inFlight = true
return true
}
@@ -479,13 +482,14 @@ func (w *packetWriter) appendNewConnectionIDFrame(seq, retirePriorTo int64, conn
return true
}
-func (w *packetWriter) appendRetireConnectionIDFrame(seq uint64) (added bool) {
- if w.avail() < 1+sizeVarint(seq) {
+func (w *packetWriter) appendRetireConnectionIDFrame(seq int64) (added bool) {
+ if w.avail() < 1+sizeVarint(uint64(seq)) {
return false
}
w.b = append(w.b, frameTypeRetireConnectionID)
- w.b = appendVarint(w.b, seq)
+ w.b = appendVarint(w.b, uint64(seq))
w.sent.appendAckElicitingFrame(frameTypeRetireConnectionID)
+ w.sent.appendInt(uint64(seq))
return true
}
diff --git a/internal/quic/ping.go b/internal/quic/ping.go
new file mode 100644
index 0000000000..3e7d9c51bd
--- /dev/null
+++ b/internal/quic/ping.go
@@ -0,0 +1,16 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build go1.21
+
+package quic
+
+import "time"
+
+func (c *Conn) ping(space numberSpace) {
+ c.sendMsg(func(now time.Time, c *Conn) {
+ c.testSendPing.setUnsent()
+ c.testSendPingSpace = space
+ })
+}
diff --git a/internal/quic/ping_test.go b/internal/quic/ping_test.go
new file mode 100644
index 0000000000..a8fdf2567e
--- /dev/null
+++ b/internal/quic/ping_test.go
@@ -0,0 +1,43 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build go1.21
+
+package quic
+
+import "testing"
+
+func TestPing(t *testing.T) {
+ tc := newTestConn(t, clientSide)
+ tc.handshake()
+
+ tc.conn.ping(appDataSpace)
+ tc.wantFrame("connection should send a PING frame",
+ packetType1RTT, debugFramePing{})
+
+ tc.advanceToTimer()
+ tc.wantFrame("on PTO, connection should send another PING frame",
+ packetType1RTT, debugFramePing{})
+
+ tc.wantIdle("after sending PTO probe, no additional frames to send")
+}
+
+func TestAck(t *testing.T) {
+ tc := newTestConn(t, serverSide)
+ tc.handshake()
+
+ // Send two packets, to trigger an immediate ACK.
+ tc.writeFrames(packetType1RTT,
+ debugFramePing{},
+ )
+ tc.writeFrames(packetType1RTT,
+ debugFramePing{},
+ )
+ tc.wantFrame("connection should respond to ack-eliciting packet with an ACK frame",
+ packetType1RTT,
+ debugFrameAck{
+ ranges: []i64range[packetNumber]{{0, 4}},
+ },
+ )
+}
diff --git a/internal/quic/pipe.go b/internal/quic/pipe.go
new file mode 100644
index 0000000000..978a4f3d8b
--- /dev/null
+++ b/internal/quic/pipe.go
@@ -0,0 +1,149 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build go1.21
+
+package quic
+
+import (
+ "sync"
+)
+
+// A pipe is a byte buffer used in implementing streams.
+//
+// A pipe contains a window of stream data.
+// Random access reads and writes are supported within the window.
+// Writing past the end of the window extends it.
+// Data may be discarded from the start of the pipe, advancing the window.
+type pipe struct {
+ start int64
+ end int64
+ head *pipebuf
+ tail *pipebuf
+}
+
+type pipebuf struct {
+ off int64
+ b []byte
+ next *pipebuf
+}
+
+func (pb *pipebuf) end() int64 {
+ return pb.off + int64(len(pb.b))
+}
+
+var pipebufPool = sync.Pool{
+ New: func() any {
+ return &pipebuf{
+ b: make([]byte, 4096),
+ }
+ },
+}
+
+func newPipebuf() *pipebuf {
+ return pipebufPool.Get().(*pipebuf)
+}
+
+func (b *pipebuf) recycle() {
+ b.off = 0
+ b.next = nil
+ pipebufPool.Put(b)
+}
+
+// writeAt writes len(b) bytes to the pipe at offset off.
+//
+// Writes to offsets before p.start are discarded.
+// Writes to offsets after p.end extend the pipe window.
+func (p *pipe) writeAt(b []byte, off int64) {
+ end := off + int64(len(b))
+ if end > p.end {
+ p.end = end
+ } else if end <= p.start {
+ return
+ }
+
+ if off < p.start {
+ // Discard the portion of b which falls before p.start.
+ trim := p.start - off
+ b = b[trim:]
+ off = p.start
+ }
+
+ if p.head == nil {
+ p.head = newPipebuf()
+ p.head.off = p.start
+ p.tail = p.head
+ }
+ pb := p.head
+ if off >= p.tail.off {
+ // Common case: Writing past the end of the pipe.
+ pb = p.tail
+ }
+ for {
+ pboff := off - pb.off
+ if pboff < int64(len(pb.b)) {
+ n := copy(pb.b[pboff:], b)
+ if n == len(b) {
+ return
+ }
+ off += int64(n)
+ b = b[n:]
+ }
+ if pb.next == nil {
+ pb.next = newPipebuf()
+ pb.next.off = pb.off + int64(len(pb.b))
+ p.tail = pb.next
+ }
+ pb = pb.next
+ }
+}
+
+// copy copies len(b) bytes into b starting from off.
+// The pipe must contain [off, off+len(b)).
+func (p *pipe) copy(off int64, b []byte) {
+ dst := b[:0]
+ p.read(off, len(b), func(c []byte) error {
+ dst = append(dst, c...)
+ return nil
+ })
+}
+
+// read calls f with the data in [off, off+n)
+// The data may be provided sequentially across multiple calls to f.
+func (p *pipe) read(off int64, n int, f func([]byte) error) error {
+ if off < p.start {
+ panic("invalid read range")
+ }
+ for pb := p.head; pb != nil && n > 0; pb = pb.next {
+ if off >= pb.end() {
+ continue
+ }
+ b := pb.b[off-pb.off:]
+ if len(b) > n {
+ b = b[:n]
+ }
+ off += int64(len(b))
+ n -= len(b)
+ if err := f(b); err != nil {
+ return err
+ }
+ }
+ if n > 0 {
+ panic("invalid read range")
+ }
+ return nil
+}
+
+// discardBefore discards all data prior to off.
+func (p *pipe) discardBefore(off int64) {
+ for p.head != nil && p.head.end() < off {
+ head := p.head
+ p.head = p.head.next
+ head.recycle()
+ }
+ if p.head == nil {
+ p.tail = nil
+ }
+ p.start = off
+}
diff --git a/internal/quic/pipe_test.go b/internal/quic/pipe_test.go
new file mode 100644
index 0000000000..7a05ff4d47
--- /dev/null
+++ b/internal/quic/pipe_test.go
@@ -0,0 +1,95 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build go1.21
+
+package quic
+
+import (
+ "bytes"
+ "math/rand"
+ "testing"
+)
+
+func TestPipeWrites(t *testing.T) {
+ type writeOp struct {
+ start, end int64
+ }
+ type discardBeforeOp struct {
+ off int64
+ }
+ type op any
+ src := make([]byte, 65536)
+ rand.New(rand.NewSource(0)).Read(src)
+ for _, test := range []struct {
+ desc string
+ ops []op
+ }{{
+ desc: "sequential writes",
+ ops: []op{
+ writeOp{0, 1024},
+ writeOp{1024, 4096},
+ writeOp{4096, 65536},
+ },
+ }, {
+ desc: "disordered overlapping writes",
+ ops: []op{
+ writeOp{2000, 8000},
+ writeOp{0, 3000},
+ writeOp{7000, 12000},
+ },
+ }, {
+ desc: "write to discarded region",
+ ops: []op{
+ writeOp{0, 65536},
+ discardBeforeOp{32768},
+ writeOp{0, 1000},
+ writeOp{3000, 5000},
+ writeOp{0, 32768},
+ },
+ }, {
+ desc: "write overlaps discarded region",
+ ops: []op{
+ discardBeforeOp{10000},
+ writeOp{0, 20000},
+ },
+ }, {
+ desc: "discard everything",
+ ops: []op{
+ writeOp{0, 10000},
+ discardBeforeOp{10000},
+ writeOp{10000, 20000},
+ },
+ }} {
+ var p pipe
+ var wantset rangeset[int64]
+ var wantStart, wantEnd int64
+ for i, o := range test.ops {
+ switch o := o.(type) {
+ case writeOp:
+ p.writeAt(src[o.start:o.end], o.start)
+ wantset.add(o.start, o.end)
+ wantset.sub(0, wantStart)
+ if o.end > wantEnd {
+ wantEnd = o.end
+ }
+ case discardBeforeOp:
+ p.discardBefore(o.off)
+ wantset.sub(0, o.off)
+ wantStart = o.off
+ }
+ if p.start != wantStart || p.end != wantEnd {
+ t.Errorf("%v: after %#v p contains [%v,%v), want [%v,%v)", test.desc, test.ops[:i+1], p.start, p.end, wantStart, wantEnd)
+ }
+ for _, r := range wantset {
+ want := src[r.start:][:r.size()]
+ got := make([]byte, r.size())
+ p.copy(r.start, got)
+ if !bytes.Equal(got, want) {
+ t.Errorf("%v after %#v, mismatch in data in %v", test.desc, test.ops[:i+1], r)
+ }
+ }
+ }
+ }
+}
diff --git a/internal/quic/queue.go b/internal/quic/queue.go
new file mode 100644
index 0000000000..7085e578b6
--- /dev/null
+++ b/internal/quic/queue.go
@@ -0,0 +1,65 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build go1.21
+
+package quic
+
+import "context"
+
+// A queue is an unbounded queue of some item (new connections and streams).
+type queue[T any] struct {
+ // The gate condition is set if the queue is non-empty or closed.
+ gate gate
+ err error
+ q []T
+}
+
+func newQueue[T any]() queue[T] {
+ return queue[T]{gate: newGate()}
+}
+
+// close closes the queue, causing pending and future pop operations
+// to return immediately with err.
+func (q *queue[T]) close(err error) {
+ q.gate.lock()
+ defer q.unlock()
+ if q.err == nil {
+ q.err = err
+ }
+}
+
+// put appends an item to the queue.
+// It returns true if the item was added, false if the queue is closed.
+func (q *queue[T]) put(v T) bool {
+ q.gate.lock()
+ defer q.unlock()
+ if q.err != nil {
+ return false
+ }
+ q.q = append(q.q, v)
+ return true
+}
+
+// get removes the first item from the queue, blocking until ctx is done, an item is available,
+// or the queue is closed.
+func (q *queue[T]) get(ctx context.Context, testHooks connTestHooks) (T, error) {
+ var zero T
+ if err := q.gate.waitAndLock(ctx, testHooks); err != nil {
+ return zero, err
+ }
+ defer q.unlock()
+ if q.err != nil {
+ return zero, q.err
+ }
+ v := q.q[0]
+ copy(q.q[:], q.q[1:])
+ q.q[len(q.q)-1] = zero
+ q.q = q.q[:len(q.q)-1]
+ return v, nil
+}
+
+func (q *queue[T]) unlock() {
+ q.gate.unlock(q.err != nil || len(q.q) > 0)
+}
diff --git a/internal/quic/queue_test.go b/internal/quic/queue_test.go
new file mode 100644
index 0000000000..d78216b0ec
--- /dev/null
+++ b/internal/quic/queue_test.go
@@ -0,0 +1,59 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build go1.21
+
+package quic
+
+import (
+ "context"
+ "io"
+ "testing"
+ "time"
+)
+
+func TestQueue(t *testing.T) {
+ nonblocking, cancel := context.WithCancel(context.Background())
+ cancel()
+
+ q := newQueue[int]()
+ if got, err := q.get(nonblocking, nil); err != context.Canceled {
+ t.Fatalf("q.get() = %v, %v, want nil, contex.Canceled", got, err)
+ }
+
+ if !q.put(1) {
+ t.Fatalf("q.put(1) = false, want true")
+ }
+ if !q.put(2) {
+ t.Fatalf("q.put(2) = false, want true")
+ }
+ if got, err := q.get(nonblocking, nil); got != 1 || err != nil {
+ t.Fatalf("q.get() = %v, %v, want 1, nil", got, err)
+ }
+ if got, err := q.get(nonblocking, nil); got != 2 || err != nil {
+ t.Fatalf("q.get() = %v, %v, want 2, nil", got, err)
+ }
+ if got, err := q.get(nonblocking, nil); err != context.Canceled {
+ t.Fatalf("q.get() = %v, %v, want nil, contex.Canceled", got, err)
+ }
+
+ go func() {
+ time.Sleep(1 * time.Millisecond)
+ q.put(3)
+ }()
+ if got, err := q.get(context.Background(), nil); got != 3 || err != nil {
+ t.Fatalf("q.get() = %v, %v, want 3, nil", got, err)
+ }
+
+ if !q.put(4) {
+ t.Fatalf("q.put(2) = false, want true")
+ }
+ q.close(io.EOF)
+ if got, err := q.get(context.Background(), nil); got != 0 || err != io.EOF {
+ t.Fatalf("q.get() = %v, %v, want 0, io.EOF", got, err)
+ }
+ if q.put(5) {
+ t.Fatalf("q.put(5) = true, want false")
+ }
+}
diff --git a/internal/quic/quic.go b/internal/quic/quic.go
index 982c6751b7..9de97b6d88 100644
--- a/internal/quic/quic.go
+++ b/internal/quic/quic.go
@@ -10,6 +10,13 @@ import (
"time"
)
+// QUIC versions.
+// We only support v1 at this time.
+const (
+ quicVersion1 = 1
+ quicVersion2 = 0x6b3343cf // https://www.rfc-editor.org/rfc/rfc9369
+)
+
// connIDLen is the length in bytes of connection IDs chosen by this package.
// Since 1-RTT packets don't include a connection ID length field,
// we use a consistent length for all our IDs.
@@ -19,6 +26,8 @@ const connIDLen = 8
// Local values of various transport parameters.
// https://www.rfc-editor.org/rfc/rfc9000.html#section-18.2
const (
+ defaultMaxIdleTimeout = 30 * time.Second // max_idle_timeout
+
// The max_udp_payload_size transport parameter is the size of our
// network receive buffer.
//
@@ -33,12 +42,37 @@ const (
ackDelayExponent = 3 // ack_delay_exponent
maxAckDelay = 25 * time.Millisecond // max_ack_delay
+
+ // The active_conn_id_limit transport parameter is the maximum
+ // number of connection IDs from the peer we're willing to store.
+ //
+ // maxPeerActiveConnIDLimit is the maximum number of connection IDs
+ // we're willing to send to the peer.
+ //
+ // https://www.rfc-editor.org/rfc/rfc9000.html#section-18.2-6.2.1
+ activeConnIDLimit = 2
+ maxPeerActiveConnIDLimit = 4
)
// Local timer granularity.
// https://www.rfc-editor.org/rfc/rfc9002.html#section-6.1.2-6
const timerGranularity = 1 * time.Millisecond
+// Minimum size of a UDP datagram sent by a client carrying an Initial packet.
+// https://www.rfc-editor.org/rfc/rfc9000#section-14.1
+const minimumClientInitialDatagramSize = 1200
+
+// Maximum number of streams of a given type which may be created.
+// https://www.rfc-editor.org/rfc/rfc9000.html#section-4.6-2
+const maxStreamsLimit = 1 << 60
+
+// Maximum number of streams we will allow the peer to create implicitly.
+// A stream ID that is used out of order results in all streams of that type
+// with lower-numbered IDs also being opened. To limit the amount of work we
+// will do in response to a single frame, we cap the peer's stream limit to
+// this value.
+const implicitStreamLimit = 100
+
// A connSide distinguishes between the client and server sides of a connection.
type connSide int8
@@ -58,6 +92,14 @@ func (s connSide) String() string {
}
}
+func (s connSide) peer() connSide {
+ if s == clientSide {
+ return serverSide
+ } else {
+ return clientSide
+ }
+}
+
// A numberSpace is the context in which a packet number applies.
// https://www.rfc-editor.org/rfc/rfc9000.html#section-12.3-7
type numberSpace byte
@@ -88,6 +130,7 @@ type streamType uint8
const (
bidiStream = streamType(iota)
uniStream
+ streamTypeCount
)
func (s streamType) String() string {
diff --git a/internal/quic/quic_test.go b/internal/quic/quic_test.go
new file mode 100644
index 0000000000..1281b54eec
--- /dev/null
+++ b/internal/quic/quic_test.go
@@ -0,0 +1,37 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build go1.21
+
+package quic
+
+import (
+ "testing"
+)
+
+func testSides(t *testing.T, name string, f func(*testing.T, connSide)) {
+ if name != "" {
+ name += "/"
+ }
+ t.Run(name+"server", func(t *testing.T) { f(t, serverSide) })
+ t.Run(name+"client", func(t *testing.T) { f(t, clientSide) })
+}
+
+func testStreamTypes(t *testing.T, name string, f func(*testing.T, streamType)) {
+ if name != "" {
+ name += "/"
+ }
+ t.Run(name+"bidi", func(t *testing.T) { f(t, bidiStream) })
+ t.Run(name+"uni", func(t *testing.T) { f(t, uniStream) })
+}
+
+func testSidesAndStreamTypes(t *testing.T, name string, f func(*testing.T, connSide, streamType)) {
+ if name != "" {
+ name += "/"
+ }
+ t.Run(name+"server/bidi", func(t *testing.T) { f(t, serverSide, bidiStream) })
+ t.Run(name+"client/bidi", func(t *testing.T) { f(t, clientSide, bidiStream) })
+ t.Run(name+"server/uni", func(t *testing.T) { f(t, serverSide, uniStream) })
+ t.Run(name+"client/uni", func(t *testing.T) { f(t, clientSide, uniStream) })
+}
diff --git a/internal/quic/rangeset.go b/internal/quic/rangeset.go
index 5339c5ac51..4966a99d2c 100644
--- a/internal/quic/rangeset.go
+++ b/internal/quic/rangeset.go
@@ -154,6 +154,11 @@ func (s rangeset[T]) end() T {
return s[len(s)-1].end
}
+// numRanges returns the number of ranges in the rangeset.
+func (s rangeset[T]) numRanges() int {
+ return len(s)
+}
+
// isrange reports if the rangeset covers exactly the range [start, end).
func (s rangeset[T]) isrange(start, end T) bool {
switch len(s) {
diff --git a/internal/quic/rangeset_test.go b/internal/quic/rangeset_test.go
index 308046905a..2027f14b88 100644
--- a/internal/quic/rangeset_test.go
+++ b/internal/quic/rangeset_test.go
@@ -295,3 +295,23 @@ func TestRangesetIsRange(t *testing.T) {
}
}
}
+
+func TestRangesetNumRanges(t *testing.T) {
+ for _, test := range []struct {
+ s rangeset[int64]
+ want int
+ }{{
+ s: rangeset[int64]{},
+ want: 0,
+ }, {
+ s: rangeset[int64]{{0, 100}},
+ want: 1,
+ }, {
+ s: rangeset[int64]{{0, 100}, {200, 300}},
+ want: 2,
+ }} {
+ if got, want := test.s.numRanges(), test.want; got != want {
+ t.Errorf("%+v.numRanges() = %v, want %v", test.s, got, want)
+ }
+ }
+}
diff --git a/internal/quic/sent_packet.go b/internal/quic/sent_packet.go
index e5a80be3bb..4f11aa1368 100644
--- a/internal/quic/sent_packet.go
+++ b/internal/quic/sent_packet.go
@@ -29,6 +29,8 @@ type sentPacket struct {
// we need to process an ack for or loss of this packet.
// For example, a CRYPTO frame is recorded as the frame type (0x06), offset, and length,
// but does not include the sent data.
+ //
+ // This buffer is written by packetWriter.append* and read by Conn.handleAckOrLoss.
b []byte
n int // read offset into b
}
diff --git a/internal/quic/sent_val.go b/internal/quic/sent_val.go
index b33d8b00f2..31f69e47d0 100644
--- a/internal/quic/sent_val.go
+++ b/internal/quic/sent_val.go
@@ -37,7 +37,7 @@ func (s sentVal) isSet() bool { return s != 0 }
// shouldSend reports whether the value is set and has not been sent to the peer.
func (s sentVal) shouldSend() bool { return s.state() == sentValUnsent }
-// shouldSend reports whether the the value needs to be sent to the peer.
+// shouldSend reports whether the value needs to be sent to the peer.
// The value needs to be sent if it is set and has not been sent.
// If pto is true, indicating that we are sending a PTO probe, the value
// should also be sent if it is set and has not been acknowledged.
diff --git a/internal/quic/stream.go b/internal/quic/stream.go
new file mode 100644
index 0000000000..89036b19b6
--- /dev/null
+++ b/internal/quic/stream.go
@@ -0,0 +1,801 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build go1.21
+
+package quic
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "io"
+)
+
+type Stream struct {
+ id streamID
+ conn *Conn
+
+ // ingate's lock guards all receive-related state.
+ //
+ // The gate condition is set if a read from the stream will not block,
+ // either because the stream has available data or because the read will fail.
+ ingate gate
+ in pipe // received data
+ inwin int64 // last MAX_STREAM_DATA sent to the peer
+ insendmax sentVal // set when we should send MAX_STREAM_DATA to the peer
+ inmaxbuf int64 // maximum amount of data we will buffer
+ insize int64 // stream final size; -1 before this is known
+ inset rangeset[int64] // received ranges
+ inclosed sentVal // set by CloseRead
+ inresetcode int64 // RESET_STREAM code received from the peer; -1 if not reset
+
+ // outgate's lock guards all send-related state.
+ //
+ // The gate condition is set if a write to the stream will not block,
+ // either because the stream has available flow control or because
+ // the write will fail.
+ outgate gate
+ out pipe // buffered data to send
+ outwin int64 // maximum MAX_STREAM_DATA received from the peer
+ outmaxsent int64 // maximum data offset we've sent to the peer
+ outmaxbuf int64 // maximum amount of data we will buffer
+ outunsent rangeset[int64] // ranges buffered but not yet sent
+ outacked rangeset[int64] // ranges sent and acknowledged
+ outopened sentVal // set if we should open the stream
+ outclosed sentVal // set by CloseWrite
+ outblocked sentVal // set when a write to the stream is blocked by flow control
+ outreset sentVal // set by Reset
+ outresetcode uint64 // reset code to send in RESET_STREAM
+ outdone chan struct{} // closed when all data sent
+
+ // Atomic stream state bits.
+ //
+ // These bits provide a fast way to coordinate between the
+ // send and receive sides of the stream, and the conn's loop.
+ //
+ // streamIn* bits must be set with ingate held.
+ // streamOut* bits must be set with outgate held.
+ // streamConn* bits are set by the conn's loop.
+ // streamQueue* bits must be set with streamsState.sendMu held.
+ state atomicBits[streamState]
+
+ prev, next *Stream // guarded by streamsState.sendMu
+}
+
+type streamState uint32
+
+const (
+ // streamInSendMeta is set when there are frames to send for the
+ // inbound side of the stream. For example, MAX_STREAM_DATA.
+ // Inbound frames are never flow-controlled.
+ streamInSendMeta = streamState(1 << iota)
+
+ // streamOutSendMeta is set when there are non-flow-controlled frames
+ // to send for the outbound side of the stream. For example, STREAM_DATA_BLOCKED.
+ // streamOutSendData is set when there are no non-flow-controlled outbound frames
+ // and the stream has data to send.
+ //
+ // At most one of streamOutSendMeta and streamOutSendData is set at any time.
+ streamOutSendMeta
+ streamOutSendData
+
+ // streamInDone and streamOutDone are set when the inbound or outbound
+ // sides of the stream are finished. When both are set, the stream
+ // can be removed from the Conn and forgotten.
+ streamInDone
+ streamOutDone
+
+ // streamConnRemoved is set when the stream has been removed from the conn.
+ streamConnRemoved
+
+ // streamQueueMeta and streamQueueData indicate which of the streamsState
+ // send queues the conn is currently on.
+ streamQueueMeta
+ streamQueueData
+)
+
+type streamQueue int
+
+const (
+ noQueue = streamQueue(iota)
+ metaQueue // streamsState.queueMeta
+ dataQueue // streamsState.queueData
+)
+
+// wantQueue returns the send queue the stream should be on.
+func (s streamState) wantQueue() streamQueue {
+ switch {
+ case s&(streamInSendMeta|streamOutSendMeta) != 0:
+ return metaQueue
+ case s&(streamInDone|streamOutDone|streamConnRemoved) == streamInDone|streamOutDone:
+ return metaQueue
+ case s&streamOutSendData != 0:
+ // The stream has no non-flow-controlled frames to send,
+ // but does have data. Put it on the data queue, which is only
+ // processed when flow control is available.
+ return dataQueue
+ }
+ return noQueue
+}
+
+// inQueue returns the send queue the stream is currently on.
+func (s streamState) inQueue() streamQueue {
+ switch {
+ case s&streamQueueMeta != 0:
+ return metaQueue
+ case s&streamQueueData != 0:
+ return dataQueue
+ }
+ return noQueue
+}
+
+// newStream returns a new stream.
+//
+// The stream's ingate and outgate are locked.
+// (We create the stream with locked gates so after the caller
+// initializes the flow control window,
+// unlocking outgate will set the stream writability state.)
+func newStream(c *Conn, id streamID) *Stream {
+ s := &Stream{
+ conn: c,
+ id: id,
+ insize: -1, // -1 indicates the stream size is unknown
+ inresetcode: -1, // -1 indicates no RESET_STREAM received
+ ingate: newLockedGate(),
+ outgate: newLockedGate(),
+ }
+ if !s.IsReadOnly() {
+ s.outdone = make(chan struct{})
+ }
+ return s
+}
+
+// IsReadOnly reports whether the stream is read-only
+// (a unidirectional stream created by the peer).
+func (s *Stream) IsReadOnly() bool {
+ return s.id.streamType() == uniStream && s.id.initiator() != s.conn.side
+}
+
+// IsWriteOnly reports whether the stream is write-only
+// (a unidirectional stream created locally).
+func (s *Stream) IsWriteOnly() bool {
+ return s.id.streamType() == uniStream && s.id.initiator() == s.conn.side
+}
+
+// Read reads data from the stream.
+// See ReadContext for more details.
+func (s *Stream) Read(b []byte) (n int, err error) {
+ return s.ReadContext(context.Background(), b)
+}
+
+// ReadContext reads data from the stream.
+//
+// ReadContext returns as soon as at least one byte of data is available.
+//
+// If the peer closes the stream cleanly, ReadContext returns io.EOF after
+// returning all data sent by the peer.
+// If the peer aborts reads on the stream, ReadContext returns
+// an error wrapping StreamResetCode.
+func (s *Stream) ReadContext(ctx context.Context, b []byte) (n int, err error) {
+ if s.IsWriteOnly() {
+ return 0, errors.New("read from write-only stream")
+ }
+ if err := s.ingate.waitAndLock(ctx, s.conn.testHooks); err != nil {
+ return 0, err
+ }
+ defer func() {
+ s.inUnlock()
+ s.conn.handleStreamBytesReadOffLoop(int64(n)) // must be done with ingate unlocked
+ }()
+ if s.inresetcode != -1 {
+ return 0, fmt.Errorf("stream reset by peer: %w", StreamErrorCode(s.inresetcode))
+ }
+ if s.inclosed.isSet() {
+ return 0, errors.New("read from closed stream")
+ }
+ if s.insize == s.in.start {
+ return 0, io.EOF
+ }
+ // Getting here indicates the stream contains data to be read.
+ if len(s.inset) < 1 || s.inset[0].start != 0 || s.inset[0].end <= s.in.start {
+ panic("BUG: inconsistent input stream state")
+ }
+ if size := int(s.inset[0].end - s.in.start); size < len(b) {
+ b = b[:size]
+ }
+ start := s.in.start
+ end := start + int64(len(b))
+ s.in.copy(start, b)
+ s.in.discardBefore(end)
+ if s.insize == -1 || s.insize > s.inwin {
+ if shouldUpdateFlowControl(s.inmaxbuf, s.in.start+s.inmaxbuf-s.inwin) {
+ // Update stream flow control with a STREAM_MAX_DATA frame.
+ s.insendmax.setUnsent()
+ }
+ }
+ if end == s.insize {
+ return len(b), io.EOF
+ }
+ return len(b), nil
+}
+
+// shouldUpdateFlowControl determines whether to send a flow control window update.
+//
+// We want to balance keeping the peer well-supplied with flow control with not sending
+// many small updates.
+func shouldUpdateFlowControl(maxWindow, addedWindow int64) bool {
+ return addedWindow >= maxWindow/8
+}
+
+// Write writes data to the stream.
+// See WriteContext for more details.
+func (s *Stream) Write(b []byte) (n int, err error) {
+ return s.WriteContext(context.Background(), b)
+}
+
+// WriteContext writes data to the stream.
+//
+// WriteContext writes data to the stream write buffer.
+// Buffered data is only sent when the buffer is sufficiently full.
+// Call the Flush method to ensure buffered data is sent.
+//
+// TODO: Implement Flush.
+func (s *Stream) WriteContext(ctx context.Context, b []byte) (n int, err error) {
+ if s.IsReadOnly() {
+ return 0, errors.New("write to read-only stream")
+ }
+ canWrite := s.outgate.lock()
+ for {
+ // The first time through this loop, we may or may not be write blocked.
+ // We exit the loop after writing all data, so on subsequent passes through
+ // the loop we are always write blocked.
+ if len(b) > 0 && !canWrite {
+ // Our send buffer is full. Wait for the peer to ack some data.
+ s.outUnlock()
+ if err := s.outgate.waitAndLock(ctx, s.conn.testHooks); err != nil {
+ return n, err
+ }
+ // Successfully returning from waitAndLockGate means we are no longer
+ // write blocked. (Unlike traditional condition variables, gates do not
+ // have spurious wakeups.)
+ }
+ if s.outreset.isSet() {
+ s.outUnlock()
+ return n, errors.New("write to reset stream")
+ }
+ if s.outclosed.isSet() {
+ s.outUnlock()
+ return n, errors.New("write to closed stream")
+ }
+ // We set outopened here rather than below,
+ // so if this is a zero-length write we still
+ // open the stream despite not writing any data to it.
+ s.outopened.set()
+ if len(b) == 0 {
+ break
+ }
+ // Write limit is our send buffer limit.
+ // This is a stream offset.
+ lim := s.out.start + s.outmaxbuf
+ // Amount to write is min(the full buffer, data up to the write limit).
+ // This is a number of bytes.
+ nn := min(int64(len(b)), lim-s.out.end)
+ // Copy the data into the output buffer and mark it as unsent.
+ if s.out.end <= s.outwin {
+ s.outunsent.add(s.out.end, min(s.out.end+nn, s.outwin))
+ }
+ s.out.writeAt(b[:nn], s.out.end)
+ b = b[nn:]
+ n += int(nn)
+ if s.out.end > s.outwin {
+ // We're blocked by flow control.
+ // Send a STREAM_DATA_BLOCKED frame to let the peer know.
+ s.outblocked.set()
+ }
+ // If we have bytes left to send, we're blocked.
+ canWrite = false
+ }
+ s.outUnlock()
+ return n, nil
+}
+
+// Close closes the stream.
+// See CloseContext for more details.
+func (s *Stream) Close() error {
+ return s.CloseContext(context.Background())
+}
+
+// CloseContext closes the stream.
+// Any blocked stream operations will be unblocked and return errors.
+//
+// CloseContext flushes any data in the stream write buffer and waits for the peer to
+// acknowledge receipt of the data.
+// If the stream has been reset, it waits for the peer to acknowledge the reset.
+// If the context expires before the peer receives the stream's data,
+// CloseContext discards the buffer and returns the context error.
+func (s *Stream) CloseContext(ctx context.Context) error {
+ s.CloseRead()
+ if s.IsReadOnly() {
+ return nil
+ }
+ s.CloseWrite()
+ // TODO: Return code from peer's RESET_STREAM frame?
+ return s.conn.waitOnDone(ctx, s.outdone)
+}
+
+// CloseRead aborts reads on the stream.
+// Any blocked reads will be unblocked and return errors.
+//
+// CloseRead notifies the peer that the stream has been closed for reading.
+// It does not wait for the peer to acknowledge the closure.
+// Use CloseContext to wait for the peer's acknowledgement.
+func (s *Stream) CloseRead() {
+ if s.IsWriteOnly() {
+ return
+ }
+ s.ingate.lock()
+ if s.inset.isrange(0, s.insize) || s.inresetcode != -1 {
+ // We've already received all data from the peer,
+ // so there's no need to send STOP_SENDING.
+ // This is the same as saying we sent one and they got it.
+ s.inclosed.setReceived()
+ } else {
+ s.inclosed.set()
+ }
+ discarded := s.in.end - s.in.start
+ s.in.discardBefore(s.in.end)
+ s.inUnlock()
+ s.conn.handleStreamBytesReadOffLoop(discarded) // must be done with ingate unlocked
+}
+
+// CloseWrite aborts writes on the stream.
+// Any blocked writes will be unblocked and return errors.
+//
+// CloseWrite sends any data in the stream write buffer to the peer.
+// It does not wait for the peer to acknowledge receipt of the data.
+// Use CloseContext to wait for the peer's acknowledgement.
+func (s *Stream) CloseWrite() {
+ if s.IsReadOnly() {
+ return
+ }
+ s.outgate.lock()
+ defer s.outUnlock()
+ s.outclosed.set()
+}
+
+// Reset aborts writes on the stream and notifies the peer
+// that the stream was terminated abruptly.
+// Any blocked writes will be unblocked and return errors.
+//
+// Reset sends the application protocol error code, which must be
+// less than 2^62, to the peer.
+// It does not wait for the peer to acknowledge receipt of the error.
+// Use CloseContext to wait for the peer's acknowledgement.
+//
+// Reset does not affect reads.
+// Use CloseRead to abort reads on the stream.
+func (s *Stream) Reset(code uint64) {
+ const userClosed = true
+ s.resetInternal(code, userClosed)
+}
+
+// resetInternal resets the send side of the stream.
+//
+// If userClosed is true, this is s.Reset.
+// If userClosed is false, this is a reaction to a STOP_SENDING frame.
+func (s *Stream) resetInternal(code uint64, userClosed bool) {
+ s.outgate.lock()
+ defer s.outUnlock()
+ if s.IsReadOnly() {
+ return
+ }
+ if userClosed {
+ // Mark that the user closed the stream.
+ s.outclosed.set()
+ }
+ if s.outreset.isSet() {
+ return
+ }
+ if code > maxVarint {
+ code = maxVarint
+ }
+ // We could check here to see if the stream is closed and the
+ // peer has acked all the data and the FIN, but sending an
+ // extra RESET_STREAM in this case is harmless.
+ s.outreset.set()
+ s.outresetcode = code
+ s.out.discardBefore(s.out.end)
+ s.outunsent = rangeset[int64]{}
+ s.outblocked.clear()
+}
+
+// inUnlock unlocks s.ingate.
+// It sets the gate condition if reads from s will not block.
+// If s has receive-related frames to write or if both directions
+// are done and the stream should be removed, it notifies the Conn.
+func (s *Stream) inUnlock() {
+ state := s.inUnlockNoQueue()
+ s.conn.maybeQueueStreamForSend(s, state)
+}
+
+// inUnlockNoQueue is inUnlock,
+// but reports whether s has frames to write rather than notifying the Conn.
+func (s *Stream) inUnlockNoQueue() streamState {
+ canRead := s.inset.contains(s.in.start) || // data available to read
+ s.insize == s.in.start || // at EOF
+ s.inresetcode != -1 || // reset by peer
+ s.inclosed.isSet() // closed locally
+ defer s.ingate.unlock(canRead)
+ var state streamState
+ switch {
+ case s.IsWriteOnly():
+ state = streamInDone
+ case s.inresetcode != -1: // reset by peer
+ fallthrough
+ case s.in.start == s.insize: // all data received and read
+ // We don't increase MAX_STREAMS until the user calls ReadClose or Close,
+ // so the receive side is not finished until inclosed is set.
+ if s.inclosed.isSet() {
+ state = streamInDone
+ }
+ case s.insendmax.shouldSend(): // STREAM_MAX_DATA
+ state = streamInSendMeta
+ case s.inclosed.shouldSend(): // STOP_SENDING
+ state = streamInSendMeta
+ }
+ const mask = streamInDone | streamInSendMeta
+ return s.state.set(state, mask)
+}
+
+// outUnlock unlocks s.outgate.
+// It sets the gate condition if writes to s will not block.
+// If s has send-related frames to write or if both directions
+// are done and the stream should be removed, it notifies the Conn.
+func (s *Stream) outUnlock() {
+ state := s.outUnlockNoQueue()
+ s.conn.maybeQueueStreamForSend(s, state)
+}
+
+// outUnlockNoQueue is outUnlock,
+// but reports whether s has frames to write rather than notifying the Conn.
+func (s *Stream) outUnlockNoQueue() streamState {
+ isDone := s.outclosed.isReceived() && s.outacked.isrange(0, s.out.end) || // all data acked
+ s.outreset.isSet() // reset locally
+ if isDone {
+ select {
+ case <-s.outdone:
+ default:
+ if !s.IsReadOnly() {
+ close(s.outdone)
+ }
+ }
+ }
+ lim := s.out.start + s.outmaxbuf
+ canWrite := lim > s.out.end || // available send buffer
+ s.outclosed.isSet() || // closed locally
+ s.outreset.isSet() // reset locally
+ defer s.outgate.unlock(canWrite)
+ var state streamState
+ switch {
+ case s.IsReadOnly():
+ state = streamOutDone
+ case s.outclosed.isReceived() && s.outacked.isrange(0, s.out.end): // all data sent and acked
+ fallthrough
+ case s.outreset.isReceived(): // RESET_STREAM sent and acked
+ // We don't increase MAX_STREAMS until the user calls WriteClose or Close,
+ // so the send side is not finished until outclosed is set.
+ if s.outclosed.isSet() {
+ state = streamOutDone
+ }
+ case s.outreset.shouldSend(): // RESET_STREAM
+ state = streamOutSendMeta
+ case s.outreset.isSet(): // RESET_STREAM sent but not acknowledged
+ case s.outblocked.shouldSend(): // STREAM_DATA_BLOCKED
+ state = streamOutSendMeta
+ case len(s.outunsent) > 0: // STREAM frame with data
+ if s.outunsent.min() < s.outmaxsent {
+ state = streamOutSendMeta // resent data, will not consume flow control
+ } else {
+ state = streamOutSendData // new data, requires flow control
+ }
+ case s.outclosed.shouldSend() && s.out.end == s.outmaxsent: // empty STREAM frame with FIN bit
+ state = streamOutSendMeta
+ case s.outopened.shouldSend(): // STREAM frame with no data
+ state = streamOutSendMeta
+ }
+ const mask = streamOutDone | streamOutSendMeta | streamOutSendData
+ return s.state.set(state, mask)
+}
+
+// handleData handles data received in a STREAM frame.
+func (s *Stream) handleData(off int64, b []byte, fin bool) error {
+ s.ingate.lock()
+ defer s.inUnlock()
+ end := off + int64(len(b))
+ if err := s.checkStreamBounds(end, fin); err != nil {
+ return err
+ }
+ if s.inclosed.isSet() || s.inresetcode != -1 {
+ // The user read-closed the stream, or the peer reset it.
+ // Either way, we can discard this frame.
+ return nil
+ }
+ if s.insize == -1 && end > s.in.end {
+ added := end - s.in.end
+ if err := s.conn.handleStreamBytesReceived(added); err != nil {
+ return err
+ }
+ }
+ s.in.writeAt(b, off)
+ s.inset.add(off, end)
+ if fin {
+ s.insize = end
+ // The peer has enough flow control window to send the entire stream.
+ s.insendmax.clear()
+ }
+ return nil
+}
+
+// handleReset handles a RESET_STREAM frame.
+func (s *Stream) handleReset(code uint64, finalSize int64) error {
+ s.ingate.lock()
+ defer s.inUnlock()
+ const fin = true
+ if err := s.checkStreamBounds(finalSize, fin); err != nil {
+ return err
+ }
+ if s.inresetcode != -1 {
+ // The stream was already reset.
+ return nil
+ }
+ if s.insize == -1 {
+ added := finalSize - s.in.end
+ if err := s.conn.handleStreamBytesReceived(added); err != nil {
+ return err
+ }
+ }
+ s.conn.handleStreamBytesReadOnLoop(finalSize - s.in.start)
+ s.in.discardBefore(s.in.end)
+ s.inresetcode = int64(code)
+ s.insize = finalSize
+ return nil
+}
+
+// checkStreamBounds validates the stream offset in a STREAM or RESET_STREAM frame.
+func (s *Stream) checkStreamBounds(end int64, fin bool) error {
+ if end > s.inwin {
+ // The peer sent us data past the maximum flow control window we gave them.
+ return localTransportError(errFlowControl)
+ }
+ if s.insize != -1 && end > s.insize {
+ // The peer sent us data past the final size of the stream they previously gave us.
+ return localTransportError(errFinalSize)
+ }
+ if fin && s.insize != -1 && end != s.insize {
+ // The peer changed the final size of the stream.
+ return localTransportError(errFinalSize)
+ }
+ if fin && end < s.in.end {
+ // The peer has previously sent us data past the final size.
+ return localTransportError(errFinalSize)
+ }
+ return nil
+}
+
+// handleStopSending handles a STOP_SENDING frame.
+func (s *Stream) handleStopSending(code uint64) error {
+ // Peer requests that we reset this stream.
+ // https://www.rfc-editor.org/rfc/rfc9000#section-3.5-4
+ const userReset = false
+ s.resetInternal(code, userReset)
+ return nil
+}
+
+// handleMaxStreamData handles an update received in a MAX_STREAM_DATA frame.
+func (s *Stream) handleMaxStreamData(maxStreamData int64) error {
+ s.outgate.lock()
+ defer s.outUnlock()
+ if maxStreamData <= s.outwin {
+ return nil
+ }
+ if s.out.end > s.outwin {
+ s.outunsent.add(s.outwin, min(maxStreamData, s.out.end))
+ }
+ s.outwin = maxStreamData
+ if s.out.end > s.outwin {
+ // We've still got more data than flow control window.
+ s.outblocked.setUnsent()
+ } else {
+ s.outblocked.clear()
+ }
+ return nil
+}
+
+// ackOrLoss handles the fate of stream frames other than STREAM.
+func (s *Stream) ackOrLoss(pnum packetNumber, ftype byte, fate packetFate) {
+ // Frames which carry new information each time they are sent
+ // (MAX_STREAM_DATA, STREAM_DATA_BLOCKED) must only be marked
+ // as received if the most recent packet carrying this frame is acked.
+ //
+ // Frames which are always the same (STOP_SENDING, RESET_STREAM)
+ // can be marked as received if any packet carrying this frame is acked.
+ switch ftype {
+ case frameTypeResetStream:
+ s.outgate.lock()
+ s.outreset.ackOrLoss(pnum, fate)
+ s.outUnlock()
+ case frameTypeStopSending:
+ s.ingate.lock()
+ s.inclosed.ackOrLoss(pnum, fate)
+ s.inUnlock()
+ case frameTypeMaxStreamData:
+ s.ingate.lock()
+ s.insendmax.ackLatestOrLoss(pnum, fate)
+ s.inUnlock()
+ case frameTypeStreamDataBlocked:
+ s.outgate.lock()
+ s.outblocked.ackLatestOrLoss(pnum, fate)
+ s.outUnlock()
+ default:
+ panic("unhandled frame type")
+ }
+}
+
+// ackOrLossData handles the fate of a STREAM frame.
+func (s *Stream) ackOrLossData(pnum packetNumber, start, end int64, fin bool, fate packetFate) {
+ s.outgate.lock()
+ defer s.outUnlock()
+ s.outopened.ackOrLoss(pnum, fate)
+ if fin {
+ s.outclosed.ackOrLoss(pnum, fate)
+ }
+ if s.outreset.isSet() {
+ // If the stream has been reset, we don't care any more.
+ return
+ }
+ switch fate {
+ case packetAcked:
+ s.outacked.add(start, end)
+ s.outunsent.sub(start, end)
+ // If this ack is for data at the start of the send buffer, we can now discard it.
+ if s.outacked.contains(s.out.start) {
+ s.out.discardBefore(s.outacked[0].end)
+ }
+ case packetLost:
+ // Mark everything lost, but not previously acked, as needing retransmission.
+ // We do this by adding all the lost bytes to outunsent, and then
+ // removing everything already acked.
+ s.outunsent.add(start, end)
+ for _, a := range s.outacked {
+ s.outunsent.sub(a.start, a.end)
+ }
+ }
+}
+
+// appendInFramesLocked appends STOP_SENDING and MAX_STREAM_DATA frames
+// to the current packet.
+//
+// It returns true if no more frames need appending,
+// false if not everything fit in the current packet.
+func (s *Stream) appendInFramesLocked(w *packetWriter, pnum packetNumber, pto bool) bool {
+ if s.inclosed.shouldSendPTO(pto) {
+ // We don't currently have an API for setting the error code.
+ // Just send zero.
+ code := uint64(0)
+ if !w.appendStopSendingFrame(s.id, code) {
+ return false
+ }
+ s.inclosed.setSent(pnum)
+ }
+ // TODO: STOP_SENDING
+ if s.insendmax.shouldSendPTO(pto) {
+ // MAX_STREAM_DATA
+ maxStreamData := s.in.start + s.inmaxbuf
+ if !w.appendMaxStreamDataFrame(s.id, maxStreamData) {
+ return false
+ }
+ s.inwin = maxStreamData
+ s.insendmax.setSent(pnum)
+ }
+ return true
+}
+
+// appendOutFramesLocked appends RESET_STREAM, STREAM_DATA_BLOCKED, and STREAM frames
+// to the current packet.
+//
+// It returns true if no more frames need appending,
+// false if not everything fit in the current packet.
+func (s *Stream) appendOutFramesLocked(w *packetWriter, pnum packetNumber, pto bool) bool {
+ if s.outreset.isSet() {
+ // RESET_STREAM
+ if s.outreset.shouldSendPTO(pto) {
+ if !w.appendResetStreamFrame(s.id, s.outresetcode, min(s.outwin, s.out.end)) {
+ return false
+ }
+ s.outreset.setSent(pnum)
+ s.frameOpensStream(pnum)
+ }
+ return true
+ }
+ if s.outblocked.shouldSendPTO(pto) {
+ // STREAM_DATA_BLOCKED
+ if !w.appendStreamDataBlockedFrame(s.id, s.outwin) {
+ return false
+ }
+ s.outblocked.setSent(pnum)
+ s.frameOpensStream(pnum)
+ }
+ for {
+ // STREAM
+ off, size := dataToSend(min(s.out.start, s.outwin), min(s.out.end, s.outwin), s.outunsent, s.outacked, pto)
+ if end := off + size; end > s.outmaxsent {
+ // This will require connection-level flow control to send.
+ end = min(end, s.outmaxsent+s.conn.streams.outflow.avail())
+ size = end - off
+ }
+ fin := s.outclosed.isSet() && off+size == s.out.end
+ shouldSend := size > 0 || // have data to send
+ s.outopened.shouldSendPTO(pto) || // should open the stream
+ (fin && s.outclosed.shouldSendPTO(pto)) // should close the stream
+ if !shouldSend {
+ return true
+ }
+ b, added := w.appendStreamFrame(s.id, off, int(size), fin)
+ if !added {
+ return false
+ }
+ s.out.copy(off, b)
+ end := off + int64(len(b))
+ if end > s.outmaxsent {
+ s.conn.streams.outflow.consume(end - s.outmaxsent)
+ s.outmaxsent = end
+ }
+ s.outunsent.sub(off, end)
+ s.frameOpensStream(pnum)
+ if fin {
+ s.outclosed.setSent(pnum)
+ }
+ if pto {
+ return true
+ }
+ if int64(len(b)) < size {
+ return false
+ }
+ }
+}
+
+// frameOpensStream records that we're sending a frame that will open the stream.
+//
+// If we don't have an acknowledgement from the peer for a previous frame opening the stream,
+// record this packet as being the latest one to open it.
+func (s *Stream) frameOpensStream(pnum packetNumber) {
+ if !s.outopened.isReceived() {
+ s.outopened.setSent(pnum)
+ }
+}
+
+// dataToSend returns the next range of data to send in a STREAM or CRYPTO_STREAM.
+func dataToSend(start, end int64, outunsent, outacked rangeset[int64], pto bool) (sendStart, size int64) {
+ switch {
+ case pto:
+ // On PTO, resend unacked data that fits in the probe packet.
+ // For simplicity, we send the range starting at s.out.start
+ // (which is definitely unacked, or else we would have discarded it)
+ // up to the next acked byte (if any).
+ //
+ // This may miss unacked data starting after that acked byte,
+ // but avoids resending data the peer has acked.
+ for _, r := range outacked {
+ if r.start > start {
+ return start, r.start - start
+ }
+ }
+ return start, end - start
+ case outunsent.numRanges() > 0:
+ return outunsent.min(), outunsent[0].size()
+ default:
+ return end, 0
+ }
+}
diff --git a/internal/quic/stream_limits.go b/internal/quic/stream_limits.go
new file mode 100644
index 0000000000..6eda7883b9
--- /dev/null
+++ b/internal/quic/stream_limits.go
@@ -0,0 +1,113 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build go1.21
+
+package quic
+
+import (
+ "context"
+)
+
+// Limits on the number of open streams.
+// Every connection has separate limits for bidirectional and unidirectional streams.
+//
+// Note that the MAX_STREAMS limit includes closed as well as open streams.
+// Closing a stream doesn't enable an endpoint to open a new one;
+// only an increase in the MAX_STREAMS limit does.
+
+// localStreamLimits are limits on the number of open streams created by us.
+type localStreamLimits struct {
+ gate gate
+ max int64 // peer-provided MAX_STREAMS
+ opened int64 // number of streams opened by us
+}
+
+func (lim *localStreamLimits) init() {
+ lim.gate = newGate()
+}
+
+// open creates a new local stream, blocking until MAX_STREAMS quota is available.
+func (lim *localStreamLimits) open(ctx context.Context, c *Conn) (num int64, err error) {
+ // TODO: Send a STREAMS_BLOCKED when blocked.
+ if err := lim.gate.waitAndLock(ctx, c.testHooks); err != nil {
+ return 0, err
+ }
+ n := lim.opened
+ lim.opened++
+ lim.gate.unlock(lim.opened < lim.max)
+ return n, nil
+}
+
+// setMax sets the MAX_STREAMS provided by the peer.
+func (lim *localStreamLimits) setMax(maxStreams int64) {
+ lim.gate.lock()
+ lim.max = max(lim.max, maxStreams)
+ lim.gate.unlock(lim.opened < lim.max)
+}
+
+// remoteStreamLimits are limits on the number of open streams created by the peer.
+type remoteStreamLimits struct {
+ max int64 // last MAX_STREAMS sent to the peer
+ opened int64 // number of streams opened by the peer (including subsequently closed ones)
+ closed int64 // number of peer streams in the "closed" state
+ maxOpen int64 // how many streams we want to let the peer simultaneously open
+ sendMax sentVal // set when we should send MAX_STREAMS
+}
+
+func (lim *remoteStreamLimits) init(maxOpen int64) {
+ lim.maxOpen = maxOpen
+ lim.max = min(maxOpen, implicitStreamLimit) // initial limit sent in transport parameters
+ lim.opened = 0
+}
+
+// open handles the peer opening a new stream.
+func (lim *remoteStreamLimits) open(id streamID) error {
+ num := id.num()
+ if num >= lim.max {
+ return localTransportError(errStreamLimit)
+ }
+ if num >= lim.opened {
+ lim.opened = num + 1
+ lim.maybeUpdateMax()
+ }
+ return nil
+}
+
+// close handles the peer closing an open stream.
+func (lim *remoteStreamLimits) close() {
+ lim.closed++
+ lim.maybeUpdateMax()
+}
+
+// maybeUpdateMax updates the MAX_STREAMS value we will send to the peer.
+func (lim *remoteStreamLimits) maybeUpdateMax() {
+ newMax := min(
+ // Max streams the peer can have open at once.
+ lim.closed+lim.maxOpen,
+ // Max streams the peer can open with a single frame.
+ lim.opened+implicitStreamLimit,
+ )
+ avail := lim.max - lim.opened
+ if newMax > lim.max && (avail < 8 || newMax-lim.max >= 2*avail) {
+ // If the peer has less than 8 streams, or if increasing the peer's
+ // stream limit would double it, then send a MAX_STREAMS.
+ lim.max = newMax
+ lim.sendMax.setUnsent()
+ }
+}
+
+// appendFrame appends a MAX_STREAMS frame to the current packet, if necessary.
+//
+// It returns true if no more frames need appending,
+// false if not everything fit in the current packet.
+func (lim *remoteStreamLimits) appendFrame(w *packetWriter, typ streamType, pnum packetNumber, pto bool) bool {
+ if lim.sendMax.shouldSendPTO(pto) {
+ if !w.appendMaxStreamsFrame(typ, lim.max) {
+ return false
+ }
+ lim.sendMax.setSent(pnum)
+ }
+ return true
+}
diff --git a/internal/quic/stream_limits_test.go b/internal/quic/stream_limits_test.go
new file mode 100644
index 0000000000..3f291e9f4c
--- /dev/null
+++ b/internal/quic/stream_limits_test.go
@@ -0,0 +1,269 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build go1.21
+
+package quic
+
+import (
+ "context"
+ "crypto/tls"
+ "testing"
+)
+
+func TestStreamLimitNewStreamBlocked(t *testing.T) {
+ // "An endpoint that receives a frame with a stream ID exceeding the limit
+ // it has sent MUST treat this as a connection error of type STREAM_LIMIT_ERROR [...]"
+ // https://www.rfc-editor.org/rfc/rfc9000#section-4.6-3
+ testStreamTypes(t, "", func(t *testing.T, styp streamType) {
+ ctx := canceledContext()
+ tc := newTestConn(t, clientSide,
+ permissiveTransportParameters,
+ func(p *transportParameters) {
+ p.initialMaxStreamsBidi = 0
+ p.initialMaxStreamsUni = 0
+ })
+ tc.handshake()
+ tc.ignoreFrame(frameTypeAck)
+ opening := runAsync(tc, func(ctx context.Context) (*Stream, error) {
+ return tc.conn.newLocalStream(ctx, styp)
+ })
+ if _, err := opening.result(); err != errNotDone {
+ t.Fatalf("new stream blocked by limit: %v, want errNotDone", err)
+ }
+ tc.writeFrames(packetType1RTT, debugFrameMaxStreams{
+ streamType: styp,
+ max: 1,
+ })
+ if _, err := opening.result(); err != nil {
+ t.Fatalf("new stream not created after limit raised: %v", err)
+ }
+ if _, err := tc.conn.newLocalStream(ctx, styp); err == nil {
+ t.Fatalf("new stream blocked by raised limit: %v, want error", err)
+ }
+ })
+}
+
+func TestStreamLimitMaxStreamsDecreases(t *testing.T) {
+ // "MAX_STREAMS frames that do not increase the stream limit MUST be ignored."
+ // https://www.rfc-editor.org/rfc/rfc9000#section-4.6-4
+ testStreamTypes(t, "", func(t *testing.T, styp streamType) {
+ ctx := canceledContext()
+ tc := newTestConn(t, clientSide,
+ permissiveTransportParameters,
+ func(p *transportParameters) {
+ p.initialMaxStreamsBidi = 0
+ p.initialMaxStreamsUni = 0
+ })
+ tc.handshake()
+ tc.ignoreFrame(frameTypeAck)
+ tc.writeFrames(packetType1RTT, debugFrameMaxStreams{
+ streamType: styp,
+ max: 2,
+ })
+ tc.writeFrames(packetType1RTT, debugFrameMaxStreams{
+ streamType: styp,
+ max: 1,
+ })
+ if _, err := tc.conn.newLocalStream(ctx, styp); err != nil {
+ t.Fatalf("open stream 1, limit 2, got error: %v", err)
+ }
+ if _, err := tc.conn.newLocalStream(ctx, styp); err != nil {
+ t.Fatalf("open stream 2, limit 2, got error: %v", err)
+ }
+ if _, err := tc.conn.newLocalStream(ctx, styp); err == nil {
+ t.Fatalf("open stream 3, limit 2, got error: %v", err)
+ }
+ })
+}
+
+func TestStreamLimitViolated(t *testing.T) {
+ testStreamTypes(t, "", func(t *testing.T, styp streamType) {
+ tc := newTestConn(t, serverSide,
+ func(c *Config) {
+ if styp == bidiStream {
+ c.MaxBidiRemoteStreams = 10
+ } else {
+ c.MaxUniRemoteStreams = 10
+ }
+ })
+ tc.handshake()
+ tc.ignoreFrame(frameTypeAck)
+ tc.writeFrames(packetType1RTT, debugFrameStream{
+ id: newStreamID(clientSide, styp, 9),
+ })
+ tc.wantIdle("stream number 9 is within the limit")
+ tc.writeFrames(packetType1RTT, debugFrameStream{
+ id: newStreamID(clientSide, styp, 10),
+ })
+ tc.wantFrame("stream number 10 is beyond the limit",
+ packetType1RTT, debugFrameConnectionCloseTransport{
+ code: errStreamLimit,
+ },
+ )
+ })
+}
+
+func TestStreamLimitImplicitStreams(t *testing.T) {
+ testStreamTypes(t, "", func(t *testing.T, styp streamType) {
+ tc := newTestConn(t, serverSide,
+ func(c *Config) {
+ c.MaxBidiRemoteStreams = 1 << 60
+ c.MaxUniRemoteStreams = 1 << 60
+ })
+ tc.handshake()
+ tc.ignoreFrame(frameTypeAck)
+ if got, want := tc.sentTransportParameters.initialMaxStreamsBidi, int64(implicitStreamLimit); got != want {
+ t.Errorf("sent initial_max_streams_bidi = %v, want %v", got, want)
+ }
+ if got, want := tc.sentTransportParameters.initialMaxStreamsUni, int64(implicitStreamLimit); got != want {
+ t.Errorf("sent initial_max_streams_uni = %v, want %v", got, want)
+ }
+
+ // Create stream 0.
+ tc.writeFrames(packetType1RTT, debugFrameStream{
+ id: newStreamID(clientSide, styp, 0),
+ })
+ tc.wantIdle("max streams not increased enough to send a new frame")
+
+ // Create streams [0, implicitStreamLimit).
+ tc.writeFrames(packetType1RTT, debugFrameStream{
+ id: newStreamID(clientSide, styp, implicitStreamLimit-1),
+ })
+ tc.wantFrame("max streams increases to implicit stream limit",
+ packetType1RTT, debugFrameMaxStreams{
+ streamType: styp,
+ max: 2 * implicitStreamLimit,
+ })
+
+ // Create a stream past the limit.
+ tc.writeFrames(packetType1RTT, debugFrameStream{
+ id: newStreamID(clientSide, styp, 2*implicitStreamLimit),
+ })
+ tc.wantFrame("stream is past the limit",
+ packetType1RTT, debugFrameConnectionCloseTransport{
+ code: errStreamLimit,
+ },
+ )
+ })
+}
+
+func TestStreamLimitMaxStreamsTransportParameterTooLarge(t *testing.T) {
+ // "If a max_streams transport parameter [...] is received with
+ // a value greater than 2^60 [...] the connection MUST be closed
+ // immediately with a connection error of type TRANSPORT_PARAMETER_ERROR [...]"
+ // https://www.rfc-editor.org/rfc/rfc9000#section-4.6-2
+ testStreamTypes(t, "", func(t *testing.T, styp streamType) {
+ tc := newTestConn(t, serverSide,
+ func(p *transportParameters) {
+ if styp == bidiStream {
+ p.initialMaxStreamsBidi = 1<<60 + 1
+ } else {
+ p.initialMaxStreamsUni = 1<<60 + 1
+ }
+ })
+ tc.writeFrames(packetTypeInitial, debugFrameCrypto{
+ data: tc.cryptoDataIn[tls.QUICEncryptionLevelInitial],
+ })
+ tc.wantFrame("max streams transport parameter is too large",
+ packetTypeInitial, debugFrameConnectionCloseTransport{
+ code: errTransportParameter,
+ },
+ )
+ })
+}
+
+func TestStreamLimitMaxStreamsFrameTooLarge(t *testing.T) {
+ // "If [...] a MAX_STREAMS frame is received with a value
+ // greater than 2^60 [...] the connection MUST be closed immediately
+ // with a connection error [...] of type FRAME_ENCODING_ERROR [...]"
+ // https://www.rfc-editor.org/rfc/rfc9000#section-4.6-2
+ testStreamTypes(t, "", func(t *testing.T, styp streamType) {
+ tc := newTestConn(t, serverSide)
+ tc.handshake()
+ tc.writeFrames(packetTypeInitial,
+ debugFrameCrypto{
+ data: tc.cryptoDataIn[tls.QUICEncryptionLevelInitial],
+ })
+ tc.writeFrames(packetType1RTT, debugFrameMaxStreams{
+ streamType: styp,
+ max: 1<<60 + 1,
+ })
+ tc.wantFrame("MAX_STREAMS value is too large",
+ packetType1RTT, debugFrameConnectionCloseTransport{
+ code: errFrameEncoding,
+ },
+ )
+ })
+}
+
+func TestStreamLimitSendUpdatesMaxStreams(t *testing.T) {
+ testStreamTypes(t, "", func(t *testing.T, styp streamType) {
+ ctx := canceledContext()
+ tc := newTestConn(t, serverSide, func(c *Config) {
+ if styp == uniStream {
+ c.MaxUniRemoteStreams = 4
+ c.MaxBidiRemoteStreams = 0
+ } else {
+ c.MaxUniRemoteStreams = 0
+ c.MaxBidiRemoteStreams = 4
+ }
+ })
+ tc.handshake()
+ tc.ignoreFrame(frameTypeAck)
+ var streams []*Stream
+ for i := 0; i < 4; i++ {
+ tc.writeFrames(packetType1RTT, debugFrameStream{
+ id: newStreamID(clientSide, styp, int64(i)),
+ fin: true,
+ })
+ s, err := tc.conn.AcceptStream(ctx)
+ if err != nil {
+ t.Fatalf("AcceptStream = %v", err)
+ }
+ streams = append(streams, s)
+ }
+ streams[3].CloseContext(ctx)
+ if styp == bidiStream {
+ tc.wantFrame("stream is closed",
+ packetType1RTT, debugFrameStream{
+ id: streams[3].id,
+ fin: true,
+ data: []byte{},
+ })
+ tc.writeAckForAll()
+ }
+ tc.wantFrame("closing a stream when peer is at limit immediately extends the limit",
+ packetType1RTT, debugFrameMaxStreams{
+ streamType: styp,
+ max: 5,
+ })
+ })
+}
+
+func TestStreamLimitStopSendingDoesNotUpdateMaxStreams(t *testing.T) {
+ tc, s := newTestConnAndRemoteStream(t, serverSide, bidiStream, func(c *Config) {
+ c.MaxBidiRemoteStreams = 1
+ })
+ tc.writeFrames(packetType1RTT, debugFrameStream{
+ id: s.id,
+ fin: true,
+ })
+ s.CloseRead()
+ tc.writeFrames(packetType1RTT, debugFrameStopSending{
+ id: s.id,
+ })
+ tc.wantFrame("recieved STOP_SENDING, send RESET_STREAM",
+ packetType1RTT, debugFrameResetStream{
+ id: s.id,
+ })
+ tc.writeAckForAll()
+ tc.wantIdle("MAX_STREAMS is not extended until the user fully closes the stream")
+ s.CloseWrite()
+ tc.wantFrame("user closing the stream triggers MAX_STREAMS update",
+ packetType1RTT, debugFrameMaxStreams{
+ streamType: bidiStream,
+ max: 2,
+ })
+}
diff --git a/internal/quic/stream_test.go b/internal/quic/stream_test.go
new file mode 100644
index 0000000000..7c1377faee
--- /dev/null
+++ b/internal/quic/stream_test.go
@@ -0,0 +1,1335 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build go1.21
+
+package quic
+
+import (
+ "bytes"
+ "context"
+ "crypto/rand"
+ "errors"
+ "fmt"
+ "io"
+ "reflect"
+ "strings"
+ "testing"
+)
+
+func TestStreamWriteBlockedByOutputBuffer(t *testing.T) {
+ testStreamTypes(t, "", func(t *testing.T, styp streamType) {
+ ctx := canceledContext()
+ want := []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}
+ const writeBufferSize = 4
+ tc := newTestConn(t, clientSide, permissiveTransportParameters, func(c *Config) {
+ c.MaxStreamWriteBufferSize = writeBufferSize
+ })
+ tc.handshake()
+ tc.ignoreFrame(frameTypeAck)
+
+ s, err := tc.conn.newLocalStream(ctx, styp)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // Non-blocking write.
+ n, err := s.WriteContext(ctx, want)
+ if n != writeBufferSize || err != context.Canceled {
+ t.Fatalf("s.WriteContext() = %v, %v; want %v, context.Canceled", n, err, writeBufferSize)
+ }
+ tc.wantFrame("first write buffer of data sent",
+ packetType1RTT, debugFrameStream{
+ id: s.id,
+ data: want[:writeBufferSize],
+ })
+ off := int64(writeBufferSize)
+
+ // Blocking write, which must wait for buffer space.
+ w := runAsync(tc, func(ctx context.Context) (int, error) {
+ return s.WriteContext(ctx, want[writeBufferSize:])
+ })
+ tc.wantIdle("write buffer is full, no more data can be sent")
+
+ // The peer's ack of the STREAM frame allows progress.
+ tc.writeAckForAll()
+ tc.wantFrame("second write buffer of data sent",
+ packetType1RTT, debugFrameStream{
+ id: s.id,
+ off: off,
+ data: want[off:][:writeBufferSize],
+ })
+ off += writeBufferSize
+ tc.wantIdle("write buffer is full, no more data can be sent")
+
+ // The peer's ack of the second STREAM frame allows sending the remaining data.
+ tc.writeAckForAll()
+ tc.wantFrame("remaining data sent",
+ packetType1RTT, debugFrameStream{
+ id: s.id,
+ off: off,
+ data: want[off:],
+ })
+
+ if n, err := w.result(); n != len(want)-writeBufferSize || err != nil {
+ t.Fatalf("s.WriteContext() = %v, %v; want %v, nil",
+ len(want)-writeBufferSize, err, writeBufferSize)
+ }
+ })
+}
+
+func TestStreamWriteBlockedByStreamFlowControl(t *testing.T) {
+ testStreamTypes(t, "", func(t *testing.T, styp streamType) {
+ ctx := canceledContext()
+ want := []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}
+ tc := newTestConn(t, clientSide, func(p *transportParameters) {
+ p.initialMaxStreamsBidi = 100
+ p.initialMaxStreamsUni = 100
+ p.initialMaxData = 1 << 20
+ })
+ tc.handshake()
+ tc.ignoreFrame(frameTypeAck)
+
+ s, err := tc.conn.newLocalStream(ctx, styp)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // Data is written to the stream output buffer, but we have no flow control.
+ _, err = s.WriteContext(ctx, want[:1])
+ if err != nil {
+ t.Fatalf("write with available output buffer: unexpected error: %v", err)
+ }
+ tc.wantFrame("write blocked by flow control triggers a STREAM_DATA_BLOCKED frame",
+ packetType1RTT, debugFrameStreamDataBlocked{
+ id: s.id,
+ max: 0,
+ })
+
+ // Write more data.
+ _, err = s.WriteContext(ctx, want[1:])
+ if err != nil {
+ t.Fatalf("write with available output buffer: unexpected error: %v", err)
+ }
+ tc.wantIdle("adding more blocked data does not trigger another STREAM_DATA_BLOCKED")
+
+ // Provide some flow control window.
+ tc.writeFrames(packetType1RTT, debugFrameMaxStreamData{
+ id: s.id,
+ max: 4,
+ })
+ tc.wantFrame("stream window extended, but still more data to write",
+ packetType1RTT, debugFrameStreamDataBlocked{
+ id: s.id,
+ max: 4,
+ })
+ tc.wantFrame("stream window extended to 4, expect blocked write to progress",
+ packetType1RTT, debugFrameStream{
+ id: s.id,
+ data: want[:4],
+ })
+
+ // Provide more flow control window.
+ tc.writeFrames(packetType1RTT, debugFrameMaxStreamData{
+ id: s.id,
+ max: int64(len(want)),
+ })
+ tc.wantFrame("stream window extended further, expect blocked write to finish",
+ packetType1RTT, debugFrameStream{
+ id: s.id,
+ off: 4,
+ data: want[4:],
+ })
+ })
+}
+
+func TestStreamIgnoresMaxStreamDataReduction(t *testing.T) {
+ // "A sender MUST ignore any MAX_STREAM_DATA [...] frames that
+ // do not increase flow control limits."
+ // https://www.rfc-editor.org/rfc/rfc9000#section-4.1-9
+ testStreamTypes(t, "", func(t *testing.T, styp streamType) {
+ ctx := canceledContext()
+ want := []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}
+ tc := newTestConn(t, clientSide, func(p *transportParameters) {
+ if styp == uniStream {
+ p.initialMaxStreamsUni = 1
+ p.initialMaxStreamDataUni = 4
+ } else {
+ p.initialMaxStreamsBidi = 1
+ p.initialMaxStreamDataBidiRemote = 4
+ }
+ p.initialMaxData = 1 << 20
+ })
+ tc.handshake()
+ tc.ignoreFrame(frameTypeAck)
+ tc.ignoreFrame(frameTypeStreamDataBlocked)
+
+ // Write [0,1).
+ s, err := tc.conn.newLocalStream(ctx, styp)
+ if err != nil {
+ t.Fatal(err)
+ }
+ s.WriteContext(ctx, want[:1])
+ tc.wantFrame("sent data (1 byte) fits within flow control limit",
+ packetType1RTT, debugFrameStream{
+ id: s.id,
+ off: 0,
+ data: want[:1],
+ })
+
+ // MAX_STREAM_DATA tries to decrease limit, and is ignored.
+ tc.writeFrames(packetType1RTT, debugFrameMaxStreamData{
+ id: s.id,
+ max: 2,
+ })
+
+ // Write [1,4).
+ s.WriteContext(ctx, want[1:])
+ tc.wantFrame("stream limit is 4 bytes, ignoring decrease in MAX_STREAM_DATA",
+ packetType1RTT, debugFrameStream{
+ id: s.id,
+ off: 1,
+ data: want[1:4],
+ })
+
+ // MAX_STREAM_DATA increases limit.
+ // Second MAX_STREAM_DATA decreases it, and is ignored.
+ tc.writeFrames(packetType1RTT, debugFrameMaxStreamData{
+ id: s.id,
+ max: 8,
+ })
+ tc.writeFrames(packetType1RTT, debugFrameMaxStreamData{
+ id: s.id,
+ max: 6,
+ })
+
+ // Write [1,4).
+ s.WriteContext(ctx, want[4:])
+ tc.wantFrame("stream limit is 8 bytes, ignoring decrease in MAX_STREAM_DATA",
+ packetType1RTT, debugFrameStream{
+ id: s.id,
+ off: 4,
+ data: want[4:8],
+ })
+ })
+}
+
+func TestStreamWriteBlockedByWriteBufferLimit(t *testing.T) {
+ testStreamTypes(t, "", func(t *testing.T, styp streamType) {
+ ctx := canceledContext()
+ want := []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}
+ const maxWriteBuffer = 4
+ tc := newTestConn(t, clientSide, func(p *transportParameters) {
+ p.initialMaxStreamsBidi = 100
+ p.initialMaxStreamsUni = 100
+ p.initialMaxData = 1 << 20
+ p.initialMaxStreamDataBidiRemote = 1 << 20
+ p.initialMaxStreamDataUni = 1 << 20
+ }, func(c *Config) {
+ c.MaxStreamWriteBufferSize = maxWriteBuffer
+ })
+ tc.handshake()
+ tc.ignoreFrame(frameTypeAck)
+
+ // Write more data than StreamWriteBufferSize.
+ // The peer has given us plenty of flow control,
+ // so we're just blocked by our local limit.
+ s, err := tc.conn.newLocalStream(ctx, styp)
+ if err != nil {
+ t.Fatal(err)
+ }
+ w := runAsync(tc, func(ctx context.Context) (int, error) {
+ return s.WriteContext(ctx, want)
+ })
+ tc.wantFrame("stream write should send as much data as write buffer allows",
+ packetType1RTT, debugFrameStream{
+ id: s.id,
+ off: 0,
+ data: want[:maxWriteBuffer],
+ })
+ tc.wantIdle("no STREAM_DATA_BLOCKED, we're blocked locally not by flow control")
+
+ // ACK for previously-sent data allows making more progress.
+ tc.writeAckForAll()
+ tc.wantFrame("ACK for previous data allows making progress",
+ packetType1RTT, debugFrameStream{
+ id: s.id,
+ off: maxWriteBuffer,
+ data: want[maxWriteBuffer:][:maxWriteBuffer],
+ })
+
+ // Cancel the write with data left to send.
+ w.cancel()
+ n, err := w.result()
+ if n != 2*maxWriteBuffer || err == nil {
+ t.Fatalf("WriteContext() = %v, %v; want %v bytes, error", n, err, 2*maxWriteBuffer)
+ }
+ })
+}
+
+func TestStreamReceive(t *testing.T) {
+ // "Endpoints MUST be able to deliver stream data to an application as
+ // an ordered byte stream."
+ // https://www.rfc-editor.org/rfc/rfc9000#section-2.2-2
+ want := make([]byte, 5000)
+ for i := range want {
+ want[i] = byte(i)
+ }
+ type frame struct {
+ start int64
+ end int64
+ fin bool
+ want int
+ wantEOF bool
+ }
+ for _, test := range []struct {
+ name string
+ frames []frame
+ }{{
+ name: "linear",
+ frames: []frame{{
+ start: 0,
+ end: 1000,
+ want: 1000,
+ }, {
+ start: 1000,
+ end: 2000,
+ want: 2000,
+ }, {
+ start: 2000,
+ end: 3000,
+ want: 3000,
+ fin: true,
+ wantEOF: true,
+ }},
+ }, {
+ name: "out of order",
+ frames: []frame{{
+ start: 1000,
+ end: 2000,
+ }, {
+ start: 2000,
+ end: 3000,
+ }, {
+ start: 0,
+ end: 1000,
+ want: 3000,
+ }},
+ }, {
+ name: "resent",
+ frames: []frame{{
+ start: 0,
+ end: 1000,
+ want: 1000,
+ }, {
+ start: 0,
+ end: 1000,
+ want: 1000,
+ }, {
+ start: 1000,
+ end: 2000,
+ want: 2000,
+ }, {
+ start: 0,
+ end: 1000,
+ want: 2000,
+ }, {
+ start: 1000,
+ end: 2000,
+ want: 2000,
+ }},
+ }, {
+ name: "overlapping",
+ frames: []frame{{
+ start: 0,
+ end: 1000,
+ want: 1000,
+ }, {
+ start: 3000,
+ end: 4000,
+ want: 1000,
+ }, {
+ start: 2000,
+ end: 3000,
+ want: 1000,
+ }, {
+ start: 1000,
+ end: 3000,
+ want: 4000,
+ }},
+ }, {
+ name: "early eof",
+ frames: []frame{{
+ start: 3000,
+ end: 3000,
+ fin: true,
+ want: 0,
+ }, {
+ start: 1000,
+ end: 2000,
+ want: 0,
+ }, {
+ start: 0,
+ end: 1000,
+ want: 2000,
+ }, {
+ start: 2000,
+ end: 3000,
+ want: 3000,
+ wantEOF: true,
+ }},
+ }, {
+ name: "empty eof",
+ frames: []frame{{
+ start: 0,
+ end: 1000,
+ want: 1000,
+ }, {
+ start: 1000,
+ end: 1000,
+ fin: true,
+ want: 1000,
+ wantEOF: true,
+ }},
+ }} {
+ testStreamTypes(t, test.name, func(t *testing.T, styp streamType) {
+ ctx := canceledContext()
+ tc := newTestConn(t, serverSide)
+ tc.handshake()
+ sid := newStreamID(clientSide, styp, 0)
+ var s *Stream
+ got := make([]byte, len(want))
+ var total int
+ for _, f := range test.frames {
+ t.Logf("receive [%v,%v)", f.start, f.end)
+ tc.writeFrames(packetType1RTT, debugFrameStream{
+ id: sid,
+ off: f.start,
+ data: want[f.start:f.end],
+ fin: f.fin,
+ })
+ if s == nil {
+ var err error
+ s, err = tc.conn.AcceptStream(ctx)
+ if err != nil {
+ tc.t.Fatalf("conn.AcceptStream() = %v", err)
+ }
+ }
+ for {
+ n, err := s.ReadContext(ctx, got[total:])
+ t.Logf("s.ReadContext() = %v, %v", n, err)
+ total += n
+ if f.wantEOF && err != io.EOF {
+ t.Fatalf("ReadContext() error = %v; want io.EOF", err)
+ }
+ if !f.wantEOF && err == io.EOF {
+ t.Fatalf("ReadContext() error = io.EOF, want something else")
+ }
+ if err != nil {
+ break
+ }
+ }
+ if total != f.want {
+ t.Fatalf("total bytes read = %v, want %v", total, f.want)
+ }
+ for i := 0; i < total; i++ {
+ if got[i] != want[i] {
+ t.Fatalf("byte %v differs: got %v, want %v", i, got[i], want[i])
+ }
+ }
+ }
+ })
+ }
+
+}
+
+func TestStreamReceiveExtendsStreamWindow(t *testing.T) {
+ testStreamTypes(t, "", func(t *testing.T, styp streamType) {
+ const maxWindowSize = 20
+ ctx := canceledContext()
+ tc := newTestConn(t, serverSide, func(c *Config) {
+ c.MaxStreamReadBufferSize = maxWindowSize
+ })
+ tc.handshake()
+ tc.ignoreFrame(frameTypeAck)
+ sid := newStreamID(clientSide, styp, 0)
+ tc.writeFrames(packetType1RTT, debugFrameStream{
+ id: sid,
+ off: 0,
+ data: make([]byte, maxWindowSize),
+ })
+ s, err := tc.conn.AcceptStream(ctx)
+ if err != nil {
+ t.Fatalf("AcceptStream: %v", err)
+ }
+ tc.wantIdle("stream window is not extended before data is read")
+ buf := make([]byte, maxWindowSize+1)
+ if n, err := s.ReadContext(ctx, buf); n != maxWindowSize || err != nil {
+ t.Fatalf("s.ReadContext() = %v, %v; want %v, nil", n, err, maxWindowSize)
+ }
+ tc.wantFrame("stream window is extended after reading data",
+ packetType1RTT, debugFrameMaxStreamData{
+ id: sid,
+ max: maxWindowSize * 2,
+ })
+ tc.writeFrames(packetType1RTT, debugFrameStream{
+ id: sid,
+ off: maxWindowSize,
+ data: make([]byte, maxWindowSize),
+ fin: true,
+ })
+ if n, err := s.ReadContext(ctx, buf); n != maxWindowSize || err != io.EOF {
+ t.Fatalf("s.ReadContext() = %v, %v; want %v, io.EOF", n, err, maxWindowSize)
+ }
+ tc.wantIdle("stream window is not extended after FIN")
+ })
+}
+
+func TestStreamReceiveViolatesStreamDataLimit(t *testing.T) {
+ // "A receiver MUST close the connection with an error of type FLOW_CONTROL_ERROR if
+ // the sender violates the advertised [...] stream data limits [...]"
+ // https://www.rfc-editor.org/rfc/rfc9000#section-4.1-8
+ testStreamTypes(t, "", func(t *testing.T, styp streamType) {
+ const maxStreamData = 10
+ for _, test := range []struct {
+ off int64
+ size int64
+ }{{
+ off: maxStreamData,
+ size: 1,
+ }, {
+ off: 0,
+ size: maxStreamData + 1,
+ }, {
+ off: maxStreamData - 1,
+ size: 2,
+ }} {
+ tc := newTestConn(t, serverSide, func(c *Config) {
+ c.MaxStreamReadBufferSize = maxStreamData
+ })
+ tc.handshake()
+ tc.ignoreFrame(frameTypeAck)
+ tc.writeFrames(packetType1RTT, debugFrameStream{
+ id: newStreamID(clientSide, styp, 0),
+ off: test.off,
+ data: make([]byte, test.size),
+ })
+ tc.wantFrame(
+ fmt.Sprintf("data [%v,%v) violates stream data limit and closes connection",
+ test.off, test.off+test.size),
+ packetType1RTT, debugFrameConnectionCloseTransport{
+ code: errFlowControl,
+ },
+ )
+ }
+ })
+}
+
+func TestStreamReceiveDuplicateDataDoesNotViolateLimits(t *testing.T) {
+ testStreamTypes(t, "", func(t *testing.T, styp streamType) {
+ const maxData = 10
+ tc := newTestConn(t, serverSide, func(c *Config) {
+ // TODO: Add connection-level maximum data here as well.
+ c.MaxStreamReadBufferSize = maxData
+ })
+ tc.handshake()
+ tc.ignoreFrame(frameTypeAck)
+ for i := 0; i < 3; i++ {
+ tc.writeFrames(packetType1RTT, debugFrameStream{
+ id: newStreamID(clientSide, styp, 0),
+ off: 0,
+ data: make([]byte, maxData),
+ })
+ tc.wantIdle(fmt.Sprintf("conn sends no frames after receiving data frame %v", i))
+ }
+ })
+}
+
+func finalSizeTest(t *testing.T, wantErr transportError, f func(tc *testConn, sid streamID) (finalSize int64), opts ...any) {
+ testStreamTypes(t, "", func(t *testing.T, styp streamType) {
+ for _, test := range []struct {
+ name string
+ finalFrame func(tc *testConn, sid streamID, finalSize int64)
+ }{{
+ name: "FIN",
+ finalFrame: func(tc *testConn, sid streamID, finalSize int64) {
+ tc.writeFrames(packetType1RTT, debugFrameStream{
+ id: sid,
+ off: finalSize,
+ fin: true,
+ })
+ },
+ }, {
+ name: "RESET_STREAM",
+ finalFrame: func(tc *testConn, sid streamID, finalSize int64) {
+ tc.writeFrames(packetType1RTT, debugFrameResetStream{
+ id: sid,
+ finalSize: finalSize,
+ })
+ },
+ }} {
+ t.Run(test.name, func(t *testing.T) {
+ tc := newTestConn(t, serverSide, opts...)
+ tc.handshake()
+ sid := newStreamID(clientSide, styp, 0)
+ finalSize := f(tc, sid)
+ test.finalFrame(tc, sid, finalSize)
+ tc.wantFrame("change in final size of stream is an error",
+ packetType1RTT, debugFrameConnectionCloseTransport{
+ code: wantErr,
+ },
+ )
+ })
+ }
+ })
+}
+
+func TestStreamFinalSizeChangedAfterFin(t *testing.T) {
+ // "If a RESET_STREAM or STREAM frame is received indicating a change
+ // in the final size for the stream, an endpoint SHOULD respond with
+ // an error of type FINAL_SIZE_ERROR [...]"
+ // https://www.rfc-editor.org/rfc/rfc9000#section-4.5-5
+ finalSizeTest(t, errFinalSize, func(tc *testConn, sid streamID) (finalSize int64) {
+ tc.writeFrames(packetType1RTT, debugFrameStream{
+ id: sid,
+ off: 10,
+ fin: true,
+ })
+ return 9
+ })
+}
+
+func TestStreamFinalSizeBeforePreviousData(t *testing.T) {
+ finalSizeTest(t, errFinalSize, func(tc *testConn, sid streamID) (finalSize int64) {
+ tc.writeFrames(packetType1RTT, debugFrameStream{
+ id: sid,
+ off: 10,
+ data: []byte{0},
+ })
+ return 9
+ })
+}
+
+func TestStreamFinalSizePastMaxStreamData(t *testing.T) {
+ finalSizeTest(t, errFlowControl, func(tc *testConn, sid streamID) (finalSize int64) {
+ return 11
+ }, func(c *Config) {
+ c.MaxStreamReadBufferSize = 10
+ })
+}
+
+func TestStreamDataBeyondFinalSize(t *testing.T) {
+ // "A receiver SHOULD treat receipt of data at or beyond
+ // the final size as an error of type FINAL_SIZE_ERROR [...]"
+ // https://www.rfc-editor.org/rfc/rfc9000#section-4.5-5
+ testStreamTypes(t, "", func(t *testing.T, styp streamType) {
+ tc := newTestConn(t, serverSide)
+ tc.handshake()
+ sid := newStreamID(clientSide, styp, 0)
+
+ const write1size = 4
+ tc.writeFrames(packetType1RTT, debugFrameStream{
+ id: sid,
+ off: 0,
+ data: make([]byte, 16),
+ fin: true,
+ })
+ tc.writeFrames(packetType1RTT, debugFrameStream{
+ id: sid,
+ off: 16,
+ data: []byte{0},
+ })
+ tc.wantFrame("received data past final size of stream",
+ packetType1RTT, debugFrameConnectionCloseTransport{
+ code: errFinalSize,
+ },
+ )
+ })
+}
+
+func TestStreamReceiveUnblocksReader(t *testing.T) {
+ testStreamTypes(t, "", func(t *testing.T, styp streamType) {
+ tc := newTestConn(t, serverSide)
+ tc.handshake()
+ want := []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}
+ sid := newStreamID(clientSide, styp, 0)
+
+ // AcceptStream blocks until a STREAM frame is received.
+ accept := runAsync(tc, func(ctx context.Context) (*Stream, error) {
+ return tc.conn.AcceptStream(ctx)
+ })
+ const write1size = 4
+ tc.writeFrames(packetType1RTT, debugFrameStream{
+ id: sid,
+ off: 0,
+ data: want[:write1size],
+ })
+ s, err := accept.result()
+ if err != nil {
+ t.Fatalf("AcceptStream() = %v", err)
+ }
+
+ // ReadContext succeeds immediately, since we already have data.
+ got := make([]byte, len(want))
+ read := runAsync(tc, func(ctx context.Context) (int, error) {
+ return s.ReadContext(ctx, got)
+ })
+ if n, err := read.result(); n != write1size || err != nil {
+ t.Fatalf("ReadContext = %v, %v; want %v, nil", n, err, write1size)
+ }
+
+ // ReadContext blocks waiting for more data.
+ read = runAsync(tc, func(ctx context.Context) (int, error) {
+ return s.ReadContext(ctx, got[write1size:])
+ })
+ tc.writeFrames(packetType1RTT, debugFrameStream{
+ id: sid,
+ off: write1size,
+ data: want[write1size:],
+ fin: true,
+ })
+ if n, err := read.result(); n != len(want)-write1size || err != io.EOF {
+ t.Fatalf("ReadContext = %v, %v; want %v, io.EOF", n, err, len(want)-write1size)
+ }
+ if !bytes.Equal(got, want) {
+ t.Fatalf("read bytes %x, want %x", got, want)
+ }
+ })
+}
+
+// testStreamSendFrameInvalidState calls the test func with a stream ID for:
+//
+// - a remote bidirectional stream that the peer has not created
+// - a remote unidirectional stream
+//
+// It then sends the returned frame (STREAM, STREAM_DATA_BLOCKED, etc.)
+// to the conn and expects a STREAM_STATE_ERROR.
+func testStreamSendFrameInvalidState(t *testing.T, f func(sid streamID) debugFrame) {
+ testSides(t, "stream_not_created", func(t *testing.T, side connSide) {
+ tc := newTestConn(t, side, permissiveTransportParameters)
+ tc.handshake()
+ tc.writeFrames(packetType1RTT, f(newStreamID(side, bidiStream, 0)))
+ tc.wantFrame("frame for local stream which has not been created",
+ packetType1RTT, debugFrameConnectionCloseTransport{
+ code: errStreamState,
+ })
+ })
+ testSides(t, "uni_stream", func(t *testing.T, side connSide) {
+ ctx := canceledContext()
+ tc := newTestConn(t, side, permissiveTransportParameters)
+ tc.handshake()
+ sid := newStreamID(side, uniStream, 0)
+ s, err := tc.conn.NewSendOnlyStream(ctx)
+ if err != nil {
+ t.Fatal(err)
+ }
+ s.Write(nil) // open the stream
+ tc.wantFrame("new stream is opened",
+ packetType1RTT, debugFrameStream{
+ id: sid,
+ data: []byte{},
+ })
+ tc.writeFrames(packetType1RTT, f(sid))
+ tc.wantFrame("send-oriented frame for send-only stream",
+ packetType1RTT, debugFrameConnectionCloseTransport{
+ code: errStreamState,
+ })
+ })
+}
+
+func TestStreamResetStreamInvalidState(t *testing.T) {
+ // "An endpoint that receives a RESET_STREAM frame for a send-only
+ // stream MUST terminate the connection with error STREAM_STATE_ERROR."
+ // https://www.rfc-editor.org/rfc/rfc9000#section-19.4-3
+ testStreamSendFrameInvalidState(t, func(sid streamID) debugFrame {
+ return debugFrameResetStream{
+ id: sid,
+ code: 0,
+ finalSize: 0,
+ }
+ })
+}
+
+func TestStreamStreamFrameInvalidState(t *testing.T) {
+ // "An endpoint MUST terminate the connection with error STREAM_STATE_ERROR
+ // if it receives a STREAM frame for a locally initiated stream
+ // that has not yet been created, or for a send-only stream."
+ // https://www.rfc-editor.org/rfc/rfc9000.html#section-19.8-3
+ testStreamSendFrameInvalidState(t, func(sid streamID) debugFrame {
+ return debugFrameStream{
+ id: sid,
+ }
+ })
+}
+
+func TestStreamDataBlockedInvalidState(t *testing.T) {
+ // "An endpoint MUST terminate the connection with error STREAM_STATE_ERROR
+ // if it receives a STREAM frame for a locally initiated stream
+ // that has not yet been created, or for a send-only stream."
+ // https://www.rfc-editor.org/rfc/rfc9000.html#section-19.8-3
+ testStreamSendFrameInvalidState(t, func(sid streamID) debugFrame {
+ return debugFrameStream{
+ id: sid,
+ }
+ })
+}
+
+// testStreamReceiveFrameInvalidState calls the test func with a stream ID for:
+//
+// - a remote bidirectional stream that the peer has not created
+// - a local unidirectional stream
+//
+// It then sends the returned frame (MAX_STREAM_DATA, STOP_SENDING, etc.)
+// to the conn and expects a STREAM_STATE_ERROR.
+func testStreamReceiveFrameInvalidState(t *testing.T, f func(sid streamID) debugFrame) {
+ testSides(t, "stream_not_created", func(t *testing.T, side connSide) {
+ tc := newTestConn(t, side)
+ tc.handshake()
+ tc.writeFrames(packetType1RTT, f(newStreamID(side, bidiStream, 0)))
+ tc.wantFrame("frame for local stream which has not been created",
+ packetType1RTT, debugFrameConnectionCloseTransport{
+ code: errStreamState,
+ })
+ })
+ testSides(t, "uni_stream", func(t *testing.T, side connSide) {
+ tc := newTestConn(t, side)
+ tc.handshake()
+ tc.writeFrames(packetType1RTT, f(newStreamID(side.peer(), uniStream, 0)))
+ tc.wantFrame("receive-oriented frame for receive-only stream",
+ packetType1RTT, debugFrameConnectionCloseTransport{
+ code: errStreamState,
+ })
+ })
+}
+
+func TestStreamStopSendingInvalidState(t *testing.T) {
+ // "Receiving a STOP_SENDING frame for a locally initiated stream
+ // that has not yet been created MUST be treated as a connection error
+ // of type STREAM_STATE_ERROR. An endpoint that receives a STOP_SENDING
+ // frame for a receive-only stream MUST terminate the connection with
+ // error STREAM_STATE_ERROR."
+ // https://www.rfc-editor.org/rfc/rfc9000#section-19.5-2
+ testStreamReceiveFrameInvalidState(t, func(sid streamID) debugFrame {
+ return debugFrameStopSending{
+ id: sid,
+ }
+ })
+}
+
+func TestStreamMaxStreamDataInvalidState(t *testing.T) {
+ // "Receiving a MAX_STREAM_DATA frame for a locally initiated stream
+ // that has not yet been created MUST be treated as a connection error
+ // of type STREAM_STATE_ERROR. An endpoint that receives a MAX_STREAM_DATA
+ // frame for a receive-only stream MUST terminate the connection
+ // with error STREAM_STATE_ERROR."
+ // https://www.rfc-editor.org/rfc/rfc9000#section-19.10-2
+ testStreamReceiveFrameInvalidState(t, func(sid streamID) debugFrame {
+ return debugFrameMaxStreamData{
+ id: sid,
+ max: 1000,
+ }
+ })
+}
+
+func TestStreamOffsetTooLarge(t *testing.T) {
+ // "Receipt of a frame that exceeds [2^62-1] MUST be treated as a
+ // connection error of type FRAME_ENCODING_ERROR or FLOW_CONTROL_ERROR."
+ // https://www.rfc-editor.org/rfc/rfc9000.html#section-19.8-9
+ tc := newTestConn(t, serverSide)
+ tc.handshake()
+
+ tc.writeFrames(packetType1RTT,
+ debugFrameStream{
+ id: newStreamID(clientSide, bidiStream, 0),
+ off: 1<<62 - 1,
+ data: []byte{0},
+ })
+ got, _ := tc.readFrame()
+ want1 := debugFrameConnectionCloseTransport{code: errFrameEncoding}
+ want2 := debugFrameConnectionCloseTransport{code: errFlowControl}
+ if !reflect.DeepEqual(got, want1) && !reflect.DeepEqual(got, want2) {
+ t.Fatalf("STREAM offset exceeds 2^62-1\ngot: %v\nwant: %v\n or: %v", got, want1, want2)
+ }
+}
+
+func TestStreamReadFromWriteOnlyStream(t *testing.T) {
+ _, s := newTestConnAndLocalStream(t, serverSide, uniStream, permissiveTransportParameters)
+ buf := make([]byte, 10)
+ wantErr := "read from write-only stream"
+ if n, err := s.Read(buf); err == nil || !strings.Contains(err.Error(), wantErr) {
+ t.Errorf("s.Read() = %v, %v; want error %q", n, err, wantErr)
+ }
+}
+
+func TestStreamWriteToReadOnlyStream(t *testing.T) {
+ _, s := newTestConnAndRemoteStream(t, serverSide, uniStream)
+ buf := make([]byte, 10)
+ wantErr := "write to read-only stream"
+ if n, err := s.Write(buf); err == nil || !strings.Contains(err.Error(), wantErr) {
+ t.Errorf("s.Write() = %v, %v; want error %q", n, err, wantErr)
+ }
+}
+
+func TestStreamReadFromClosedStream(t *testing.T) {
+ tc, s := newTestConnAndRemoteStream(t, serverSide, bidiStream, permissiveTransportParameters)
+ s.CloseRead()
+ tc.wantFrame("CloseRead sends a STOP_SENDING frame",
+ packetType1RTT, debugFrameStopSending{
+ id: s.id,
+ })
+ wantErr := "read from closed stream"
+ if n, err := s.Read(make([]byte, 16)); err == nil || !strings.Contains(err.Error(), wantErr) {
+ t.Errorf("s.Read() = %v, %v; want error %q", n, err, wantErr)
+ }
+ // Data which shows up after STOP_SENDING is discarded.
+ tc.writeFrames(packetType1RTT, debugFrameStream{
+ id: s.id,
+ data: []byte{1, 2, 3},
+ fin: true,
+ })
+ if n, err := s.Read(make([]byte, 16)); err == nil || !strings.Contains(err.Error(), wantErr) {
+ t.Errorf("s.Read() = %v, %v; want error %q", n, err, wantErr)
+ }
+}
+
+func TestStreamCloseReadWithAllDataReceived(t *testing.T) {
+ tc, s := newTestConnAndRemoteStream(t, serverSide, bidiStream, permissiveTransportParameters)
+ tc.writeFrames(packetType1RTT, debugFrameStream{
+ id: s.id,
+ data: []byte{1, 2, 3},
+ fin: true,
+ })
+ s.CloseRead()
+ tc.wantIdle("CloseRead in Data Recvd state doesn't need to send STOP_SENDING")
+ // We had all the data for the stream, but CloseRead discarded it.
+ wantErr := "read from closed stream"
+ if n, err := s.Read(make([]byte, 16)); err == nil || !strings.Contains(err.Error(), wantErr) {
+ t.Errorf("s.Read() = %v, %v; want error %q", n, err, wantErr)
+ }
+}
+
+func TestStreamWriteToClosedStream(t *testing.T) {
+ tc, s := newTestConnAndLocalStream(t, serverSide, bidiStream, permissiveTransportParameters)
+ s.CloseWrite()
+ tc.wantFrame("stream is opened after being closed",
+ packetType1RTT, debugFrameStream{
+ id: s.id,
+ off: 0,
+ fin: true,
+ data: []byte{},
+ })
+ wantErr := "write to closed stream"
+ if n, err := s.Write([]byte{}); err == nil || !strings.Contains(err.Error(), wantErr) {
+ t.Errorf("s.Write() = %v, %v; want error %q", n, err, wantErr)
+ }
+}
+
+func TestStreamResetBlockedStream(t *testing.T) {
+ tc, s := newTestConnAndLocalStream(t, serverSide, bidiStream, permissiveTransportParameters,
+ func(c *Config) {
+ c.MaxStreamWriteBufferSize = 4
+ })
+ tc.ignoreFrame(frameTypeStreamDataBlocked)
+ writing := runAsync(tc, func(ctx context.Context) (int, error) {
+ return s.WriteContext(ctx, []byte{0, 1, 2, 3, 4, 5, 6, 7})
+ })
+ tc.wantFrame("stream writes data until write buffer fills",
+ packetType1RTT, debugFrameStream{
+ id: s.id,
+ off: 0,
+ data: []byte{0, 1, 2, 3},
+ })
+ s.Reset(42)
+ tc.wantFrame("stream is reset",
+ packetType1RTT, debugFrameResetStream{
+ id: s.id,
+ code: 42,
+ finalSize: 4,
+ })
+ wantErr := "write to reset stream"
+ if n, err := writing.result(); n != 4 || !strings.Contains(err.Error(), wantErr) {
+ t.Errorf("s.Write() interrupted by Reset: %v, %q; want 4, %q", n, err, wantErr)
+ }
+ tc.writeAckForAll()
+ tc.wantIdle("buffer space is available, but stream has been reset")
+ s.Reset(100)
+ tc.wantIdle("resetting stream a second time has no effect")
+ if n, err := s.Write([]byte{}); err == nil || !strings.Contains(err.Error(), wantErr) {
+ t.Errorf("s.Write() = %v, %v; want error %q", n, err, wantErr)
+ }
+}
+
+func TestStreamWriteMoreThanOnePacketOfData(t *testing.T) {
+ tc, s := newTestConnAndLocalStream(t, serverSide, uniStream, func(p *transportParameters) {
+ p.initialMaxStreamsUni = 1
+ p.initialMaxData = 1 << 20
+ p.initialMaxStreamDataUni = 1 << 20
+ })
+ want := make([]byte, 4096)
+ rand.Read(want) // doesn't need to be crypto/rand, but non-deprecated and harmless
+ w := runAsync(tc, func(ctx context.Context) (int, error) {
+ return s.WriteContext(ctx, want)
+ })
+ got := make([]byte, 0, len(want))
+ for {
+ f, _ := tc.readFrame()
+ if f == nil {
+ break
+ }
+ sf, ok := f.(debugFrameStream)
+ if !ok {
+ t.Fatalf("unexpected frame: %v", sf)
+ }
+ if len(got) != int(sf.off) {
+ t.Fatalf("got frame: %v\nwant offset %v", sf, len(got))
+ }
+ got = append(got, sf.data...)
+ }
+ if n, err := w.result(); n != len(want) || err != nil {
+ t.Fatalf("s.WriteContext() = %v, %v; want %v, nil", n, err, len(want))
+ }
+ if !bytes.Equal(got, want) {
+ t.Fatalf("mismatch in received stream data")
+ }
+}
+
+func TestStreamCloseWaitsForAcks(t *testing.T) {
+ ctx := canceledContext()
+ tc, s := newTestConnAndLocalStream(t, serverSide, uniStream, permissiveTransportParameters)
+ data := make([]byte, 100)
+ s.WriteContext(ctx, data)
+ tc.wantFrame("conn sends data for the stream",
+ packetType1RTT, debugFrameStream{
+ id: s.id,
+ data: data,
+ })
+ if err := s.CloseContext(ctx); err != context.Canceled {
+ t.Fatalf("s.Close() = %v, want context.Canceled (data not acked yet)", err)
+ }
+ tc.wantFrame("conn sends FIN for closed stream",
+ packetType1RTT, debugFrameStream{
+ id: s.id,
+ off: int64(len(data)),
+ fin: true,
+ data: []byte{},
+ })
+ closing := runAsync(tc, func(ctx context.Context) (struct{}, error) {
+ return struct{}{}, s.CloseContext(ctx)
+ })
+ if _, err := closing.result(); err != errNotDone {
+ t.Fatalf("s.CloseContext() = %v, want it to block waiting for acks", err)
+ }
+ tc.writeAckForAll()
+ if _, err := closing.result(); err != nil {
+ t.Fatalf("s.CloseContext() = %v, want nil (all data acked)", err)
+ }
+}
+
+func TestStreamCloseReadOnly(t *testing.T) {
+ tc, s := newTestConnAndRemoteStream(t, serverSide, uniStream, permissiveTransportParameters)
+ if err := s.CloseContext(canceledContext()); err != nil {
+ t.Errorf("s.CloseContext() = %v, want nil", err)
+ }
+ tc.wantFrame("closed stream sends STOP_SENDING",
+ packetType1RTT, debugFrameStopSending{
+ id: s.id,
+ })
+}
+
+func TestStreamCloseUnblocked(t *testing.T) {
+ for _, test := range []struct {
+ name string
+ unblock func(tc *testConn, s *Stream)
+ }{{
+ name: "data received",
+ unblock: func(tc *testConn, s *Stream) {
+ tc.writeAckForAll()
+ },
+ }, {
+ name: "stop sending received",
+ unblock: func(tc *testConn, s *Stream) {
+ tc.writeFrames(packetType1RTT, debugFrameStopSending{
+ id: s.id,
+ })
+ },
+ }, {
+ name: "stream reset",
+ unblock: func(tc *testConn, s *Stream) {
+ s.Reset(0)
+ tc.wait() // wait for test conn to process the Reset
+ },
+ }} {
+ t.Run(test.name, func(t *testing.T) {
+ ctx := canceledContext()
+ tc, s := newTestConnAndLocalStream(t, serverSide, uniStream, permissiveTransportParameters)
+ data := make([]byte, 100)
+ s.WriteContext(ctx, data)
+ tc.wantFrame("conn sends data for the stream",
+ packetType1RTT, debugFrameStream{
+ id: s.id,
+ data: data,
+ })
+ if err := s.CloseContext(ctx); err != context.Canceled {
+ t.Fatalf("s.Close() = %v, want context.Canceled (data not acked yet)", err)
+ }
+ tc.wantFrame("conn sends FIN for closed stream",
+ packetType1RTT, debugFrameStream{
+ id: s.id,
+ off: int64(len(data)),
+ fin: true,
+ data: []byte{},
+ })
+ closing := runAsync(tc, func(ctx context.Context) (struct{}, error) {
+ return struct{}{}, s.CloseContext(ctx)
+ })
+ if _, err := closing.result(); err != errNotDone {
+ t.Fatalf("s.CloseContext() = %v, want it to block waiting for acks", err)
+ }
+ test.unblock(tc, s)
+ if _, err := closing.result(); err != nil {
+ t.Fatalf("s.CloseContext() = %v, want nil (all data acked)", err)
+ }
+ })
+ }
+}
+
+func TestStreamCloseWriteWhenBlockedByStreamFlowControl(t *testing.T) {
+ ctx := canceledContext()
+ tc, s := newTestConnAndLocalStream(t, serverSide, uniStream, permissiveTransportParameters,
+ func(p *transportParameters) {
+ //p.initialMaxData = 0
+ p.initialMaxStreamDataUni = 0
+ })
+ tc.ignoreFrame(frameTypeStreamDataBlocked)
+ if _, err := s.WriteContext(ctx, []byte{0, 1}); err != nil {
+ t.Fatalf("s.Write = %v", err)
+ }
+ s.CloseWrite()
+ tc.wantIdle("stream write is blocked by flow control")
+
+ tc.writeFrames(packetType1RTT, debugFrameMaxStreamData{
+ id: s.id,
+ max: 1,
+ })
+ tc.wantFrame("send data up to flow control limit",
+ packetType1RTT, debugFrameStream{
+ id: s.id,
+ data: []byte{0},
+ })
+ tc.wantIdle("stream write is again blocked by flow control")
+
+ tc.writeFrames(packetType1RTT, debugFrameMaxStreamData{
+ id: s.id,
+ max: 2,
+ })
+ tc.wantFrame("send remaining data and FIN",
+ packetType1RTT, debugFrameStream{
+ id: s.id,
+ off: 1,
+ data: []byte{1},
+ fin: true,
+ })
+}
+
+func TestStreamPeerResetsWithUnreadAndUnsentData(t *testing.T) {
+ testStreamTypes(t, "", func(t *testing.T, styp streamType) {
+ ctx := canceledContext()
+ tc, s := newTestConnAndRemoteStream(t, serverSide, styp)
+ data := []byte{0, 1, 2, 3, 4, 5, 6, 7}
+ tc.writeFrames(packetType1RTT, debugFrameStream{
+ id: s.id,
+ data: data,
+ })
+ got := make([]byte, 4)
+ if n, err := s.ReadContext(ctx, got); n != len(got) || err != nil {
+ t.Fatalf("Read start of stream: got %v, %v; want %v, nil", n, err, len(got))
+ }
+ const sentCode = 42
+ tc.writeFrames(packetType1RTT, debugFrameResetStream{
+ id: s.id,
+ finalSize: 20,
+ code: sentCode,
+ })
+ wantErr := StreamErrorCode(sentCode)
+ if n, err := s.ReadContext(ctx, got); n != 0 || !errors.Is(err, wantErr) {
+ t.Fatalf("Read reset stream: got %v, %v; want 0, %v", n, err, wantErr)
+ }
+ })
+}
+
+func TestStreamPeerResetWakesBlockedRead(t *testing.T) {
+ testStreamTypes(t, "", func(t *testing.T, styp streamType) {
+ tc, s := newTestConnAndRemoteStream(t, serverSide, styp)
+ reader := runAsync(tc, func(ctx context.Context) (int, error) {
+ got := make([]byte, 4)
+ return s.ReadContext(ctx, got)
+ })
+ const sentCode = 42
+ tc.writeFrames(packetType1RTT, debugFrameResetStream{
+ id: s.id,
+ finalSize: 20,
+ code: sentCode,
+ })
+ wantErr := StreamErrorCode(sentCode)
+ if n, err := reader.result(); n != 0 || !errors.Is(err, wantErr) {
+ t.Fatalf("Read reset stream: got %v, %v; want 0, %v", n, err, wantErr)
+ }
+ })
+}
+
+func TestStreamPeerResetFollowedByData(t *testing.T) {
+ testStreamTypes(t, "", func(t *testing.T, styp streamType) {
+ tc, s := newTestConnAndRemoteStream(t, serverSide, styp)
+ tc.writeFrames(packetType1RTT, debugFrameResetStream{
+ id: s.id,
+ finalSize: 4,
+ code: 1,
+ })
+ tc.writeFrames(packetType1RTT, debugFrameStream{
+ id: s.id,
+ data: []byte{0, 1, 2, 3},
+ })
+ // Another reset with a different code, for good measure.
+ tc.writeFrames(packetType1RTT, debugFrameResetStream{
+ id: s.id,
+ finalSize: 4,
+ code: 2,
+ })
+ wantErr := StreamErrorCode(1)
+ if n, err := s.Read(make([]byte, 16)); n != 0 || !errors.Is(err, wantErr) {
+ t.Fatalf("Read from reset stream: got %v, %v; want 0, %v", n, err, wantErr)
+ }
+ })
+}
+
+func TestStreamResetInvalidCode(t *testing.T) {
+ tc, s := newTestConnAndLocalStream(t, serverSide, uniStream, permissiveTransportParameters)
+ s.Reset(1 << 62)
+ tc.wantFrame("reset with invalid code sends a RESET_STREAM anyway",
+ packetType1RTT, debugFrameResetStream{
+ id: s.id,
+ // The code we send here isn't specified,
+ // so this could really be any value.
+ code: (1 << 62) - 1,
+ })
+}
+
+func TestStreamResetReceiveOnly(t *testing.T) {
+ tc, s := newTestConnAndRemoteStream(t, serverSide, uniStream)
+ s.Reset(0)
+ tc.wantIdle("resetting a receive-only stream has no effect")
+}
+
+func TestStreamPeerStopSendingForActiveStream(t *testing.T) {
+ // "An endpoint that receives a STOP_SENDING frame MUST send a RESET_STREAM frame if
+ // the stream is in the "Ready" or "Send" state."
+ // https://www.rfc-editor.org/rfc/rfc9000#section-3.5-4
+ testStreamTypes(t, "", func(t *testing.T, styp streamType) {
+ tc, s := newTestConnAndLocalStream(t, serverSide, styp, permissiveTransportParameters)
+ for i := 0; i < 4; i++ {
+ s.Write([]byte{byte(i)})
+ tc.wantFrame("write sends a STREAM frame to peer",
+ packetType1RTT, debugFrameStream{
+ id: s.id,
+ off: int64(i),
+ data: []byte{byte(i)},
+ })
+ }
+ tc.writeFrames(packetType1RTT, debugFrameStopSending{
+ id: s.id,
+ code: 42,
+ })
+ tc.wantFrame("receiving STOP_SENDING causes stream reset",
+ packetType1RTT, debugFrameResetStream{
+ id: s.id,
+ code: 42,
+ finalSize: 4,
+ })
+ if n, err := s.Write([]byte{0}); err == nil {
+ t.Errorf("s.Write() after STOP_SENDING = %v, %v; want error", n, err)
+ }
+ // This ack will result in some of the previous frames being marked as lost.
+ tc.writeAckForLatest()
+ tc.wantIdle("lost STREAM frames for reset stream are not resent")
+ })
+}
+
+func TestStreamReceiveDataBlocked(t *testing.T) {
+ tc := newTestConn(t, serverSide, permissiveTransportParameters)
+ tc.handshake()
+ tc.ignoreFrame(frameTypeAck)
+
+ // We don't do anything with these frames,
+ // but should accept them if the peer sends one.
+ tc.writeFrames(packetType1RTT, debugFrameStreamDataBlocked{
+ id: newStreamID(clientSide, bidiStream, 0),
+ max: 100,
+ })
+ tc.writeFrames(packetType1RTT, debugFrameDataBlocked{
+ max: 100,
+ })
+ tc.wantIdle("no response to STREAM_DATA_BLOCKED and DATA_BLOCKED")
+}
+
+type streamSide string
+
+const (
+ localStream = streamSide("local")
+ remoteStream = streamSide("remote")
+)
+
+func newTestConnAndStream(t *testing.T, side connSide, sside streamSide, styp streamType, opts ...any) (*testConn, *Stream) {
+ if sside == localStream {
+ return newTestConnAndLocalStream(t, side, styp, opts...)
+ } else {
+ return newTestConnAndRemoteStream(t, side, styp, opts...)
+ }
+}
+
+func newTestConnAndLocalStream(t *testing.T, side connSide, styp streamType, opts ...any) (*testConn, *Stream) {
+ t.Helper()
+ ctx := canceledContext()
+ tc := newTestConn(t, side, opts...)
+ tc.handshake()
+ tc.ignoreFrame(frameTypeAck)
+ s, err := tc.conn.newLocalStream(ctx, styp)
+ if err != nil {
+ t.Fatalf("conn.newLocalStream(%v) = %v", styp, err)
+ }
+ return tc, s
+}
+
+func newTestConnAndRemoteStream(t *testing.T, side connSide, styp streamType, opts ...any) (*testConn, *Stream) {
+ t.Helper()
+ ctx := canceledContext()
+ tc := newTestConn(t, side, opts...)
+ tc.handshake()
+ tc.ignoreFrame(frameTypeAck)
+ tc.writeFrames(packetType1RTT, debugFrameStream{
+ id: newStreamID(side.peer(), styp, 0),
+ })
+ s, err := tc.conn.AcceptStream(ctx)
+ if err != nil {
+ t.Fatalf("conn.AcceptStream() = %v", err)
+ }
+ return tc, s
+}
+
+// permissiveTransportParameters may be passed as an option to newTestConn.
+func permissiveTransportParameters(p *transportParameters) {
+ p.initialMaxStreamsBidi = maxStreamsLimit
+ p.initialMaxStreamsUni = maxStreamsLimit
+ p.initialMaxData = maxVarint
+ p.initialMaxStreamDataBidiRemote = maxVarint
+ p.initialMaxStreamDataBidiLocal = maxVarint
+ p.initialMaxStreamDataUni = maxVarint
+}
+
+func makeTestData(n int) []byte {
+ b := make([]byte, n)
+ for i := 0; i < n; i++ {
+ b[i] = byte(i)
+ }
+ return b
+}
diff --git a/internal/quic/tls.go b/internal/quic/tls.go
new file mode 100644
index 0000000000..a37e26fb8e
--- /dev/null
+++ b/internal/quic/tls.go
@@ -0,0 +1,119 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build go1.21
+
+package quic
+
+import (
+ "context"
+ "crypto/tls"
+ "errors"
+ "fmt"
+ "time"
+)
+
+// startTLS starts the TLS handshake.
+func (c *Conn) startTLS(now time.Time, initialConnID []byte, params transportParameters) error {
+ c.keysInitial = initialKeys(initialConnID, c.side)
+
+ qconfig := &tls.QUICConfig{TLSConfig: c.config.TLSConfig}
+ if c.side == clientSide {
+ c.tls = tls.QUICClient(qconfig)
+ } else {
+ c.tls = tls.QUICServer(qconfig)
+ }
+ c.tls.SetTransportParameters(marshalTransportParameters(params))
+ // TODO: We don't need or want a context for cancelation here,
+ // but users can use a context to plumb values through to hooks defined
+ // in the tls.Config. Pass through a context.
+ if err := c.tls.Start(context.TODO()); err != nil {
+ return err
+ }
+ return c.handleTLSEvents(now)
+}
+
+func (c *Conn) handleTLSEvents(now time.Time) error {
+ for {
+ e := c.tls.NextEvent()
+ if c.testHooks != nil {
+ c.testHooks.handleTLSEvent(e)
+ }
+ switch e.Kind {
+ case tls.QUICNoEvent:
+ return nil
+ case tls.QUICSetReadSecret:
+ if err := checkCipherSuite(e.Suite); err != nil {
+ return err
+ }
+ switch e.Level {
+ case tls.QUICEncryptionLevelHandshake:
+ c.keysHandshake.r.init(e.Suite, e.Data)
+ case tls.QUICEncryptionLevelApplication:
+ c.keysAppData.r.init(e.Suite, e.Data)
+ }
+ case tls.QUICSetWriteSecret:
+ if err := checkCipherSuite(e.Suite); err != nil {
+ return err
+ }
+ switch e.Level {
+ case tls.QUICEncryptionLevelHandshake:
+ c.keysHandshake.w.init(e.Suite, e.Data)
+ case tls.QUICEncryptionLevelApplication:
+ c.keysAppData.w.init(e.Suite, e.Data)
+ }
+ case tls.QUICWriteData:
+ var space numberSpace
+ switch e.Level {
+ case tls.QUICEncryptionLevelInitial:
+ space = initialSpace
+ case tls.QUICEncryptionLevelHandshake:
+ space = handshakeSpace
+ case tls.QUICEncryptionLevelApplication:
+ space = appDataSpace
+ default:
+ return fmt.Errorf("quic: internal error: write handshake data at level %v", e.Level)
+ }
+ c.crypto[space].write(e.Data)
+ case tls.QUICHandshakeDone:
+ if c.side == serverSide {
+ // "[...] the TLS handshake is considered confirmed
+ // at the server when the handshake completes."
+ // https://www.rfc-editor.org/rfc/rfc9001#section-4.1.2-1
+ c.confirmHandshake(now)
+ }
+ c.handshakeDone()
+ case tls.QUICTransportParameters:
+ params, err := unmarshalTransportParams(e.Data)
+ if err != nil {
+ return err
+ }
+ if err := c.receiveTransportParameters(params); err != nil {
+ return err
+ }
+ }
+ }
+}
+
+// handleCrypto processes data received in a CRYPTO frame.
+func (c *Conn) handleCrypto(now time.Time, space numberSpace, off int64, data []byte) error {
+ var level tls.QUICEncryptionLevel
+ switch space {
+ case initialSpace:
+ level = tls.QUICEncryptionLevelInitial
+ case handshakeSpace:
+ level = tls.QUICEncryptionLevelHandshake
+ case appDataSpace:
+ level = tls.QUICEncryptionLevelApplication
+ default:
+ return errors.New("quic: internal error: received CRYPTO frame in unexpected number space")
+ }
+ err := c.crypto[space].handleCrypto(off, data, func(b []byte) error {
+ return c.tls.HandleData(level, b)
+ })
+ if err != nil {
+ return err
+ }
+ return c.handleTLSEvents(now)
+}
diff --git a/internal/quic/tls_test.go b/internal/quic/tls_test.go
new file mode 100644
index 0000000000..81d17b8587
--- /dev/null
+++ b/internal/quic/tls_test.go
@@ -0,0 +1,603 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build go1.21
+
+package quic
+
+import (
+ "crypto/tls"
+ "crypto/x509"
+ "errors"
+ "reflect"
+ "testing"
+ "time"
+)
+
+// handshake executes the handshake.
+func (tc *testConn) handshake() {
+ tc.t.Helper()
+ if *testVV {
+ *testVV = false
+ defer func() {
+ tc.t.Helper()
+ *testVV = true
+ tc.t.Logf("performed connection handshake")
+ }()
+ }
+ defer func(saved map[byte]bool) {
+ tc.ignoreFrames = saved
+ }(tc.ignoreFrames)
+ tc.ignoreFrames = nil
+ t := tc.t
+ dgrams := handshakeDatagrams(tc)
+ i := 0
+ for {
+ if i == len(dgrams)-1 {
+ if tc.conn.side == clientSide {
+ want := tc.now.Add(maxAckDelay - timerGranularity)
+ if !tc.timer.Equal(want) {
+ t.Fatalf("want timer = %v (max_ack_delay), got %v", want, tc.timer)
+ }
+ if got := tc.readDatagram(); got != nil {
+ t.Fatalf("client unexpectedly sent: %v", got)
+ }
+ }
+ tc.advance(maxAckDelay)
+ }
+
+ // Check that we're sending exactly the data we expect.
+ // Any variation from the norm here should be intentional.
+ got := tc.readDatagram()
+ var want *testDatagram
+ if !(tc.conn.side == serverSide && i == 0) && i < len(dgrams) {
+ want = dgrams[i]
+ fillCryptoFrames(want, tc.cryptoDataOut)
+ i++
+ }
+ if !reflect.DeepEqual(got, want) {
+ t.Fatalf("dgram %v:\ngot %v\n\nwant %v", i, got, want)
+ }
+ if i >= len(dgrams) {
+ break
+ }
+
+ fillCryptoFrames(dgrams[i], tc.cryptoDataIn)
+ tc.write(dgrams[i])
+ i++
+ }
+}
+
+func handshakeDatagrams(tc *testConn) (dgrams []*testDatagram) {
+ var (
+ clientConnIDs [][]byte
+ serverConnIDs [][]byte
+ transientConnID []byte
+ )
+ localConnIDs := [][]byte{
+ testLocalConnID(0),
+ testLocalConnID(1),
+ }
+ peerConnIDs := [][]byte{
+ testPeerConnID(0),
+ testPeerConnID(1),
+ }
+ if tc.conn.side == clientSide {
+ clientConnIDs = localConnIDs
+ serverConnIDs = peerConnIDs
+ transientConnID = testLocalConnID(-1)
+ } else {
+ clientConnIDs = peerConnIDs
+ serverConnIDs = localConnIDs
+ transientConnID = []byte{0xde, 0xad, 0xbe, 0xef}
+ }
+ return []*testDatagram{{
+ // Client Initial
+ packets: []*testPacket{{
+ ptype: packetTypeInitial,
+ num: 0,
+ version: quicVersion1,
+ srcConnID: clientConnIDs[0],
+ dstConnID: transientConnID,
+ frames: []debugFrame{
+ debugFrameCrypto{},
+ },
+ }},
+ paddedSize: 1200,
+ }, {
+ // Server Initial + Handshake + 1-RTT
+ packets: []*testPacket{{
+ ptype: packetTypeInitial,
+ num: 0,
+ version: quicVersion1,
+ srcConnID: serverConnIDs[0],
+ dstConnID: clientConnIDs[0],
+ frames: []debugFrame{
+ debugFrameAck{
+ ranges: []i64range[packetNumber]{{0, 1}},
+ },
+ debugFrameCrypto{},
+ },
+ }, {
+ ptype: packetTypeHandshake,
+ num: 0,
+ version: quicVersion1,
+ srcConnID: serverConnIDs[0],
+ dstConnID: clientConnIDs[0],
+ frames: []debugFrame{
+ debugFrameCrypto{},
+ },
+ }, {
+ ptype: packetType1RTT,
+ num: 0,
+ dstConnID: clientConnIDs[0],
+ frames: []debugFrame{
+ debugFrameNewConnectionID{
+ seq: 1,
+ connID: serverConnIDs[1],
+ },
+ },
+ }},
+ }, {
+ // Client Initial + Handshake + 1-RTT
+ packets: []*testPacket{{
+ ptype: packetTypeInitial,
+ num: 1,
+ version: quicVersion1,
+ srcConnID: clientConnIDs[0],
+ dstConnID: serverConnIDs[0],
+ frames: []debugFrame{
+ debugFrameAck{
+ ranges: []i64range[packetNumber]{{0, 1}},
+ },
+ },
+ }, {
+ ptype: packetTypeHandshake,
+ num: 0,
+ version: quicVersion1,
+ srcConnID: clientConnIDs[0],
+ dstConnID: serverConnIDs[0],
+ frames: []debugFrame{
+ debugFrameAck{
+ ranges: []i64range[packetNumber]{{0, 1}},
+ },
+ debugFrameCrypto{},
+ },
+ }, {
+ ptype: packetType1RTT,
+ num: 0,
+ dstConnID: serverConnIDs[0],
+ frames: []debugFrame{
+ debugFrameAck{
+ ranges: []i64range[packetNumber]{{0, 1}},
+ },
+ debugFrameNewConnectionID{
+ seq: 1,
+ connID: clientConnIDs[1],
+ },
+ },
+ }},
+ paddedSize: 1200,
+ }, {
+ // Server HANDSHAKE_DONE
+ packets: []*testPacket{{
+ ptype: packetType1RTT,
+ num: 1,
+ dstConnID: clientConnIDs[0],
+ frames: []debugFrame{
+ debugFrameAck{
+ ranges: []i64range[packetNumber]{{0, 1}},
+ },
+ debugFrameHandshakeDone{},
+ },
+ }},
+ }, {
+ // Client ack (after max_ack_delay)
+ packets: []*testPacket{{
+ ptype: packetType1RTT,
+ num: 1,
+ dstConnID: serverConnIDs[0],
+ frames: []debugFrame{
+ debugFrameAck{
+ ackDelay: unscaledAckDelayFromDuration(
+ maxAckDelay, ackDelayExponent),
+ ranges: []i64range[packetNumber]{{0, 2}},
+ },
+ },
+ }},
+ }}
+}
+
+func fillCryptoFrames(d *testDatagram, data map[tls.QUICEncryptionLevel][]byte) {
+ for _, p := range d.packets {
+ var level tls.QUICEncryptionLevel
+ switch p.ptype {
+ case packetTypeInitial:
+ level = tls.QUICEncryptionLevelInitial
+ case packetTypeHandshake:
+ level = tls.QUICEncryptionLevelHandshake
+ case packetType1RTT:
+ level = tls.QUICEncryptionLevelApplication
+ default:
+ continue
+ }
+ for i := range p.frames {
+ c, ok := p.frames[i].(debugFrameCrypto)
+ if !ok {
+ continue
+ }
+ c.data = data[level]
+ data[level] = nil
+ p.frames[i] = c
+ }
+ }
+}
+
+// uncheckedHandshake executes the handshake.
+//
+// Unlike testConn.handshake, it sends nothing unnecessary
+// (in particular, no NEW_CONNECTION_ID frames),
+// and does not validate the conn's responses.
+//
+// Useful for testing scenarios where configuration has
+// changed the handshake responses in some way.
+func (tc *testConn) uncheckedHandshake() {
+ tc.t.Helper()
+ defer func(saved map[byte]bool) {
+ tc.ignoreFrames = saved
+ }(tc.ignoreFrames)
+ tc.ignoreFrames = map[byte]bool{
+ frameTypeAck: true,
+ frameTypeCrypto: true,
+ frameTypeNewConnectionID: true,
+ }
+ if tc.conn.side == serverSide {
+ tc.writeFrames(packetTypeInitial,
+ debugFrameCrypto{
+ data: tc.cryptoDataIn[tls.QUICEncryptionLevelInitial],
+ })
+ tc.writeFrames(packetTypeHandshake,
+ debugFrameCrypto{
+ data: tc.cryptoDataIn[tls.QUICEncryptionLevelHandshake],
+ })
+ tc.wantFrame("send HANDSHAKE_DONE after handshake completes",
+ packetType1RTT, debugFrameHandshakeDone{})
+ tc.writeFrames(packetType1RTT,
+ debugFrameAck{
+ ackDelay: unscaledAckDelayFromDuration(
+ maxAckDelay, ackDelayExponent),
+ ranges: []i64range[packetNumber]{{0, tc.lastPacket.num + 1}},
+ })
+ } else {
+ tc.wantIdle("initial frames are ignored")
+ tc.writeFrames(packetTypeInitial,
+ debugFrameCrypto{
+ data: tc.cryptoDataIn[tls.QUICEncryptionLevelInitial],
+ })
+ tc.writeFrames(packetTypeHandshake,
+ debugFrameCrypto{
+ data: tc.cryptoDataIn[tls.QUICEncryptionLevelHandshake],
+ })
+ tc.wantIdle("don't expect any frames we aren't ignoring")
+ // Send the next two frames in separate packets, so the client sends an
+ // ack immediately without delay. We want to consume that ack here, rather
+ // than returning with a delayed ack waiting to be sent.
+ tc.ignoreFrames = nil
+ tc.writeFrames(packetType1RTT,
+ debugFrameHandshakeDone{})
+ tc.writeFrames(packetType1RTT,
+ debugFrameCrypto{
+ data: tc.cryptoDataIn[tls.QUICEncryptionLevelApplication],
+ })
+ tc.wantFrame("client ACKs server's first 1-RTT packet",
+ packetType1RTT, debugFrameAck{
+ ranges: []i64range[packetNumber]{{0, 2}},
+ })
+
+ }
+ tc.wantIdle("handshake is done")
+}
+
+func TestConnClientHandshake(t *testing.T) {
+ tc := newTestConn(t, clientSide)
+ tc.handshake()
+ tc.advance(1 * time.Second)
+ tc.wantIdle("no packets should be sent by an idle conn after the handshake")
+}
+
+func TestConnServerHandshake(t *testing.T) {
+ tc := newTestConn(t, serverSide)
+ tc.handshake()
+ tc.advance(1 * time.Second)
+ tc.wantIdle("no packets should be sent by an idle conn after the handshake")
+}
+
+func TestConnKeysDiscardedClient(t *testing.T) {
+ tc := newTestConn(t, clientSide)
+ tc.ignoreFrame(frameTypeAck)
+
+ tc.wantFrame("client sends Initial CRYPTO frame",
+ packetTypeInitial, debugFrameCrypto{
+ data: tc.cryptoDataOut[tls.QUICEncryptionLevelInitial],
+ })
+ tc.writeFrames(packetTypeInitial,
+ debugFrameCrypto{
+ data: tc.cryptoDataIn[tls.QUICEncryptionLevelInitial],
+ })
+ tc.writeFrames(packetTypeHandshake,
+ debugFrameCrypto{
+ data: tc.cryptoDataIn[tls.QUICEncryptionLevelHandshake],
+ })
+ tc.wantFrame("client sends Handshake CRYPTO frame",
+ packetTypeHandshake, debugFrameCrypto{
+ data: tc.cryptoDataOut[tls.QUICEncryptionLevelHandshake],
+ })
+ tc.wantFrame("client provides an additional connection ID",
+ packetType1RTT, debugFrameNewConnectionID{
+ seq: 1,
+ connID: testLocalConnID(1),
+ })
+
+ // The client discards Initial keys after sending a Handshake packet.
+ tc.writeFrames(packetTypeInitial,
+ debugFrameConnectionCloseTransport{code: errInternal})
+ tc.wantIdle("client has discarded Initial keys, cannot read CONNECTION_CLOSE")
+
+ // The client discards Handshake keys after receiving a HANDSHAKE_DONE frame.
+ tc.writeFrames(packetType1RTT,
+ debugFrameHandshakeDone{})
+ tc.writeFrames(packetTypeHandshake,
+ debugFrameConnectionCloseTransport{code: errInternal})
+ tc.wantIdle("client has discarded Handshake keys, cannot read CONNECTION_CLOSE")
+
+ tc.writeFrames(packetType1RTT,
+ debugFrameConnectionCloseTransport{code: errInternal})
+ tc.conn.Abort(nil)
+ tc.wantFrame("client closes connection after 1-RTT CONNECTION_CLOSE",
+ packetType1RTT, debugFrameConnectionCloseTransport{
+ code: errNo,
+ })
+}
+
+func TestConnKeysDiscardedServer(t *testing.T) {
+ tc := newTestConn(t, serverSide)
+ tc.ignoreFrame(frameTypeAck)
+
+ tc.writeFrames(packetTypeInitial,
+ debugFrameCrypto{
+ data: tc.cryptoDataIn[tls.QUICEncryptionLevelInitial],
+ })
+ tc.wantFrame("server sends Initial CRYPTO frame",
+ packetTypeInitial, debugFrameCrypto{
+ data: tc.cryptoDataOut[tls.QUICEncryptionLevelInitial],
+ })
+ tc.wantFrame("server sends Handshake CRYPTO frame",
+ packetTypeHandshake, debugFrameCrypto{
+ data: tc.cryptoDataOut[tls.QUICEncryptionLevelHandshake],
+ })
+
+ // The server discards Initial keys after receiving a Handshake packet.
+ // The Handshake packet contains only the start of the client's CRYPTO flight here,
+ // to avoids completing the handshake yet.
+ tc.writeFrames(packetTypeHandshake,
+ debugFrameCrypto{
+ data: tc.cryptoDataIn[tls.QUICEncryptionLevelHandshake][:1],
+ })
+ tc.writeFrames(packetTypeInitial,
+ debugFrameConnectionCloseTransport{code: errInternal})
+ tc.wantFrame("server provides an additional connection ID",
+ packetType1RTT, debugFrameNewConnectionID{
+ seq: 1,
+ connID: testLocalConnID(1),
+ })
+ tc.wantIdle("server has discarded Initial keys, cannot read CONNECTION_CLOSE")
+
+ // The server discards Handshake keys after sending a HANDSHAKE_DONE frame.
+ tc.writeFrames(packetTypeHandshake,
+ debugFrameCrypto{
+ off: 1,
+ data: tc.cryptoDataIn[tls.QUICEncryptionLevelHandshake][1:],
+ })
+ tc.wantFrame("server sends HANDSHAKE_DONE after handshake completes",
+ packetType1RTT, debugFrameHandshakeDone{})
+ tc.writeFrames(packetTypeHandshake,
+ debugFrameConnectionCloseTransport{code: errInternal})
+ tc.wantIdle("server has discarded Handshake keys, cannot read CONNECTION_CLOSE")
+
+ tc.writeFrames(packetType1RTT,
+ debugFrameConnectionCloseTransport{code: errInternal})
+ tc.conn.Abort(nil)
+ tc.wantFrame("server closes connection after 1-RTT CONNECTION_CLOSE",
+ packetType1RTT, debugFrameConnectionCloseTransport{
+ code: errNo,
+ })
+}
+
+func TestConnInvalidCryptoData(t *testing.T) {
+ tc := newTestConn(t, clientSide)
+ tc.ignoreFrame(frameTypeAck)
+
+ tc.wantFrame("client sends Initial CRYPTO frame",
+ packetTypeInitial, debugFrameCrypto{
+ data: tc.cryptoDataOut[tls.QUICEncryptionLevelInitial],
+ })
+ tc.writeFrames(packetTypeInitial,
+ debugFrameCrypto{
+ data: tc.cryptoDataIn[tls.QUICEncryptionLevelInitial],
+ })
+
+ // Render the server's response invalid.
+ //
+ // The client closes the connection with CRYPTO_ERROR.
+ //
+ // Changing the first byte will change the TLS message type,
+ // so we can reasonably assume that this is an unexpected_message alert (10).
+ tc.cryptoDataIn[tls.QUICEncryptionLevelHandshake][0] ^= 0x1
+ tc.writeFrames(packetTypeHandshake,
+ debugFrameCrypto{
+ data: tc.cryptoDataIn[tls.QUICEncryptionLevelHandshake],
+ })
+ tc.wantFrame("client closes connection due to TLS handshake error",
+ packetTypeInitial, debugFrameConnectionCloseTransport{
+ code: errTLSBase + 10,
+ })
+}
+
+func TestConnInvalidPeerCertificate(t *testing.T) {
+ tc := newTestConn(t, clientSide, func(c *tls.Config) {
+ c.VerifyPeerCertificate = func([][]byte, [][]*x509.Certificate) error {
+ return errors.New("I will not buy this certificate. It is scratched.")
+ }
+ })
+ tc.ignoreFrame(frameTypeAck)
+
+ tc.wantFrame("client sends Initial CRYPTO frame",
+ packetTypeInitial, debugFrameCrypto{
+ data: tc.cryptoDataOut[tls.QUICEncryptionLevelInitial],
+ })
+ tc.writeFrames(packetTypeInitial,
+ debugFrameCrypto{
+ data: tc.cryptoDataIn[tls.QUICEncryptionLevelInitial],
+ })
+ tc.writeFrames(packetTypeHandshake,
+ debugFrameCrypto{
+ data: tc.cryptoDataIn[tls.QUICEncryptionLevelHandshake],
+ })
+ tc.wantFrame("client closes connection due to rejecting server certificate",
+ packetTypeInitial, debugFrameConnectionCloseTransport{
+ code: errTLSBase + 42, // 42: bad_certificate
+ })
+}
+
+func TestConnHandshakeDoneSentToServer(t *testing.T) {
+ tc := newTestConn(t, serverSide)
+ tc.handshake()
+
+ tc.writeFrames(packetType1RTT,
+ debugFrameHandshakeDone{})
+ tc.wantFrame("server closes connection when client sends a HANDSHAKE_DONE frame",
+ packetType1RTT, debugFrameConnectionCloseTransport{
+ code: errProtocolViolation,
+ })
+}
+
+func TestConnCryptoDataOutOfOrder(t *testing.T) {
+ tc := newTestConn(t, clientSide)
+ tc.ignoreFrame(frameTypeAck)
+
+ tc.wantFrame("client sends Initial CRYPTO frame",
+ packetTypeInitial, debugFrameCrypto{
+ data: tc.cryptoDataOut[tls.QUICEncryptionLevelInitial],
+ })
+ tc.writeFrames(packetTypeInitial,
+ debugFrameCrypto{
+ data: tc.cryptoDataIn[tls.QUICEncryptionLevelInitial],
+ })
+ tc.wantIdle("client is idle, server Handshake flight has not arrived")
+
+ tc.writeFrames(packetTypeHandshake,
+ debugFrameCrypto{
+ off: 15,
+ data: tc.cryptoDataIn[tls.QUICEncryptionLevelHandshake][15:],
+ })
+ tc.wantIdle("client is idle, server Handshake flight is not complete")
+
+ tc.writeFrames(packetTypeHandshake,
+ debugFrameCrypto{
+ off: 1,
+ data: tc.cryptoDataIn[tls.QUICEncryptionLevelHandshake][1:20],
+ })
+ tc.wantIdle("client is idle, server Handshake flight is still not complete")
+
+ tc.writeFrames(packetTypeHandshake,
+ debugFrameCrypto{
+ data: tc.cryptoDataIn[tls.QUICEncryptionLevelHandshake][0:1],
+ })
+ tc.wantFrame("client sends Handshake CRYPTO frame",
+ packetTypeHandshake, debugFrameCrypto{
+ data: tc.cryptoDataOut[tls.QUICEncryptionLevelHandshake],
+ })
+}
+
+func TestConnCryptoBufferSizeExceeded(t *testing.T) {
+ tc := newTestConn(t, clientSide)
+ tc.ignoreFrame(frameTypeAck)
+
+ tc.wantFrame("client sends Initial CRYPTO frame",
+ packetTypeInitial, debugFrameCrypto{
+ data: tc.cryptoDataOut[tls.QUICEncryptionLevelInitial],
+ })
+ tc.writeFrames(packetTypeInitial,
+ debugFrameCrypto{
+ off: cryptoBufferSize,
+ data: []byte{0},
+ })
+ tc.wantFrame("client closes connection after server exceeds CRYPTO buffer",
+ packetTypeInitial, debugFrameConnectionCloseTransport{
+ code: errCryptoBufferExceeded,
+ })
+}
+
+func TestConnAEADLimitReached(t *testing.T) {
+ // "[...] endpoints MUST count the number of received packets that
+ // fail authentication during the lifetime of a connection.
+ // If the total number of received packets that fail authentication [...]
+ // exceeds the integrity limit for the selected AEAD,
+ // the endpoint MUST immediately close the connection [...]"
+ // https://www.rfc-editor.org/rfc/rfc9001#section-6.6-6
+ tc := newTestConn(t, clientSide)
+ tc.handshake()
+
+ var limit int64
+ switch suite := tc.conn.keysAppData.r.suite; suite {
+ case tls.TLS_AES_128_GCM_SHA256, tls.TLS_AES_256_GCM_SHA384:
+ limit = 1 << 52
+ case tls.TLS_CHACHA20_POLY1305_SHA256:
+ limit = 1 << 36
+ default:
+ t.Fatalf("conn.keysAppData.r.suite = %v, unknown suite", suite)
+ }
+
+ dstConnID := tc.conn.connIDState.local[0].cid
+ if tc.conn.connIDState.local[0].seq == -1 {
+ // Only use the transient connection ID in Initial packets.
+ dstConnID = tc.conn.connIDState.local[1].cid
+ }
+ invalid := tc.encodeTestPacket(&testPacket{
+ ptype: packetType1RTT,
+ num: 1000,
+ frames: []debugFrame{debugFramePing{}},
+ version: quicVersion1,
+ dstConnID: dstConnID,
+ srcConnID: tc.peerConnID,
+ }, 0)
+ invalid[len(invalid)-1] ^= 1
+ sendInvalid := func() {
+ t.Logf("<- conn under test receives invalid datagram")
+ tc.conn.sendMsg(&datagram{
+ b: invalid,
+ })
+ tc.wait()
+ }
+
+ // Set the conn's auth failure count to just before the AEAD integrity limit.
+ tc.conn.keysAppData.authFailures = limit - 1
+
+ tc.writeFrames(packetType1RTT, debugFramePing{})
+ tc.advanceToTimer()
+ tc.wantFrameType("auth failures less than limit: conn ACKs packet",
+ packetType1RTT, debugFrameAck{})
+
+ sendInvalid()
+ tc.writeFrames(packetType1RTT, debugFramePing{})
+ tc.advanceToTimer()
+ tc.wantFrameType("auth failures at limit: conn closes",
+ packetType1RTT, debugFrameConnectionCloseTransport{
+ code: errAEADLimitReached,
+ })
+
+ tc.writeFrames(packetType1RTT, debugFramePing{})
+ tc.advance(1 * time.Second)
+ tc.wantIdle("auth failures at limit: conn does not process additional packets")
+}
diff --git a/internal/quic/tlsconfig_test.go b/internal/quic/tlsconfig_test.go
new file mode 100644
index 0000000000..47bfb05983
--- /dev/null
+++ b/internal/quic/tlsconfig_test.go
@@ -0,0 +1,62 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build go1.21
+
+package quic
+
+import (
+ "crypto/tls"
+ "strings"
+)
+
+func newTestTLSConfig(side connSide) *tls.Config {
+ config := &tls.Config{
+ InsecureSkipVerify: true,
+ CipherSuites: []uint16{
+ tls.TLS_AES_128_GCM_SHA256,
+ tls.TLS_AES_256_GCM_SHA384,
+ tls.TLS_CHACHA20_POLY1305_SHA256,
+ },
+ MinVersion: tls.VersionTLS13,
+ }
+ if side == serverSide {
+ config.Certificates = []tls.Certificate{testCert}
+ }
+ return config
+}
+
+var testCert = func() tls.Certificate {
+ cert, err := tls.X509KeyPair(localhostCert, localhostKey)
+ if err != nil {
+ panic(err)
+ }
+ return cert
+}()
+
+// localhostCert is a PEM-encoded TLS cert with SAN IPs
+// "127.0.0.1" and "[::1]", expiring at Jan 29 16:00:00 2084 GMT.
+// generated from src/crypto/tls:
+// go run generate_cert.go --ecdsa-curve P256 --host 127.0.0.1,::1,example.com --ca --start-date "Jan 1 00:00:00 1970" --duration=1000000h
+var localhostCert = []byte(`-----BEGIN CERTIFICATE-----
+MIIBrDCCAVKgAwIBAgIPCvPhO+Hfv+NW76kWxULUMAoGCCqGSM49BAMCMBIxEDAO
+BgNVBAoTB0FjbWUgQ28wIBcNNzAwMTAxMDAwMDAwWhgPMjA4NDAxMjkxNjAwMDBa
+MBIxEDAOBgNVBAoTB0FjbWUgQ28wWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAARh
+WRF8p8X9scgW7JjqAwI9nYV8jtkdhqAXG9gyEgnaFNN5Ze9l3Tp1R9yCDBMNsGms
+PyfMPe5Jrha/LmjgR1G9o4GIMIGFMA4GA1UdDwEB/wQEAwIChDATBgNVHSUEDDAK
+BggrBgEFBQcDATAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBSOJri/wLQxq6oC
+Y6ZImms/STbTljAuBgNVHREEJzAlggtleGFtcGxlLmNvbYcEfwAAAYcQAAAAAAAA
+AAAAAAAAAAAAATAKBggqhkjOPQQDAgNIADBFAiBUguxsW6TGhixBAdORmVNnkx40
+HjkKwncMSDbUaeL9jQIhAJwQ8zV9JpQvYpsiDuMmqCuW35XXil3cQ6Drz82c+fvE
+-----END CERTIFICATE-----`)
+
+// localhostKey is the private key for localhostCert.
+var localhostKey = []byte(testingKey(`-----BEGIN TESTING KEY-----
+MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgY1B1eL/Bbwf/MDcs
+rnvvWhFNr1aGmJJR59PdCN9lVVqhRANCAARhWRF8p8X9scgW7JjqAwI9nYV8jtkd
+hqAXG9gyEgnaFNN5Ze9l3Tp1R9yCDBMNsGmsPyfMPe5Jrha/LmjgR1G9
+-----END TESTING KEY-----`))
+
+// testingKey helps keep security scanners from getting excited about a private key in this file.
+func testingKey(s string) string { return strings.ReplaceAll(s, "TESTING KEY", "PRIVATE KEY") }
diff --git a/internal/quic/transport_params.go b/internal/quic/transport_params.go
new file mode 100644
index 0000000000..dc76d16509
--- /dev/null
+++ b/internal/quic/transport_params.go
@@ -0,0 +1,283 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build go1.21
+
+package quic
+
+import (
+ "encoding/binary"
+ "net/netip"
+ "time"
+)
+
+// transportParameters transferred in the quic_transport_parameters TLS extension.
+// https://www.rfc-editor.org/rfc/rfc9000.html#section-18.2
+type transportParameters struct {
+ originalDstConnID []byte
+ maxIdleTimeout time.Duration
+ statelessResetToken []byte
+ maxUDPPayloadSize int64
+ initialMaxData int64
+ initialMaxStreamDataBidiLocal int64
+ initialMaxStreamDataBidiRemote int64
+ initialMaxStreamDataUni int64
+ initialMaxStreamsBidi int64
+ initialMaxStreamsUni int64
+ ackDelayExponent int8
+ maxAckDelay time.Duration
+ disableActiveMigration bool
+ preferredAddrV4 netip.AddrPort
+ preferredAddrV6 netip.AddrPort
+ preferredAddrConnID []byte
+ preferredAddrResetToken []byte
+ activeConnIDLimit int64
+ initialSrcConnID []byte
+ retrySrcConnID []byte
+}
+
+const (
+ defaultParamMaxUDPPayloadSize = 65527
+ defaultParamAckDelayExponent = 3
+ defaultParamMaxAckDelayMilliseconds = 25
+ defaultParamActiveConnIDLimit = 2
+)
+
+// defaultTransportParameters is initialized to the RFC 9000 default values.
+func defaultTransportParameters() transportParameters {
+ return transportParameters{
+ maxUDPPayloadSize: defaultParamMaxUDPPayloadSize,
+ ackDelayExponent: defaultParamAckDelayExponent,
+ maxAckDelay: defaultParamMaxAckDelayMilliseconds * time.Millisecond,
+ activeConnIDLimit: defaultParamActiveConnIDLimit,
+ }
+}
+
+const (
+ paramOriginalDestinationConnectionID = 0x00
+ paramMaxIdleTimeout = 0x01
+ paramStatelessResetToken = 0x02
+ paramMaxUDPPayloadSize = 0x03
+ paramInitialMaxData = 0x04
+ paramInitialMaxStreamDataBidiLocal = 0x05
+ paramInitialMaxStreamDataBidiRemote = 0x06
+ paramInitialMaxStreamDataUni = 0x07
+ paramInitialMaxStreamsBidi = 0x08
+ paramInitialMaxStreamsUni = 0x09
+ paramAckDelayExponent = 0x0a
+ paramMaxAckDelay = 0x0b
+ paramDisableActiveMigration = 0x0c
+ paramPreferredAddress = 0x0d
+ paramActiveConnectionIDLimit = 0x0e
+ paramInitialSourceConnectionID = 0x0f
+ paramRetrySourceConnectionID = 0x10
+)
+
+func marshalTransportParameters(p transportParameters) []byte {
+ var b []byte
+ if v := p.originalDstConnID; v != nil {
+ b = appendVarint(b, paramOriginalDestinationConnectionID)
+ b = appendVarintBytes(b, v)
+ }
+ if v := uint64(p.maxIdleTimeout / time.Millisecond); v != 0 {
+ b = appendVarint(b, paramMaxIdleTimeout)
+ b = appendVarint(b, uint64(sizeVarint(v)))
+ b = appendVarint(b, uint64(v))
+ }
+ if v := p.statelessResetToken; v != nil {
+ b = appendVarint(b, paramStatelessResetToken)
+ b = appendVarintBytes(b, v)
+ }
+ if v := p.maxUDPPayloadSize; v != defaultParamMaxUDPPayloadSize {
+ b = appendVarint(b, paramMaxUDPPayloadSize)
+ b = appendVarint(b, uint64(sizeVarint(uint64(v))))
+ b = appendVarint(b, uint64(v))
+ }
+ if v := p.initialMaxData; v != 0 {
+ b = appendVarint(b, paramInitialMaxData)
+ b = appendVarint(b, uint64(sizeVarint(uint64(v))))
+ b = appendVarint(b, uint64(v))
+ }
+ if v := p.initialMaxStreamDataBidiLocal; v != 0 {
+ b = appendVarint(b, paramInitialMaxStreamDataBidiLocal)
+ b = appendVarint(b, uint64(sizeVarint(uint64(v))))
+ b = appendVarint(b, uint64(v))
+ }
+ if v := p.initialMaxStreamDataBidiRemote; v != 0 {
+ b = appendVarint(b, paramInitialMaxStreamDataBidiRemote)
+ b = appendVarint(b, uint64(sizeVarint(uint64(v))))
+ b = appendVarint(b, uint64(v))
+ }
+ if v := p.initialMaxStreamDataUni; v != 0 {
+ b = appendVarint(b, paramInitialMaxStreamDataUni)
+ b = appendVarint(b, uint64(sizeVarint(uint64(v))))
+ b = appendVarint(b, uint64(v))
+ }
+ if v := p.initialMaxStreamsBidi; v != 0 {
+ b = appendVarint(b, paramInitialMaxStreamsBidi)
+ b = appendVarint(b, uint64(sizeVarint(uint64(v))))
+ b = appendVarint(b, uint64(v))
+ }
+ if v := p.initialMaxStreamsUni; v != 0 {
+ b = appendVarint(b, paramInitialMaxStreamsUni)
+ b = appendVarint(b, uint64(sizeVarint(uint64(v))))
+ b = appendVarint(b, uint64(v))
+ }
+ if v := p.ackDelayExponent; v != defaultParamAckDelayExponent {
+ b = appendVarint(b, paramAckDelayExponent)
+ b = appendVarint(b, uint64(sizeVarint(uint64(v))))
+ b = appendVarint(b, uint64(v))
+ }
+ if v := uint64(p.maxAckDelay / time.Millisecond); v != defaultParamMaxAckDelayMilliseconds {
+ b = appendVarint(b, paramMaxAckDelay)
+ b = appendVarint(b, uint64(sizeVarint(v)))
+ b = appendVarint(b, v)
+ }
+ if p.disableActiveMigration {
+ b = appendVarint(b, paramDisableActiveMigration)
+ b = append(b, 0) // 0-length value
+ }
+ if p.preferredAddrConnID != nil {
+ b = append(b, paramPreferredAddress)
+ b = appendVarint(b, uint64(4+2+16+2+1+len(p.preferredAddrConnID)+16))
+ b = append(b, p.preferredAddrV4.Addr().AsSlice()...) // 4 bytes
+ b = binary.BigEndian.AppendUint16(b, p.preferredAddrV4.Port()) // 2 bytes
+ b = append(b, p.preferredAddrV6.Addr().AsSlice()...) // 16 bytes
+ b = binary.BigEndian.AppendUint16(b, p.preferredAddrV6.Port()) // 2 bytes
+ b = appendUint8Bytes(b, p.preferredAddrConnID) // 1 byte + len(conn_id)
+ b = append(b, p.preferredAddrResetToken...) // 16 bytes
+ }
+ if v := p.activeConnIDLimit; v != defaultParamActiveConnIDLimit {
+ b = appendVarint(b, paramActiveConnectionIDLimit)
+ b = appendVarint(b, uint64(sizeVarint(uint64(v))))
+ b = appendVarint(b, uint64(v))
+ }
+ if v := p.initialSrcConnID; v != nil {
+ b = appendVarint(b, paramInitialSourceConnectionID)
+ b = appendVarintBytes(b, v)
+ }
+ if v := p.retrySrcConnID; v != nil {
+ b = appendVarint(b, paramRetrySourceConnectionID)
+ b = appendVarintBytes(b, v)
+ }
+ return b
+}
+
+func unmarshalTransportParams(params []byte) (transportParameters, error) {
+ p := defaultTransportParameters()
+ for len(params) > 0 {
+ id, n := consumeVarint(params)
+ if n < 0 {
+ return p, localTransportError(errTransportParameter)
+ }
+ params = params[n:]
+ val, n := consumeVarintBytes(params)
+ if n < 0 {
+ return p, localTransportError(errTransportParameter)
+ }
+ params = params[n:]
+ n = 0
+ switch id {
+ case paramOriginalDestinationConnectionID:
+ p.originalDstConnID = val
+ n = len(val)
+ case paramMaxIdleTimeout:
+ var v uint64
+ v, n = consumeVarint(val)
+ // If this is unreasonably large, consider it as no timeout to avoid
+ // time.Duration overflows.
+ if v > 1<<32 {
+ v = 0
+ }
+ p.maxIdleTimeout = time.Duration(v) * time.Millisecond
+ case paramStatelessResetToken:
+ if len(val) != 16 {
+ return p, localTransportError(errTransportParameter)
+ }
+ p.statelessResetToken = val
+ n = 16
+ case paramMaxUDPPayloadSize:
+ p.maxUDPPayloadSize, n = consumeVarintInt64(val)
+ if p.maxUDPPayloadSize < 1200 {
+ return p, localTransportError(errTransportParameter)
+ }
+ case paramInitialMaxData:
+ p.initialMaxData, n = consumeVarintInt64(val)
+ case paramInitialMaxStreamDataBidiLocal:
+ p.initialMaxStreamDataBidiLocal, n = consumeVarintInt64(val)
+ case paramInitialMaxStreamDataBidiRemote:
+ p.initialMaxStreamDataBidiRemote, n = consumeVarintInt64(val)
+ case paramInitialMaxStreamDataUni:
+ p.initialMaxStreamDataUni, n = consumeVarintInt64(val)
+ case paramInitialMaxStreamsBidi:
+ p.initialMaxStreamsBidi, n = consumeVarintInt64(val)
+ if p.initialMaxStreamsBidi > maxStreamsLimit {
+ return p, localTransportError(errTransportParameter)
+ }
+ case paramInitialMaxStreamsUni:
+ p.initialMaxStreamsUni, n = consumeVarintInt64(val)
+ if p.initialMaxStreamsUni > maxStreamsLimit {
+ return p, localTransportError(errTransportParameter)
+ }
+ case paramAckDelayExponent:
+ var v uint64
+ v, n = consumeVarint(val)
+ if v > 20 {
+ return p, localTransportError(errTransportParameter)
+ }
+ p.ackDelayExponent = int8(v)
+ case paramMaxAckDelay:
+ var v uint64
+ v, n = consumeVarint(val)
+ if v >= 1<<14 {
+ return p, localTransportError(errTransportParameter)
+ }
+ p.maxAckDelay = time.Duration(v) * time.Millisecond
+ case paramDisableActiveMigration:
+ p.disableActiveMigration = true
+ case paramPreferredAddress:
+ if len(val) < 4+2+16+2+1 {
+ return p, localTransportError(errTransportParameter)
+ }
+ p.preferredAddrV4 = netip.AddrPortFrom(
+ netip.AddrFrom4(*(*[4]byte)(val[:4])),
+ binary.BigEndian.Uint16(val[4:][:2]),
+ )
+ val = val[4+2:]
+ p.preferredAddrV6 = netip.AddrPortFrom(
+ netip.AddrFrom16(*(*[16]byte)(val[:16])),
+ binary.BigEndian.Uint16(val[16:][:2]),
+ )
+ val = val[16+2:]
+ var nn int
+ p.preferredAddrConnID, nn = consumeUint8Bytes(val)
+ if nn < 0 {
+ return p, localTransportError(errTransportParameter)
+ }
+ val = val[nn:]
+ if len(val) != 16 {
+ return p, localTransportError(errTransportParameter)
+ }
+ p.preferredAddrResetToken = val
+ val = nil
+ case paramActiveConnectionIDLimit:
+ p.activeConnIDLimit, n = consumeVarintInt64(val)
+ if p.activeConnIDLimit < 2 {
+ return p, localTransportError(errTransportParameter)
+ }
+ case paramInitialSourceConnectionID:
+ p.initialSrcConnID = val
+ n = len(val)
+ case paramRetrySourceConnectionID:
+ p.retrySrcConnID = val
+ n = len(val)
+ default:
+ n = len(val)
+ }
+ if n != len(val) {
+ return p, localTransportError(errTransportParameter)
+ }
+ }
+ return p, nil
+}
diff --git a/internal/quic/transport_params_test.go b/internal/quic/transport_params_test.go
new file mode 100644
index 0000000000..cc88e83fd6
--- /dev/null
+++ b/internal/quic/transport_params_test.go
@@ -0,0 +1,388 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build go1.21
+
+package quic
+
+import (
+ "bytes"
+ "math"
+ "net/netip"
+ "reflect"
+ "testing"
+ "time"
+)
+
+func TestTransportParametersMarshalUnmarshal(t *testing.T) {
+ for _, test := range []struct {
+ params func(p *transportParameters)
+ enc []byte
+ }{{
+ params: func(p *transportParameters) {
+ p.originalDstConnID = []byte("connid")
+ },
+ enc: []byte{
+ 0x00, // original_destination_connection_id
+ byte(len("connid")),
+ 'c', 'o', 'n', 'n', 'i', 'd',
+ },
+ }, {
+ params: func(p *transportParameters) {
+ p.maxIdleTimeout = 10 * time.Millisecond
+ },
+ enc: []byte{
+ 0x01, // max_idle_timeout
+ 1, // length
+ 10, // varint msecs
+ },
+ }, {
+ params: func(p *transportParameters) {
+ p.statelessResetToken = []byte("0123456789abcdef")
+ },
+ enc: []byte{
+ 0x02, // stateless_reset_token
+ 16, // length
+ '0', '1', '2', '3', '4', '5', '6', '7',
+ '8', '9', 'a', 'b', 'c', 'd', 'e', 'f', // reset token
+ },
+ }, {
+ params: func(p *transportParameters) {
+ p.maxUDPPayloadSize = 1200
+ },
+ enc: []byte{
+ 0x03, // max_udp_payload_size
+ 2, // length
+ 0x44, 0xb0, // varint value
+ },
+ }, {
+ params: func(p *transportParameters) {
+ p.initialMaxData = 10
+ },
+ enc: []byte{
+ 0x04, // initial_max_data
+ 1, // length
+ 10, // varint value
+ },
+ }, {
+ params: func(p *transportParameters) {
+ p.initialMaxStreamDataBidiLocal = 10
+ },
+ enc: []byte{
+ 0x05, // initial_max_stream_data_bidi_local
+ 1, // length
+ 10, // varint value
+ },
+ }, {
+ params: func(p *transportParameters) {
+ p.initialMaxStreamDataBidiRemote = 10
+ },
+ enc: []byte{
+ 0x06, // initial_max_stream_data_bidi_remote
+ 1, // length
+ 10, // varint value
+ },
+ }, {
+ params: func(p *transportParameters) {
+ p.initialMaxStreamDataUni = 10
+ },
+ enc: []byte{
+ 0x07, // initial_max_stream_data_uni
+ 1, // length
+ 10, // varint value
+ },
+ }, {
+ params: func(p *transportParameters) {
+ p.initialMaxStreamsBidi = 10
+ },
+ enc: []byte{
+ 0x08, // initial_max_streams_bidi
+ 1, // length
+ 10, // varint value
+ },
+ }, {
+ params: func(p *transportParameters) {
+ p.initialMaxStreamsUni = 10
+ },
+ enc: []byte{
+ 0x09, // initial_max_streams_uni
+ 1, // length
+ 10, // varint value
+ },
+ }, {
+ params: func(p *transportParameters) {
+ p.ackDelayExponent = 4
+ },
+ enc: []byte{
+ 0x0a, // ack_delay_exponent
+ 1, // length
+ 4, // varint value
+ },
+ }, {
+ params: func(p *transportParameters) {
+ p.maxAckDelay = 10 * time.Millisecond
+ },
+ enc: []byte{
+ 0x0b, // max_ack_delay
+ 1, // length
+ 10, // varint value
+ },
+ }, {
+ params: func(p *transportParameters) {
+ p.disableActiveMigration = true
+ },
+ enc: []byte{
+ 0x0c, // disable_active_migration
+ 0, // length
+ },
+ }, {
+ params: func(p *transportParameters) {
+ p.preferredAddrV4 = netip.MustParseAddrPort("127.0.0.1:80")
+ p.preferredAddrV6 = netip.MustParseAddrPort("[fe80::1]:1024")
+ p.preferredAddrConnID = []byte("connid")
+ p.preferredAddrResetToken = []byte("0123456789abcdef")
+ },
+ enc: []byte{
+ 0x0d, // preferred_address
+ byte(4 + 2 + 16 + 2 + 1 + len("connid") + 16), // length
+ 127, 0, 0, 1, // v4 address
+ 0, 80, // v4 port
+ 0xfe, 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, // v6 address
+ 0x04, 0x00, // v6 port,
+ 6, // connection id length
+ 'c', 'o', 'n', 'n', 'i', 'd', // connection id
+ '0', '1', '2', '3', '4', '5', '6', '7',
+ '8', '9', 'a', 'b', 'c', 'd', 'e', 'f', // reset token
+ },
+ }, {
+ params: func(p *transportParameters) {
+ p.activeConnIDLimit = 10
+ },
+ enc: []byte{
+ 0x0e, // active_connection_id_limit
+ 1, // length
+ 10, // varint value
+ },
+ }, {
+ params: func(p *transportParameters) {
+ p.initialSrcConnID = []byte("connid")
+ },
+ enc: []byte{
+ 0x0f, // initial_source_connection_id
+ byte(len("connid")),
+ 'c', 'o', 'n', 'n', 'i', 'd',
+ },
+ }, {
+ params: func(p *transportParameters) {
+ p.retrySrcConnID = []byte("connid")
+ },
+ enc: []byte{
+ 0x10, // retry_source_connection_id
+ byte(len("connid")),
+ 'c', 'o', 'n', 'n', 'i', 'd',
+ },
+ }} {
+ wantParams := defaultTransportParameters()
+ test.params(&wantParams)
+ gotBytes := marshalTransportParameters(wantParams)
+ if !bytes.Equal(gotBytes, test.enc) {
+ t.Errorf("marshalTransportParameters(%#v):\n got: %x\nwant: %x", wantParams, gotBytes, test.enc)
+ }
+ gotParams, err := unmarshalTransportParams(test.enc)
+ if err != nil {
+ t.Errorf("unmarshalTransportParams(%x): unexpected error: %v", test.enc, err)
+ } else if !reflect.DeepEqual(gotParams, wantParams) {
+ t.Errorf("unmarshalTransportParams(%x):\n got: %#v\nwant: %#v", test.enc, gotParams, wantParams)
+ }
+ }
+}
+
+func TestTransportParametersErrors(t *testing.T) {
+ for _, test := range []struct {
+ desc string
+ enc []byte
+ }{{
+ desc: "invalid id",
+ enc: []byte{
+ 0x40, // too short
+ },
+ }, {
+ desc: "parameter too short",
+ enc: []byte{
+ 0x00, // original_destination_connection_id
+ 0x04, // length
+ 1, 2, 3, // not enough data
+ },
+ }, {
+ desc: "extra data in parameter",
+ enc: []byte{
+ 0x01, // max_idle_timeout
+ 2, // length
+ 10, // varint msecs
+ 0, // extra junk
+ },
+ }, {
+ desc: "invalid varint in parameter",
+ enc: []byte{
+ 0x01, // max_idle_timeout
+ 1, // length
+ 0x40, // incomplete varint
+ },
+ }, {
+ desc: "stateless_reset_token not 16 bytes",
+ enc: []byte{
+ 0x02, // stateless_reset_token,
+ 15, // length
+ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14,
+ },
+ }, {
+ desc: "initial_max_streams_bidi is too large",
+ enc: []byte{
+ 0x08, // initial_max_streams_bidi,
+ 8, // length,
+ 0xd0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01,
+ },
+ }, {
+ desc: "initial_max_streams_uni is too large",
+ enc: []byte{
+ 0x08, // initial_max_streams_uni,
+ 9, // length,
+ 0xd0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01,
+ },
+ }, {
+ desc: "preferred_address is too short",
+ enc: []byte{
+ 0x0d, // preferred_address
+ byte(3),
+ 127, 0, 0,
+ },
+ }, {
+ desc: "preferred_address reset token too short",
+ enc: []byte{
+ 0x0d, // preferred_address
+ byte(4 + 2 + 16 + 2 + 1 + len("connid") + 15), // length
+ 127, 0, 0, 1, // v4 address
+ 0, 80, // v4 port
+ 0xfe, 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, // v6 address
+ 0x04, 0x00, // v6 port,
+ 6, // connection id length
+ 'c', 'o', 'n', 'n', 'i', 'd', // connection id
+ '0', '1', '2', '3', '4', '5', '6', '7',
+ '8', '9', 'a', 'b', 'c', 'd', 'e', // reset token, one byte too short
+
+ },
+ }, {
+ desc: "preferred_address conn id too long",
+ enc: []byte{
+ 0x0d, // preferred_address
+ byte(4 + 2 + 16 + 2 + 1 + len("connid") + 16), // length
+ 127, 0, 0, 1, // v4 address
+ 0, 80, // v4 port
+ 0xfe, 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, // v6 address
+ 0x04, 0x00, // v6 port,
+ byte(len("connid")) + 16 + 1, // connection id length, too long
+ 'c', 'o', 'n', 'n', 'i', 'd', // connection id
+ '0', '1', '2', '3', '4', '5', '6', '7',
+ '8', '9', 'a', 'b', 'c', 'd', 'e', 'f', // reset token
+
+ },
+ }} {
+ _, err := unmarshalTransportParams(test.enc)
+ if err == nil {
+ t.Errorf("%v:\nunmarshalTransportParams(%x): unexpectedly succeeded", test.desc, test.enc)
+ }
+ }
+}
+
+func TestTransportParametersRangeErrors(t *testing.T) {
+ for _, test := range []struct {
+ desc string
+ params func(p *transportParameters)
+ }{{
+ desc: "max_udp_payload_size < 1200",
+ params: func(p *transportParameters) {
+ p.maxUDPPayloadSize = 1199
+ },
+ }, {
+ desc: "ack_delay_exponent > 20",
+ params: func(p *transportParameters) {
+ p.ackDelayExponent = 21
+ },
+ }, {
+ desc: "max_ack_delay > 1^14 ms",
+ params: func(p *transportParameters) {
+ p.maxAckDelay = (1 << 14) * time.Millisecond
+ },
+ }, {
+ desc: "active_connection_id_limit < 2",
+ params: func(p *transportParameters) {
+ p.activeConnIDLimit = 1
+ },
+ }} {
+ p := defaultTransportParameters()
+ test.params(&p)
+ enc := marshalTransportParameters(p)
+ _, err := unmarshalTransportParams(enc)
+ if err == nil {
+ t.Errorf("%v: unmarshalTransportParams unexpectedly succeeded", test.desc)
+ }
+ }
+}
+
+func TestTransportParameterMaxIdleTimeoutOverflowsDuration(t *testing.T) {
+ tooManyMS := 1 + (math.MaxInt64 / uint64(time.Millisecond))
+
+ var enc []byte
+ enc = appendVarint(enc, paramMaxIdleTimeout)
+ enc = appendVarint(enc, uint64(sizeVarint(tooManyMS)))
+ enc = appendVarint(enc, uint64(tooManyMS))
+
+ dec, err := unmarshalTransportParams(enc)
+ if err != nil {
+ t.Fatalf("unmarshalTransportParameters(enc) = %v", err)
+ }
+ if got, want := dec.maxIdleTimeout, time.Duration(0); got != want {
+ t.Errorf("max_idle_timeout=%v, got maxIdleTimeout=%v; want %v", tooManyMS, got, want)
+ }
+}
+
+func TestTransportParametersSkipUnknownParameters(t *testing.T) {
+ enc := []byte{
+ 0x20, // unknown transport parameter
+ 1, // length
+ 0, // varint value
+
+ 0x04, // initial_max_data
+ 1, // length
+ 10, // varint value
+
+ 0x21, // unknown transport parameter
+ 1, // length
+ 0, // varint value
+ }
+ dec, err := unmarshalTransportParams(enc)
+ if err != nil {
+ t.Fatalf("unmarshalTransportParameters(enc) = %v", err)
+ }
+ if got, want := dec.initialMaxData, int64(10); got != want {
+ t.Errorf("got initial_max_data=%v; want %v", got, want)
+ }
+}
+
+func FuzzTransportParametersMarshalUnmarshal(f *testing.F) {
+ f.Fuzz(func(t *testing.T, in []byte) {
+ p1, err := unmarshalTransportParams(in)
+ if err != nil {
+ return
+ }
+ out := marshalTransportParameters(p1)
+ p2, err := unmarshalTransportParams(out)
+ if err != nil {
+ t.Fatalf("round trip unmarshal/remarshal: unmarshal error: %v\n%x", err, in)
+ }
+ if !reflect.DeepEqual(p1, p2) {
+ t.Fatalf("round trip unmarshal/remarshal: parameters differ:\n%x\n%#v\n%#v", in, p1, p2)
+ }
+ })
+}
diff --git a/internal/quic/version_test.go b/internal/quic/version_test.go
new file mode 100644
index 0000000000..cfb7ce4be7
--- /dev/null
+++ b/internal/quic/version_test.go
@@ -0,0 +1,110 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build go1.21
+
+package quic
+
+import (
+ "bytes"
+ "context"
+ "crypto/tls"
+ "testing"
+)
+
+func TestVersionNegotiationServerReceivesUnknownVersion(t *testing.T) {
+ config := &Config{
+ TLSConfig: newTestTLSConfig(serverSide),
+ }
+ tl := newTestListener(t, config, nil)
+
+ // Packet of unknown contents for some unrecognized QUIC version.
+ dstConnID := []byte{1, 2, 3, 4}
+ srcConnID := []byte{5, 6, 7, 8}
+ pkt := []byte{
+ 0b1000_0000,
+ 0x00, 0x00, 0x00, 0x0f,
+ }
+ pkt = append(pkt, byte(len(dstConnID)))
+ pkt = append(pkt, dstConnID...)
+ pkt = append(pkt, byte(len(srcConnID)))
+ pkt = append(pkt, srcConnID...)
+ for len(pkt) < minimumClientInitialDatagramSize {
+ pkt = append(pkt, 0)
+ }
+
+ tl.write(&datagram{
+ b: pkt,
+ })
+ gotPkt := tl.read()
+ if gotPkt == nil {
+ t.Fatalf("got no response; want Version Negotiaion")
+ }
+ if got := getPacketType(gotPkt); got != packetTypeVersionNegotiation {
+ t.Fatalf("got packet type %v; want Version Negotiaion", got)
+ }
+ gotDst, gotSrc, versions := parseVersionNegotiation(gotPkt)
+ if got, want := gotDst, srcConnID; !bytes.Equal(got, want) {
+ t.Errorf("got Destination Connection ID %x, want %x", got, want)
+ }
+ if got, want := gotSrc, dstConnID; !bytes.Equal(got, want) {
+ t.Errorf("got Source Connection ID %x, want %x", got, want)
+ }
+ if got, want := versions, []byte{0, 0, 0, 1}; !bytes.Equal(got, want) {
+ t.Errorf("got Supported Version %x, want %x", got, want)
+ }
+}
+
+func TestVersionNegotiationClientAborts(t *testing.T) {
+ tc := newTestConn(t, clientSide)
+ p := tc.readPacket() // client Initial packet
+ tc.listener.write(&datagram{
+ b: appendVersionNegotiation(nil, p.srcConnID, p.dstConnID, 10),
+ })
+ tc.wantIdle("connection does not send a CONNECTION_CLOSE")
+ if err := tc.conn.waitReady(canceledContext()); err != errVersionNegotiation {
+ t.Errorf("conn.waitReady() = %v, want errVersionNegotiation", err)
+ }
+}
+
+func TestVersionNegotiationClientIgnoresAfterProcessingPacket(t *testing.T) {
+ tc := newTestConn(t, clientSide)
+ tc.ignoreFrame(frameTypeAck)
+ p := tc.readPacket() // client Initial packet
+ tc.writeFrames(packetTypeInitial,
+ debugFrameCrypto{
+ data: tc.cryptoDataIn[tls.QUICEncryptionLevelInitial],
+ })
+ tc.listener.write(&datagram{
+ b: appendVersionNegotiation(nil, p.srcConnID, p.dstConnID, 10),
+ })
+ if err := tc.conn.waitReady(canceledContext()); err != context.Canceled {
+ t.Errorf("conn.waitReady() = %v, want context.Canceled", err)
+ }
+ tc.writeFrames(packetTypeHandshake,
+ debugFrameCrypto{
+ data: tc.cryptoDataIn[tls.QUICEncryptionLevelHandshake],
+ })
+ tc.wantFrameType("conn ignores Version Negotiation and continues with handshake",
+ packetTypeHandshake, debugFrameCrypto{})
+}
+
+func TestVersionNegotiationClientIgnoresMismatchingSourceConnID(t *testing.T) {
+ tc := newTestConn(t, clientSide)
+ tc.ignoreFrame(frameTypeAck)
+ p := tc.readPacket() // client Initial packet
+ tc.listener.write(&datagram{
+ b: appendVersionNegotiation(nil, p.srcConnID, []byte("mismatch"), 10),
+ })
+ tc.writeFrames(packetTypeInitial,
+ debugFrameCrypto{
+ data: tc.cryptoDataIn[tls.QUICEncryptionLevelInitial],
+ })
+ tc.writeFrames(packetTypeHandshake,
+ debugFrameCrypto{
+ data: tc.cryptoDataIn[tls.QUICEncryptionLevelHandshake],
+ })
+ tc.wantFrameType("conn ignores Version Negotiation and continues with handshake",
+ packetTypeHandshake, debugFrameCrypto{})
+}
diff --git a/internal/quic/wire.go b/internal/quic/wire.go
index f0643c9229..8486029151 100644
--- a/internal/quic/wire.go
+++ b/internal/quic/wire.go
@@ -8,7 +8,10 @@ package quic
import "encoding/binary"
-const maxVarintSize = 8
+const (
+ maxVarintSize = 8 // encoded size in bytes
+ maxVarint = (1 << 62) - 1
+)
// consumeVarint parses a variable-length integer, reporting its length.
// It returns a negative length upon an error.
diff --git a/publicsuffix/data/children b/publicsuffix/data/children
index 1038c561ad..08261bffd1 100644
Binary files a/publicsuffix/data/children and b/publicsuffix/data/children differ
diff --git a/publicsuffix/data/nodes b/publicsuffix/data/nodes
index 34751cd5b9..1dae6ede8f 100644
Binary files a/publicsuffix/data/nodes and b/publicsuffix/data/nodes differ
diff --git a/publicsuffix/data/text b/publicsuffix/data/text
index 124dcd61f4..7e516413f6 100644
--- a/publicsuffix/data/text
+++ b/publicsuffix/data/text
@@ -1 +1 @@
-billustrationionjukudoyamakeupowiathletajimageandsoundandvision-riopretobishimagentositecnologiabiocelotenkawabipanasonicatfoodnetworkinggroupperbirdartcenterprisecloudaccesscamdvrcampaniabirkenesoddtangenovarahkkeravjuegoshikikiraraholtalenishikatakazakindependent-revieweirbirthplaceu-1bitbucketrzynishikatsuragirlyuzawabitternidiscoverybjarkoybjerkreimdbaltimore-og-romsdalp1bjugnishikawazukamishihoronobeautydalwaysdatabaseballangenkainanaejrietisalatinabenogatabitorderblackfridaybloombergbauernishimerabloxcms3-website-us-west-2blushakotanishinomiyashironocparachutingjovikarateu-2bmoattachmentsalangenishinoomotegovtattoolforgerockartuzybmsalon-1bmwellbeingzoneu-3bnrwesteuropenairbusantiquesaltdalomzaporizhzhedmarkaratsuginamikatagamilanotairesistanceu-4bondigitaloceanspacesaludishangrilanciabonnishinoshimatsusakahoginankokubunjindianapolis-a-bloggerbookonlinewjerseyboomlahppiacenzachpomorskienishiokoppegardiskussionsbereichattanooganordkapparaglidinglassassinationalheritageu-north-1boschaefflerdalondonetskarelianceu-south-1bostik-serveronagasukevje-og-hornnesalvadordalibabalatinord-aurdalipaywhirlondrinaplesknsalzburgleezextraspace-to-rentalstomakomaibarabostonakijinsekikogentappssejnyaarparalleluxembourglitcheltenham-radio-opensocialorenskogliwicebotanicalgardeno-staginglobodoes-itcouldbeworldisrechtranakamurataiwanairforcechireadthedocsxeroxfinitybotanicgardenishitosashimizunaminamiawajikindianmarketinglogowestfalenishiwakindielddanuorrindigenamsskoganeindustriabotanyanagawallonieruchomoscienceandindustrynissandiegoddabouncemerckmsdnipropetrovskjervoyageorgeorgiabounty-fullensakerrypropertiesamegawaboutiquebecommerce-shopselectaxihuanissayokkaichintaifun-dnsaliasamnangerboutireservditchyouriparasiteboyfriendoftheinternetflixjavaldaostathellevangerbozen-sudtirolottokorozawabozen-suedtirolouvreisenissedalovepoparisor-fronisshingucciprianiigataipeidsvollovesickariyakumodumeloyalistoragebplaceducatorprojectcmembersampalermomahaccapooguybrandywinevalleybrasiliadboxosascoli-picenorddalpusercontentcp4bresciaokinawashirosatobamagazineuesamsclubartowestus2brindisibenikitagataikikuchikumagayagawalmartgorybristoloseyouriparliamentjeldsundivtasvuodnakaniikawatanagurabritishcolumbialowiezaganiyodogawabroadcastlebtimnetzlgloomy-routerbroadwaybroke-itvedestrandivttasvuotnakanojohanamakindlefrakkestadiybrokerbrothermesaverdeatnulmemergencyachtsamsungloppennebrowsersafetymarketsandnessjoenl-ams-1brumunddalublindesnesandoybrunelastxn--0trq7p7nnbrusselsandvikcoromantovalle-daostavangerbruxellesanfranciscofreakunekobayashikaoirmemorialucaniabryanskodjedugit-pagespeedmobilizeroticagliaricoharuovatlassian-dev-builderscbglugsjcbnpparibashkiriabrynewmexicoacharterbuzzwfarmerseinebwhalingmbhartiffany-2bzhitomirbzzcodyn-vpndnsantacruzsantafedjeffersoncoffeedbackdropocznordlandrudupontariobranconavstackasaokamikoaniikappudownloadurbanamexhibitioncogretakamatsukawacollectioncolognewyorkshirebungoonordre-landurhamburgrimstadynamisches-dnsantamariakecolonialwilliamsburgripeeweeklylotterycoloradoplateaudnedalncolumbusheycommunexus-3community-prochowicecomobaravendbambleborkapsicilyonagoyauthgear-stagingivestbyglandroverhallair-traffic-controlleyombomloabaths-heilbronnoysunddnslivegarsheiheijibigawaustraliaustinnfshostrolekamisatokaizukameyamatotakadaustevollivornowtv-infolldalolipopmcdircompanychipstmncomparemarkerryhotelsantoandrepbodynaliasnesoddenmarkhangelskjakdnepropetrovskiervaapsteigenflfannefrankfurtjxn--12cfi8ixb8lutskashibatakashimarshallstatebankashiharacomsecaaskimitsubatamibuildingriwatarailwaycondoshichinohealth-carereformemsettlersanukindustriesteamfamberlevagangaviikanonjinfinitigotembaixadaconferenceconstructionconsuladogadollsaobernardomniweatherchanneluxuryconsultanthropologyconsultingroks-thisayamanobeokakegawacontactkmaxxn--12co0c3b4evalled-aostamayukinsuregruhostingrondarcontagematsubaravennaharimalborkashiwaracontemporaryarteducationalchikugodonnakaiwamizawashtenawsmppl-wawdev-myqnapcloudcontrolledogawarabikomaezakirunoopschlesischesaogoncartoonartdecologiacontractorskenconventureshinodearthickashiwazakiyosatokamachilloutsystemscloudsitecookingchannelsdvrdnsdojogaszkolancashirecifedexetercoolblogdnsfor-better-thanawassamukawatarikuzentakatairavpagecooperativano-frankivskygearapparochernigovernmentksatxn--1ck2e1bananarepublic-inquiryggeebinatsukigatajimidsundevelopmentatarantours3-external-1copenhagencyclopedichiropracticatholicaxiashorokanaiecoproductionsaotomeinforumzcorporationcorsicahcesuoloanswatch-and-clockercorvettenrissagaeroclubmedecincinnativeamericanantiquest-le-patron-k3sapporomuracosenzamamidorittoeigersundynathomebuiltwithdarkasserverrankoshigayaltakasugaintelligencecosidnshome-webservercellikescandypoppdaluzerncostumedicallynxn--1ctwolominamatargets-itlon-2couchpotatofriesardegnarutomobegetmyiparsardiniacouncilvivanovoldacouponsarlcozoracq-acranbrookuwanalyticsarpsborgrongausdalcrankyowariasahikawatchandclockasukabeauxartsandcraftsarufutsunomiyawakasaikaitabashijonawatecrdyndns-at-homedepotaruinterhostsolutionsasayamatta-varjjatmpartinternationalfirearmsaseboknowsitallcreditcardyndns-at-workshoppingrossetouchigasakitahiroshimansionsaskatchewancreditunioncremonashgabadaddjaguarqcxn--1lqs03ncrewhmessinarashinomutashinaintuitoyosatoyokawacricketnedalcrimeast-kazakhstanangercrotonecrownipartsassarinuyamashinazawacrsaudacruisesauheradyndns-blogsitextilegnicapetownnews-stagingroundhandlingroznycuisinellancasterculturalcentertainmentoyotapartysvardocuneocupcakecuritibabymilk3curvallee-d-aosteinkjerusalempresashibetsurugashimaringatlantajirinvestmentsavannahgacutegirlfriendyndns-freeboxoslocalzonecymrulvikasumigaurawa-mazowszexnetlifyinzairtrafficplexus-1cyonabarumesswithdnsaveincloudyndns-homednsaves-the-whalessandria-trani-barletta-andriatranibarlettaandriacyouthruherecipescaracaltanissettaishinomakilovecollegefantasyleaguernseyfembetsukumiyamazonawsglobalacceleratorahimeshimabaridagawatchesciencecentersciencehistoryfermockasuyamegurownproviderferraraferraris-a-catererferrerotikagoshimalopolskanlandyndns-picsaxofetsundyndns-remotewdyndns-ipasadenaroyfgujoinvilleitungsenfhvalerfidontexistmein-iservschulegallocalhostrodawarafieldyndns-serverdalfigueresindevicenzaolkuszczytnoipirangalsaceofilateliafilegear-augustowhoswholdingsmall-webthingscientistordalfilegear-debianfilegear-gbizfilegear-iefilegear-jpmorganfilegear-sg-1filminamiechizenfinalfinancefineartscrapper-sitefinlandyndns-weblikes-piedmonticellocus-4finnoyfirebaseappaviancarrdyndns-wikinkobearalvahkijoetsuldalvdalaskanittedallasalleasecuritytacticschoenbrunnfirenetoystre-slidrettozawafirenzefirestonefirewebpaascrappingulenfirmdaleikangerfishingoldpoint2thisamitsukefitjarvodkafjordyndns-workangerfitnessettlementozsdellogliastradingunmanxn--1qqw23afjalerfldrvalleeaosteflekkefjordyndns1flesberguovdageaidnunjargaflickragerogerscrysecretrosnubar0flierneflirfloginlinefloppythonanywhereggio-calabriafloraflorencefloridatsunangojomedicinakamagayahabackplaneapplinzis-a-celticsfanfloripadoval-daostavalleyfloristanohatakahamalselvendrellflorokunohealthcareerscwienflowerservehalflifeinsurancefltrani-andria-barletta-trani-andriaflynnhosting-clusterfnchiryukyuragifuchungbukharanzanfndynnschokokekschokoladenfnwkaszubytemarkatowicefoolfor-ourfor-somedio-campidano-mediocampidanomediofor-theaterforexrothachijolsterforgotdnservehttpbin-butterforli-cesena-forlicesenaforlillesandefjordynservebbscholarshipschoolbusinessebyforsaleirfjordynuniversityforsandasuolodingenfortalfortefortmissoulangevagrigentomologyeonggiehtavuoatnagahamaroygardencowayfortworthachinoheavyfosneservehumourfotraniandriabarlettatraniandriafoxfordecampobassociatest-iserveblogsytemp-dnserveirchitachinakagawashingtondchernivtsiciliafozfr-par-1fr-par-2franamizuhobby-sitefrancaiseharafranziskanerimalvikatsushikabedzin-addrammenuorochesterfredrikstadtvserveminecraftranoyfreeddnsfreebox-oservemp3freedesktopfizerfreemasonryfreemyiphosteurovisionfreesitefreetlservep2pgfoggiafreiburgushikamifuranorfolkebibleksvikatsuyamarugame-hostyhostingxn--2m4a15efrenchkisshikirkeneservepicservequakefreseniuscultureggio-emilia-romagnakasatsunairguardiannakadomarinebraskaunicommbankaufentigerfribourgfriuli-v-giuliafriuli-ve-giuliafriuli-vegiuliafriuli-venezia-giuliafriuli-veneziagiuliafriuli-vgiuliafriuliv-giuliafriulive-giuliafriulivegiuliafriulivenezia-giuliafriuliveneziagiuliafriulivgiuliafrlfroganservesarcasmatartanddesignfrognfrolandynv6from-akrehamnfrom-alfrom-arfrom-azurewebsiteshikagamiishibukawakepnoorfrom-capitalonewportransipharmacienservicesevastopolefrom-coalfrom-ctranslatedynvpnpluscountryestateofdelawareclaimschoolsztynsettsupportoyotomiyazakis-a-candidatefrom-dchitosetodayfrom-dediboxafrom-flandersevenassisienarvikautokeinoticeablewismillerfrom-gaulardalfrom-hichisochikuzenfrom-iafrom-idyroyrvikingruenoharafrom-ilfrom-in-berlindasewiiheyaizuwakamatsubushikusakadogawafrom-ksharpharmacyshawaiijimarcheapartmentshellaspeziafrom-kyfrom-lanshimokawafrom-mamurogawatsonfrom-mdfrom-medizinhistorischeshimokitayamattelekommunikationfrom-mifunefrom-mnfrom-modalenfrom-mshimonitayanagit-reposts-and-telecommunicationshimonosekikawafrom-mtnfrom-nchofunatoriginstantcloudfrontdoorfrom-ndfrom-nefrom-nhktistoryfrom-njshimosuwalkis-a-chefarsundyndns-mailfrom-nminamifuranofrom-nvalleedaostefrom-nynysagamiharafrom-ohdattorelayfrom-oketogolffanshimotsukefrom-orfrom-padualstackazoologicalfrom-pratogurafrom-ris-a-conservativegashimotsumayfirstockholmestrandfrom-schmidtre-gauldalfrom-sdscloudfrom-tnfrom-txn--2scrj9chonanbunkyonanaoshimakanegasakikugawaltervistailscaleforcefrom-utsiracusaikirovogradoyfrom-vald-aostarostwodzislawildlifestylefrom-vtransportefrom-wafrom-wiardwebview-assetshinichinanfrom-wvanylvenneslaskerrylogisticshinjournalismartlabelingfrom-wyfrosinonefrostalowa-wolawafroyal-commissionfruskydivingfujiiderafujikawaguchikonefujiminokamoenairkitapps-auction-rancherkasydneyfujinomiyadattowebhoptogakushimotoganefujiokayamandalfujisatoshonairlinedre-eikerfujisawafujishiroishidakabiratoridedyn-berlincolnfujitsuruokazakiryuohkurafujiyoshidavvenjargap-east-1fukayabeardubaiduckdnsncfdfukuchiyamadavvesiidappnodebalancertmgrazimutheworkpccwilliamhillfukudomigawafukuis-a-cpalacefukumitsubishigakisarazure-mobileirvikazteleportlligatransurlfukuokakamigaharafukuroishikarikaturindalfukusakishiwadazaifudaigokaseljordfukuyamagatakaharunusualpersonfunabashiriuchinadafunagatakahashimamakisofukushimangonnakatombetsumy-gatewayfunahashikamiamakusatsumasendaisenergyfundaciofunkfeuerfuoiskujukuriyamangyshlakasamatsudoomdnstracefuosskoczowinbar1furubirafurudonostiaafurukawajimaniwakuratefusodegaurafussaintlouis-a-anarchistoireggiocalabriafutabayamaguchinomihachimanagementrapaniizafutboldlygoingnowhere-for-morenakatsugawafuttsurutaharafuturecmshinjukumamotoyamashikefuturehostingfuturemailingfvghamurakamigoris-a-designerhandcraftedhandsonyhangglidinghangoutwentehannanmokuizumodenaklodzkochikuseihidorahannorthwesternmutualhanyuzenhapmircloudletshintokushimahappounzenharvestcelebrationhasamap-northeast-3hasaminami-alpshintomikasaharahashbangryhasudahasura-apphiladelphiaareadmyblogspotrdhasvikfh-muensterhatogayahoooshikamaishimofusartshinyoshitomiokamisunagawahatoyamazakitakatakanabeatshiojirishirifujiedahatsukaichikaiseiyoichimkentrendhostinghattfjelldalhayashimamotobusellfylkesbiblackbaudcdn-edgestackhero-networkisboringhazuminobushistoryhelplfinancialhelsinkitakyushuaiahembygdsforbundhemneshioyanaizuerichardlimanowarudahemsedalhepforgeblockshirahamatonbetsurgeonshalloffameiwamasoyheroyhetemlbfanhgtvaohigashiagatsumagoianiahigashichichibuskerudhigashihiroshimanehigashiizumozakitamigrationhigashikagawahigashikagurasoedahigashikawakitaaikitamotosunndalhigashikurumeeresinstaginghigashimatsushimarburghigashimatsuyamakitaakitadaitoigawahigashimurayamamotorcycleshirakokonoehigashinarusells-for-lesshiranukamitondabayashiogamagoriziahigashinehigashiomitamanortonsberghigashiosakasayamanakakogawahigashishirakawamatakanezawahigashisumiyoshikawaminamiaikitanakagusukumodernhigashitsunosegawahigashiurausukitashiobarahigashiyamatokoriyamanashifteditorxn--30rr7yhigashiyodogawahigashiyoshinogaris-a-doctorhippyhiraizumisatohnoshoohirakatashinagawahiranairportland-4-salernogiessennanjobojis-a-financialadvisor-aurdalhirarahiratsukaerusrcfastlylbanzaicloudappspotagerhirayaitakaokalmykiahistorichouseshiraois-a-geekhakassiahitachiomiyagildeskaliszhitachiotagonohejis-a-greenhitraeumtgeradegreehjartdalhjelmelandholeckodairaholidayholyhomegoodshiraokamitsuehomeiphilatelyhomelinkyard-cloudjiffyresdalhomelinuxn--32vp30hachiojiyahikobierzycehomeofficehomesecuritymacaparecidahomesecuritypchoseikarugamvikarlsoyhomesenseeringhomesklepphilipsynology-diskstationhomeunixn--3bst00minamiiserniahondahongooglecodebergentinghonjyoitakarazukaluganskharkivaporcloudhornindalhorsells-for-ustkanmakiwielunnerhortendofinternet-dnshiratakahagitapphoenixn--3ds443ghospitalhoteleshishikuis-a-guruhotelwithflightshisognehotmailhoyangerhoylandetakasagophonefosshisuifuettertdasnetzhumanitieshitaramahungryhurdalhurumajis-a-hard-workershizukuishimogosenhyllestadhyogoris-a-hunterhyugawarahyundaiwafuneis-into-carsiiitesilkharkovaresearchaeologicalvinklein-the-bandairtelebitbridgestoneenebakkeshibechambagricultureadymadealstahaugesunderseaportsinfolionetworkdalaheadjudygarlandis-into-cartoonsimple-urlis-into-gamesserlillyis-leetrentin-suedtirolis-lostre-toteneis-a-lawyeris-not-certifiedis-savedis-slickhersonis-uberleetrentino-a-adigeis-very-badajozis-a-liberalis-very-evillageis-very-goodyearis-very-niceis-very-sweetpepperugiais-with-thebandovre-eikerisleofmanaustdaljellybeanjenv-arubahccavuotnagaragusabaerobaticketsirdaljeonnamerikawauejetztrentino-aadigejevnakershusdecorativeartslupskhmelnytskyivarggatrentino-alto-adigejewelryjewishartgalleryjfkhplaystation-cloudyclusterjgorajlljls-sto1jls-sto2jls-sto3jmphotographysiojnjaworznospamproxyjoyentrentino-altoadigejoyokaichibajddarchitecturealtorlandjpnjprslzjurkotohiradomainstitutekotourakouhokutamamurakounosupabasembokukizunokunimilitarykouyamarylhurstjordalshalsenkouzushimasfjordenkozagawakozakis-a-llamarnardalkozowindowskrakowinnersnoasakatakkokamiminersokndalkpnkppspbarcelonagawakkanaibetsubamericanfamilyds3-fips-us-gov-west-1krasnikahokutokashikis-a-musiciankrasnodarkredstonekrelliankristiansandcatsolarssonkristiansundkrodsheradkrokstadelvalle-aostatic-accessolognekryminamiizukaminokawanishiaizubangekumanotteroykumatorinovecoregontrailroadkumejimashikis-a-nascarfankumenantokonamegatakatoris-a-nursells-itrentin-sud-tirolkunisakis-a-painteractivelvetrentin-sudtirolkunitachiaraindropilotsolundbecknx-serversellsyourhomeftphxn--3e0b707ekunitomigusukuleuvenetokigawakunneppuboliviajessheimpertrixcdn77-secureggioemiliaromagnamsosnowiechristiansburgminakamichiharakunstsammlungkunstunddesignkuokgroupimientaketomisatoolsomakurehabmerkurgankurobeeldengeluidkurogimimatakatsukis-a-patsfankuroisoftwarezzoologykuromatsunais-a-personaltrainerkuronkurotakikawasakis-a-photographerokussldkushirogawakustanais-a-playershiftcryptonomichigangwonkusupersalezajskomakiyosemitekutchanelkutnowruzhgorodeokuzumakis-a-republicanonoichinomiyakekvafjordkvalsundkvamscompute-1kvanangenkvinesdalkvinnheradkviteseidatingkvitsoykwpspdnsomnatalkzmisakis-a-soxfanmisasaguris-a-studentalmisawamisconfusedmishimasudamissilemisugitokuyamatsumaebashikshacknetrentino-sued-tirolmitakeharamitourismilemitoyoakemiuramiyazurecontainerdpolicemiyotamatsukuris-a-teacherkassyno-dshowamjondalenmonstermontrealestatefarmequipmentrentino-suedtirolmonza-brianzapposor-odalmonza-e-della-brianzaptokyotangotpantheonsitemonzabrianzaramonzaebrianzamonzaedellabrianzamoonscalebookinghostedpictetrentinoa-adigemordoviamoriyamatsumotofukemoriyoshiminamiashigaramormonmouthachirogatakamoriokakudamatsuemoroyamatsunomortgagemoscowiosor-varangermoseushimodatemosjoenmoskenesorfoldmossorocabalena-devicesorreisahayakawakamiichikawamisatottoris-a-techietis-a-landscaperspectakasakitchenmosvikomatsushimarylandmoteginowaniihamatamakinoharamoviemovimientolgamozilla-iotrentinoaadigemtranbytomaritimekeepingmuginozawaonsensiositemuikaminoyamaxunispacemukoebenhavnmulhouseoullensvanguardmunakatanemuncienciamuosattemupinbarclaycards3-sa-east-1murmanskomforbar2murotorcraftrentinoalto-adigemusashinoharamuseetrentinoaltoadigemuseumverenigingmusicargodaddyn-o-saurlandesortlandmutsuzawamy-wanggoupilemyactivedirectorymyamazeplaymyasustor-elvdalmycdmycloudnsoruminamimakis-a-rockstarachowicemydattolocalcertificationmyddnsgeekgalaxymydissentrentinos-tirolmydobissmarterthanyoumydrobofageologymydsoundcastronomy-vigorlicemyeffectrentinostirolmyfastly-terrariuminamiminowamyfirewalledreplittlestargardmyforuminamioguni5myfritzmyftpaccessouthcarolinaturalhistorymuseumcentermyhome-servermyjinomykolaivencloud66mymailermymediapchristmasakillucernemyokohamamatsudamypepinkommunalforbundmypetsouthwest1-uslivinghistorymyphotoshibalashovhadanorth-kazakhstanmypicturestaurantrentinosud-tirolmypsxn--3pxu8kommunemysecuritycamerakermyshopblocksowamyshopifymyspreadshopwarendalenugmythic-beastspectruminamisanrikubetsuppliesoomytis-a-bookkeepermaritimodspeedpartnermytuleap-partnersphinxn--41amyvnchromediatechnologymywirepaircraftingvollohmusashimurayamashikokuchuoplantationplantspjelkavikomorotsukagawaplatformsharis-a-therapistoiaplatter-appinokofuefukihaboromskogplatterpioneerplazaplcube-serversicherungplumbingoplurinacionalpodhalepodlasiellaktyubinskiptveterinairealmpmnpodzonepohlpoivronpokerpokrovskomvuxn--3hcrj9choyodobashichikashukujitawaraumalatvuopmicrosoftbankarmoypoliticarrierpolitiendapolkowicepoltavalle-d-aostaticspydebergpomorzeszowitdkongsbergponpesaro-urbino-pesarourbinopesaromasvuotnarusawapordenonepornporsangerporsangugeporsgrunnanyokoshibahikariwanumatakinouepoznanpraxis-a-bruinsfanprdpreservationpresidioprgmrprimetelemarkongsvingerprincipeprivatizehealthinsuranceprofesionalprogressivestfoldpromombetsupplypropertyprotectionprotonetrentinosued-tirolprudentialpruszkowithgoogleapiszprvcyberprzeworskogpulawypunyufuelveruminamiuonumassa-carrara-massacarraramassabuyshousesopotrentino-sud-tirolpupugliapussycateringebuzentsujiiepvhadselfiphdfcbankazunoticiashinkamigototalpvtrentinosuedtirolpwchungnamdalseidsbergmodellingmxn--11b4c3dray-dnsupdaterpzqhaebaruericssongdalenviknakayamaoris-a-cubicle-slavellinodeobjectshinshinotsurfashionstorebaselburguidefinimamateramochizukimobetsumidatlantichirurgiens-dentistes-en-franceqldqotoyohashimotoshimatsuzakis-an-accountantshowtimelbourneqponiatowadaqslgbtrentinsud-tirolqualifioappippueblockbusternopilawaquickconnectrentinsudtirolquicksytesrhtrentinsued-tirolquipelementsrltunestuff-4-saletunkonsulatrobeebyteappigboatsmolaquilanxessmushcdn77-sslingturystykaniepcetuscanytushuissier-justicetuvalleaostaverntuxfamilytwmailvestvagoyvevelstadvibo-valentiavibovalentiavideovillastufftoread-booksnestorfjordvinnicasadelamonedagestangevinnytsiavipsinaappiwatevirginiavirtual-uservecounterstrikevirtualcloudvirtualservervirtualuserveexchangevirtuelvisakuhokksundviterbolognagasakikonaikawagoevivianvivolkenkundenvixn--42c2d9avlaanderennesoyvladikavkazimierz-dolnyvladimirvlogintoyonezawavminanovologdanskonyveloftrentino-stirolvolvolkswagentstuttgartrentinsuedtirolvolyngdalvoorlopervossevangenvotevotingvotoyonovps-hostrowiecircustomer-ocimmobilienwixsitewloclawekoobindalwmcloudwmflabsurnadalwoodsidelmenhorstabackyardsurreyworse-thandawowithyoutuberspacekitagawawpdevcloudwpenginepoweredwphostedmailwpmucdnpixolinodeusercontentrentinosudtirolwpmudevcdnaccessokanagawawritesthisblogoipizzawroclawiwatsukiyonoshiroomgwtcirclerkstagewtfastvps-serverisignwuozuwzmiuwajimaxn--4gbriminingxn--4it168dxn--4it797kooris-a-libertarianxn--4pvxs4allxn--54b7fta0ccivilaviationredumbrellajollamericanexpressexyxn--55qw42gxn--55qx5dxn--5dbhl8dxn--5js045dxn--5rtp49civilisationrenderxn--5rtq34koperviklabudhabikinokawachinaganoharamcocottempurlxn--5su34j936bgsgxn--5tzm5gxn--6btw5axn--6frz82gxn--6orx2rxn--6qq986b3xlxn--7t0a264civilizationthewifiatmallorcafederation-webspacexn--80aaa0cvacationsusonoxn--80adxhksuzakananiimiharuxn--80ao21axn--80aqecdr1axn--80asehdbarclays3-us-east-2xn--80aswgxn--80aukraanghkembuchikujobservableusercontentrevisohughestripperxn--8dbq2axn--8ltr62koryokamikawanehonbetsuwanouchijiwadeliveryxn--8pvr4uxn--8y0a063axn--90a1affinitylotterybnikeisenbahnxn--90a3academiamicable-modemoneyxn--90aeroportalabamagasakishimabaraffleentry-snowplowiczeladzxn--90aishobarakawaharaoxn--90amckinseyxn--90azhytomyrxn--9dbhblg6dietritonxn--9dbq2axn--9et52uxn--9krt00axn--andy-iraxn--aroport-byandexcloudxn--asky-iraxn--aurskog-hland-jnbarefootballooningjerstadgcapebretonamicrolightingjesdalombardiadembroideryonagunicloudiherokuappanamasteiermarkaracoldwarszawauthgearappspacehosted-by-previderxn--avery-yuasakuragawaxn--b-5gaxn--b4w605ferdxn--balsan-sdtirol-nsbsuzukanazawaxn--bck1b9a5dre4civilwarmiasadoesntexisteingeekarpaczest-a-la-maisondre-landrayddns5yxn--bdddj-mrabdxn--bearalvhki-y4axn--berlevg-jxaxn--bhcavuotna-s4axn--bhccavuotna-k7axn--bidr-5nachikatsuuraxn--bievt-0qa2xn--bjarky-fyaotsurgeryxn--bjddar-ptargithubpreviewsaitohmannore-og-uvdalxn--blt-elabourxn--bmlo-graingerxn--bod-2naturalsciencesnaturellesuzukis-an-actorxn--bozen-sdtirol-2obanazawaxn--brnny-wuacademy-firewall-gatewayxn--brnnysund-m8accident-investigation-acornxn--brum-voagatroandinosaureportrentoyonakagyokutoyakomaganexn--btsfjord-9zaxn--bulsan-sdtirol-nsbaremetalpha-myqnapcloud9guacuiababia-goracleaningitpagexlimoldell-ogliastraderxn--c1avgxn--c2br7gxn--c3s14mincomcastreserve-onlinexn--cck2b3bargainstances3-us-gov-west-1xn--cckwcxetdxn--cesena-forl-mcbremangerxn--cesenaforl-i8axn--cg4bkis-an-actresshwindmillxn--ciqpnxn--clchc0ea0b2g2a9gcdxn--comunicaes-v6a2oxn--correios-e-telecomunicaes-ghc29axn--czr694barreaudiblebesbydgoszczecinemagnethnologyoriikaragandauthordalandroiddnss3-ap-southeast-2ix4432-balsan-suedtirolimiteddnskinggfakefurniturecreationavuotnaritakoelnayorovigotsukisosakitahatakahatakaishimoichinosekigaharaurskog-holandingitlaborxn--czrs0trogstadxn--czru2dxn--czrw28barrel-of-knowledgeappgafanquanpachicappacificurussiautomotivelandds3-ca-central-16-balsan-sudtirollagdenesnaaseinet-freaks3-ap-southeast-123websiteleaf-south-123webseiteckidsmynasushiobarackmazerbaijan-mayen-rootaribeiraogakibichuobiramusementdllpages3-ap-south-123sitewebhareidfjordvagsoyerhcloudd-dnsiskinkyolasiteastcoastaldefenceastus2038xn--d1acj3barrell-of-knowledgecomputerhistoryofscience-fictionfabricafjs3-us-west-1xn--d1alfaromeoxn--d1atromsakegawaxn--d5qv7z876clanbibaidarmeniaxn--davvenjrga-y4axn--djrs72d6uyxn--djty4kosaigawaxn--dnna-grajewolterskluwerxn--drbak-wuaxn--dyry-iraxn--e1a4cldmailukowhitesnow-dnsangohtawaramotoineppubtlsanjotelulubin-brbambinagisobetsuitagajoburgjerdrumcprequalifymein-vigorgebetsukuibmdeveloperauniteroizumizakinderoyomitanobninskanzakiyokawaraustrheimatunduhrennebulsan-suedtirololitapunk123kotisivultrobjectselinogradimo-siemenscaledekaascolipiceno-ipifony-1337xn--eckvdtc9dxn--efvn9svalbardunloppaderbornxn--efvy88hagakhanamigawaxn--ehqz56nxn--elqq16hagebostadxn--eveni-0qa01gaxn--f6qx53axn--fct429kosakaerodromegallupaasdaburxn--fhbeiarnxn--finny-yuaxn--fiq228c5hsvchurchaseljeepsondriodejaneirockyotobetsuliguriaxn--fiq64barsycenterprisesakievennodesadistcgrouplidlugolekagaminord-frontierxn--fiqs8sveioxn--fiqz9svelvikoninjambylxn--fjord-lraxn--fjq720axn--fl-ziaxn--flor-jraxn--flw351exn--forl-cesena-fcbssvizzeraxn--forlcesena-c8axn--fpcrj9c3dxn--frde-grandrapidsvn-repostorjcloud-ver-jpchowderxn--frna-woaraisaijosoyroroswedenxn--frya-hraxn--fzc2c9e2cleverappsannanxn--fzys8d69uvgmailxn--g2xx48clicketcloudcontrolapparmatsuuraxn--gckr3f0fauskedsmokorsetagayaseralingenoamishirasatogliattipschulserverxn--gecrj9clickrisinglesannohekinannestadraydnsanokaruizawaxn--ggaviika-8ya47haibarakitakamiizumisanofidelitysfjordxn--gildeskl-g0axn--givuotna-8yasakaiminatoyookaneyamazoexn--gjvik-wuaxn--gk3at1exn--gls-elacaixaxn--gmq050is-an-anarchistoricalsocietysnesigdalxn--gmqw5axn--gnstigbestellen-zvbrplsbxn--45br5cylxn--gnstigliefern-wobihirosakikamijimatsushigexn--h-2failxn--h1aeghair-surveillancexn--h1ahnxn--h1alizxn--h2breg3eveneswidnicasacampinagrandebungotakadaemongolianxn--h2brj9c8clinichippubetsuikilatironporterxn--h3cuzk1digickoseis-a-linux-usershoujis-a-knightpointtohoboleslawieconomiastalbanshizuokamogawaxn--hbmer-xqaxn--hcesuolo-7ya35barsyonlinewhampshirealtychyattorneyagawakuyabukihokumakogeniwaizumiotsurugimbalsfjordeportexaskoyabeagleboardetroitskypecorivneatonoshoes3-eu-west-3utilitiesquare7xn--hebda8basicserversaillesjabbottateshinanomachildrensgardenhlfanhsbc66xn--hery-iraxn--hgebostad-g3axn--hkkinen-5waxn--hmmrfeasta-s4accident-prevention-aptibleangaviikadenaamesjevuemielnoboribetsuckswidnikkolobrzegersundxn--hnefoss-q1axn--hobl-iraxn--holtlen-hxaxn--hpmir-xqaxn--hxt814exn--hyanger-q1axn--hylandet-54axn--i1b6b1a6a2exn--imr513nxn--indery-fyasugithubusercontentromsojamisonxn--io0a7is-an-artistgstagexn--j1adpkomonotogawaxn--j1aefbsbxn--1lqs71dyndns-office-on-the-webhostingrpassagensavonarviikamiokameokamakurazakiwakunigamihamadaxn--j1ael8basilicataniautoscanadaeguambulancentralus-2xn--j1amhakatanorthflankddiamondshinshiroxn--j6w193gxn--jlq480n2rgxn--jlq61u9w7basketballfinanzgorzeleccodespotenzakopanewspaperxn--jlster-byasuokannamihokkaidopaaskvollxn--jrpeland-54axn--jvr189miniserversusakis-a-socialistg-builderxn--k7yn95exn--karmy-yuaxn--kbrq7oxn--kcrx77d1x4axn--kfjord-iuaxn--klbu-woaxn--klt787dxn--kltp7dxn--kltx9axn--klty5xn--45brj9cistrondheimperiaxn--koluokta-7ya57hakodatexn--kprw13dxn--kpry57dxn--kput3is-an-engineeringxn--krager-gyatominamibosogndalxn--kranghke-b0axn--krdsherad-m8axn--krehamn-dxaxn--krjohka-hwab49jdevcloudfunctionsimplesitexn--ksnes-uuaxn--kvfjord-nxaxn--kvitsy-fyatsukanoyakagexn--kvnangen-k0axn--l-1fairwindswiebodzin-dslattuminamiyamashirokawanabeepilepsykkylvenicexn--l1accentureklamborghinikolaeventswinoujscienceandhistoryxn--laheadju-7yatsushiroxn--langevg-jxaxn--lcvr32dxn--ldingen-q1axn--leagaviika-52batochigifts3-us-west-2xn--lesund-huaxn--lgbbat1ad8jdfaststackschulplattformetacentrumeteorappassenger-associationxn--lgrd-poacctrusteexn--lhppi-xqaxn--linds-pramericanartrvestnestudioxn--lns-qlavagiskexn--loabt-0qaxn--lrdal-sraxn--lrenskog-54axn--lt-liacliniquedapliexn--lten-granexn--lury-iraxn--m3ch0j3axn--mely-iraxn--merker-kuaxn--mgb2ddeswisstpetersburgxn--mgb9awbfbx-ostrowwlkpmguitarschwarzgwangjuifminamidaitomanchesterxn--mgba3a3ejtrycloudflarevistaplestudynamic-dnsrvaroyxn--mgba3a4f16axn--mgba3a4fra1-deloittevaksdalxn--mgba7c0bbn0axn--mgbaakc7dvfstdlibestadxn--mgbaam7a8hakonexn--mgbab2bdxn--mgbah1a3hjkrdxn--mgbai9a5eva00batsfjordiscordsays3-website-ap-northeast-1xn--mgbai9azgqp6jejuniperxn--mgbayh7gpalmaseratis-an-entertainerxn--mgbbh1a71exn--mgbc0a9azcgxn--mgbca7dzdoxn--mgbcpq6gpa1axn--mgberp4a5d4a87gxn--mgberp4a5d4arxn--mgbgu82axn--mgbi4ecexposedxn--mgbpl2fhskosherbrookegawaxn--mgbqly7c0a67fbclintonkotsukubankarumaifarmsteadrobaknoluoktachikawakayamadridvallee-aosteroyxn--mgbqly7cvafr-1xn--mgbt3dhdxn--mgbtf8flapymntrysiljanxn--mgbtx2bauhauspostman-echocolatemasekd1xn--mgbx4cd0abbvieeexn--mix082fbxoschweizxn--mix891fedorainfraclouderaxn--mjndalen-64axn--mk0axin-vpnclothingdustdatadetectjmaxxxn--12c1fe0bradescotlandrrxn--mk1bu44cn-northwest-1xn--mkru45is-bykleclerchoshibuyachiyodancexn--mlatvuopmi-s4axn--mli-tlavangenxn--mlselv-iuaxn--moreke-juaxn--mori-qsakurais-certifiedxn--mosjen-eyawaraxn--mot-tlazioxn--mre-og-romsdal-qqbuseranishiaritakurashikis-foundationxn--msy-ula0hakubaghdadultravelchannelxn--mtta-vrjjat-k7aflakstadaokagakicks-assnasaarlandxn--muost-0qaxn--mxtq1minisitexn--ngbc5azdxn--ngbe9e0axn--ngbrxn--45q11citadelhicampinashikiminohostfoldnavyxn--nit225koshimizumakiyosunnydayxn--nmesjevuemie-tcbalestrandabergamoarekeymachineustarnbergxn--nnx388axn--nodessakyotanabelaudiopsysynology-dstreamlitappittsburghofficialxn--nqv7fs00emaxn--nry-yla5gxn--ntso0iqx3axn--ntsq17gxn--nttery-byaeserveftplanetariuminamitanexn--nvuotna-hwaxn--nyqy26axn--o1achernihivgubsxn--o3cw4hakuis-a-democratravelersinsurancexn--o3cyx2axn--od0algxn--od0aq3belementorayoshiokanumazuryukuhashimojibxos3-website-ap-southeast-1xn--ogbpf8flatangerxn--oppegrd-ixaxn--ostery-fyawatahamaxn--osyro-wuaxn--otu796dxn--p1acfedorapeoplegoismailillehammerfeste-ipatriaxn--p1ais-gonexn--pgbs0dhlx3xn--porsgu-sta26fedoraprojectoyotsukaidoxn--pssu33lxn--pssy2uxn--q7ce6axn--q9jyb4cngreaterxn--qcka1pmcpenzaporizhzhiaxn--qqqt11minnesotaketakayamassivegridxn--qxa6axn--qxamsterdamnserverbaniaxn--rady-iraxn--rdal-poaxn--rde-ulaxn--rdy-0nabaris-into-animeetrentin-sued-tirolxn--rennesy-v1axn--rhkkervju-01afeiraquarelleasingujaratoyouraxn--rholt-mragowoltlab-democraciaxn--rhqv96gxn--rht27zxn--rht3dxn--rht61exn--risa-5naturbruksgymnxn--risr-iraxn--rland-uuaxn--rlingen-mxaxn--rmskog-byaxn--rny31hakusanagochihayaakasakawaiishopitsitexn--rovu88bellevuelosangeles3-website-ap-southeast-2xn--rros-granvindafjordxn--rskog-uuaxn--rst-0naturhistorischesxn--rsta-framercanvasxn--rvc1e0am3exn--ryken-vuaxn--ryrvik-byaxn--s-1faithaldenxn--s9brj9cnpyatigorskolecznagatorodoyxn--sandnessjen-ogbellunord-odalombardyn53xn--sandy-yuaxn--sdtirol-n2axn--seral-lraxn--ses554gxn--sgne-graphoxn--4dbgdty6citichernovtsyncloudrangedaluccarbonia-iglesias-carboniaiglesiascarboniaxn--skierv-utazasxn--skjervy-v1axn--skjk-soaxn--sknit-yqaxn--sknland-fxaxn--slat-5natuurwetenschappenginexn--slt-elabcieszynh-servebeero-stageiseiroumuenchencoreapigeelvinckoshunantankmpspawnextdirectrentino-s-tirolxn--smla-hraxn--smna-gratangentlentapisa-geekosugexn--snase-nraxn--sndre-land-0cbeneventochiokinoshimaintenancebinordreisa-hockeynutazurestaticappspaceusercontentateyamaveroykenglandeltaitogitsumitakagiizeasypanelblagrarchaeologyeongbuk0emmafann-arboretumbriamallamaceiobbcg123homepagefrontappchizip61123minsidaarborteaches-yogasawaracingroks-theatree123hjemmesidealerimo-i-rana4u2-localhistorybolzano-altoadigeometre-experts-comptables3-ap-northeast-123miwebcambridgehirn4t3l3p0rtarumizusawabogadobeaemcloud-fr123paginaweberkeleyokosukanrabruzzombieidskoguchikushinonsenasakuchinotsuchiurakawafaicloudineat-url-o-g-i-naval-d-aosta-valleyokote164-b-datacentermezproxyzgoraetnabudejjudaicadaquest-mon-blogueurodirumaceratabuseating-organicbcn-north-123saitamakawabartheshopencraftrainingdyniajuedischesapeakebayernavigationavoi234lima-cityeats3-ap-northeast-20001wwwedeployokozeastasiamunemurorangecloudplatform0xn--snes-poaxn--snsa-roaxn--sr-aurdal-l8axn--sr-fron-q1axn--sr-odal-q1axn--sr-varanger-ggbentleyurihonjournalistjohnikonanporovnobserverxn--srfold-byaxn--srreisa-q1axn--srum-gratis-a-bulls-fanxn--stfold-9xaxn--stjrdal-s1axn--stjrdalshalsen-sqbeppublishproxyusuharavocatanzarowegroweiboltashkentatamotorsitestingivingjemnes3-eu-central-1kappleadpages-12hpalmspringsakerxn--stre-toten-zcbeskidyn-ip24xn--t60b56axn--tckweddingxn--tiq49xqyjelasticbeanstalkhmelnitskiyamarumorimachidaxn--tjme-hraxn--tn0agrocerydxn--tnsberg-q1axn--tor131oxn--trany-yuaxn--trentin-sd-tirol-rzbestbuyshoparenagareyamaizurugbyenvironmentalconservationflashdrivefsnillfjordiscordsezjampaleoceanographics3-website-eu-west-1xn--trentin-sdtirol-7vbetainaboxfuseekloges3-website-sa-east-1xn--trentino-sd-tirol-c3bhzcasertainaioirasebastopologyeongnamegawafflecellclstagemologicaliforniavoues3-eu-west-1xn--trentino-sdtirol-szbielawalbrzycharitypedreamhostersvp4xn--trentinosd-tirol-rzbiellaakesvuemieleccebizenakanotoddeninoheguriitatebayashiibahcavuotnagaivuotnagaokakyotambabybluebitelevisioncilla-speziaxarnetbank8s3-eu-west-2xn--trentinosdtirol-7vbieszczadygeyachimataijiiyamanouchikuhokuryugasakitaurayasudaxn--trentinsd-tirol-6vbievat-band-campaignieznombrendlyngengerdalces3-website-us-east-1xn--trentinsdtirol-nsbifukagawalesundiscountypeformelhusgardeninomiyakonojorpelandiscourses3-website-us-west-1xn--trgstad-r1axn--trna-woaxn--troms-zuaxn--tysvr-vraxn--uc0atvestre-slidrexn--uc0ay4axn--uist22halsakakinokiaxn--uisz3gxn--unjrga-rtarnobrzegyptianxn--unup4yxn--uuwu58axn--vads-jraxn--valle-aoste-ebbtularvikonskowolayangroupiemontexn--valle-d-aoste-ehboehringerikexn--valleaoste-e7axn--valledaoste-ebbvadsoccerxn--vard-jraxn--vegrshei-c0axn--vermgensberater-ctb-hostingxn--vermgensberatung-pwbigvalledaostaobaomoriguchiharag-cloud-championshiphoplixboxenirasakincheonishiazaindependent-commissionishigouvicasinordeste-idclkarasjohkamikitayamatsurindependent-inquest-a-la-masionishiharaxn--vestvgy-ixa6oxn--vg-yiabkhaziaxn--vgan-qoaxn--vgsy-qoa0jelenia-goraxn--vgu402cnsantabarbaraxn--vhquvestre-totennishiawakuraxn--vler-qoaxn--vre-eiker-k8axn--vrggt-xqadxn--vry-yla5gxn--vuq861biharstadotsubetsugaruhrxn--w4r85el8fhu5dnraxn--w4rs40lxn--wcvs22dxn--wgbh1cntjomeldaluroyxn--wgbl6axn--xhq521bihorologyusuisservegame-serverxn--xkc2al3hye2axn--xkc2dl3a5ee0hammarfeastafricaravantaaxn--y9a3aquariumintereitrentino-sudtirolxn--yer-znaumburgxn--yfro4i67oxn--ygarden-p1axn--ygbi2ammxn--4dbrk0cexn--ystre-slidre-ujbikedaejeonbukarasjokarasuyamarriottatsunoceanographiquehimejindependent-inquiryuufcfanishiizunazukindependent-panelomoliseminemrxn--zbx025dxn--zf0ao64axn--zf0avxlxn--zfr164bilbaogashimadachicagoboavistanbulsan-sudtirolbia-tempio-olbiatempioolbialystokkeliwebredirectme-south-1xnbayxz
\ No newline at end of file
+birkenesoddtangentinglogoweirbitbucketrzynishikatakayamatta-varjjatjomembersaltdalovepopartysfjordiskussionsbereichatinhlfanishikatsuragitappassenger-associationishikawazukamiokameokamakurazakitaurayasudabitternidisrechtrainingloomy-routerbjarkoybjerkreimdbalsan-suedtirololitapunkapsienamsskoganeibmdeveloperauniteroirmemorialombardiadempresashibetsukumiyamagasakinderoyonagunicloudevelopmentaxiijimarriottayninhaccanthobby-siteval-d-aosta-valleyoriikaracolognebinatsukigataiwanumatajimidsundgcahcesuolocustomer-ocimperiautoscanalytics-gatewayonagoyaveroykenflfanpachihayaakasakawaiishopitsitemasekd1kappenginedre-eikerimo-siemenscaledekaascolipicenoboribetsucks3-eu-west-3utilities-16-balestrandabergentappsseekloges3-eu-west-123paginawebcamauction-acornfshostrodawaraktyubinskaunicommbank123kotisivultrobjectselinogradimo-i-rana4u2-localhostrolekanieruchomoscientistordal-o-g-i-nikolaevents3-ap-northeast-2-ddnsking123homepagefrontappchizip61123saitamakawababia-goracleaningheannakadomarineat-urlimanowarudakuneustarostwodzislawdev-myqnapcloudcontrolledgesuite-stagingdyniamusementdllclstagehirnikonantomobelementorayokosukanoyakumoliserniaurland-4-salernord-aurdalipaywhirlimiteddnslivelanddnss3-ap-south-123siteweberlevagangaviikanonji234lima-cityeats3-ap-southeast-123webseiteambulancechireadmyblogspotaribeiraogakicks-assurfakefurniturealmpmninoheguribigawaurskog-holandinggfarsundds3-ap-southeast-20001wwwedeployokote123hjemmesidealerdalaheadjuegoshikibichuobiraustevollimombetsupplyokoze164-balena-devices3-ca-central-123websiteleaf-south-12hparliamentatsunobninsk8s3-eu-central-1337bjugnishimerablackfridaynightjxn--11b4c3ditchyouripatriabloombergretaijindustriesteinkjerbloxcmsaludivtasvuodnakaiwanairlinekobayashimodatecnologiablushakotanishinomiyashironomniwebview-assetsalvadorbmoattachmentsamegawabmsamnangerbmwellbeingzonebnrweatherchannelsdvrdnsamparalleluxenishinoomotegotsukishiwadavvenjargamvikarpaczest-a-la-maisondre-landivttasvuotnakamai-stagingloppennebomlocalzonebonavstackartuzybondigitaloceanspacesamsclubartowest1-usamsunglugsmall-webspacebookonlineboomlaakesvuemielecceboschristmasakilatiron-riopretoeidsvollovesickaruizawabostik-serverrankoshigayachtsandvikcoromantovalle-d-aostakinouebostonakijinsekikogentlentapisa-geekarumaifmemsetkmaxxn--12c1fe0bradescotksatmpaviancapitalonebouncemerckmsdscloudiybounty-fullensakerrypropertiesangovtoyosatoyokawaboutiquebecologialaichaugiangmbhartiengiangminakamichiharaboutireservdrangedalpusercontentoyotapfizerboyfriendoftheinternetflixn--12cfi8ixb8lublindesnesanjosoyrovnoticiasannanishinoshimattelemarkasaokamikitayamatsurinfinitigopocznore-og-uvdalucaniabozen-sudtiroluccanva-appstmnishiokoppegardray-dnsupdaterbozen-suedtirolukowesteuropencraftoyotomiyazakinsurealtypeformesswithdnsannohekinanporovigonohejinternationaluroybplacedogawarabikomaezakirunordkappgfoggiabrandrayddns5ybrasiliadboxoslockerbresciaogashimadachicappadovaapstemp-dnswatchest-mon-blogueurodirumagazinebrindisiciliabroadwaybroke-itvedestrandraydnsanokashibatakashimashikiyosatokigawabrokerbrothermesserlifestylebtimnetzpisdnpharmaciensantamariakebrowsersafetymarketingmodumetacentrumeteorappharmacymruovatlassian-dev-builderschaefflerbrumunddalutskashiharabrusselsantoandreclaimsanukintlon-2bryanskiptveterinaireadthedocsaobernardovre-eikerbrynebwestus2bzhitomirbzzwhitesnowflakecommunity-prochowicecomodalenissandoycompanyaarphdfcbankasumigaurawa-mazowszexn--1ck2e1bambinagisobetsuldalpha-myqnapcloudaccess3-us-east-2ixboxeroxfinityolasiteastus2comparemarkerryhotelsaves-the-whalessandria-trani-barletta-andriatranibarlettaandriacomsecaasnesoddeno-stagingrondarcondoshifteditorxn--1ctwolominamatarnobrzegrongrossetouchijiwadedyn-berlincolnissayokoshibahikariyaltakazakinzais-a-bookkeepermarshallstatebankasuyalibabahccavuotnagaraholtaleniwaizumiotsurugashimaintenanceomutazasavonarviikaminoyamaxunispaceconferenceconstructionflashdrivefsncf-ipfsaxoconsuladobeio-static-accesscamdvrcampaniaconsultantranoyconsultingroundhandlingroznysaitohnoshookuwanakayamangyshlakdnepropetrovskanlandyndns-freeboxostrowwlkpmgrphilipsyno-dschokokekscholarshipschoolbusinessebycontactivetrailcontagematsubaravendbambleborkdalvdalcest-le-patron-rancherkasydneyukuhashimokawavoues3-sa-east-1contractorskenissedalcookingruecoolblogdnsfor-better-thanhhoarairforcentralus-1cooperativano-frankivskodjeephonefosschoolsztynsetransiphotographysiocoproductionschulplattforminamiechizenisshingucciprianiigatairaumalatvuopmicrolightinguidefinimaringatlancastercorsicafjschulservercosenzakopanecosidnshome-webservercellikescandypopensocialcouchpotatofrieschwarzgwangjuh-ohtawaramotoineppueblockbusternopilawacouncilcouponscrapper-sitecozoravennaharimalborkaszubytemarketscrappinguitarscrysecretrosnubananarepublic-inquiryurihonjoyenthickaragandaxarnetbankanzakiwielunnerepairbusanagochigasakishimabarakawaharaolbia-tempio-olbiatempioolbialowiezachpomorskiengiangjesdalolipopmcdirepbodyn53cqcxn--1lqs03niyodogawacrankyotobetsumidaknongujaratmallcrdyndns-homednscwhminamifuranocreditcardyndns-iphutholdingservehttpbincheonl-ams-1creditunionionjukujitawaravpagecremonashorokanaiecrewhoswholidaycricketnedalcrimeast-kazakhstanangercrotonecrowniphuyencrsvp4cruiseservehumourcuisinellair-traffic-controllagdenesnaaseinet-freakserveircasertainaircraftingvolloansnasaarlanduponthewifidelitypedreamhostersaotomeldaluxurycuneocupcakecuritibacgiangiangryggeecurvalled-aostargets-itranslatedyndns-mailcutegirlfriendyndns-office-on-the-webhoptogurafedoraprojectransurlfeirafembetsukuis-a-bruinsfanfermodenakasatsunairportrapaniizaferraraferraris-a-bulls-fanferrerotikagoshimalopolskanittedalfetsundyndns-wikimobetsumitakagildeskaliszkolamericanfamilydservemp3fgunmaniwamannorth-kazakhstanfhvalerfilegear-augustowiiheyakagefilegear-deatnuniversitysvardofilegear-gbizfilegear-iefilegear-jpmorgangwonporterfilegear-sg-1filminamiizukamiminefinalchikugokasellfyis-a-candidatefinancefinnoyfirebaseappiemontefirenetlifylkesbiblackbaudcdn-edgestackhero-networkinggroupowiathletajimabaria-vungtaudiopsysharpigboatshawilliamhillfirenzefirestonefireweblikes-piedmontravelersinsurancefirmdalegalleryfishingoldpoint2thisamitsukefitjarfitnessettsurugiminamimakis-a-catererfjalerfkatsushikabeebyteappilottonsberguovdageaidnunjargausdalflekkefjordyndns-workservep2phxn--1lqs71dyndns-remotewdyndns-picserveminecraftransporteflesbergushikamifuranorthflankatsuyamashikokuchuoflickragerokunohealthcareershellflierneflirfloginlinefloppythonanywherealtorfloraflorencefloripalmasfjordenfloristanohatajiris-a-celticsfanfloromskogxn--2m4a15eflowershimokitayamafltravinhlonganflynnhosting-clusterfncashgabadaddjabbottoyourafndyndns1fnwkzfolldalfoolfor-ourfor-somegurownproviderfor-theaterfordebianforexrotheworkpccwinbar0emmafann-arborlandd-dnsiskinkyowariasahikawarszawashtenawsmppl-wawsglobalacceleratorahimeshimakanegasakievennodebalancern4t3l3p0rtatarantours3-ap-northeast-123minsidaarborteaches-yogano-ipifony-123miwebaccelastx4432-b-datacenterprisesakijobservableusercontentateshinanomachintaifun-dnsdojournalistoloseyouriparisor-fronavuotnarashinoharaetnabudejjunipereggio-emilia-romagnaroyboltateyamajureggiocalabriakrehamnayoro0o0forgotdnshimonitayanagithubpreviewsaikisarazure-mobileirfjordynnservepicservequakeforli-cesena-forlicesenaforlillehammerfeste-ipimientaketomisatoolshimonosekikawaforsalegoismailillesandefjordynservebbservesarcasmileforsandasuolodingenfortalfortefosneshimosuwalkis-a-chefashionstorebaseljordyndns-serverisignfotrdynulvikatowicefoxn--2scrj9casinordlandurbanamexnetgamersapporomurafozfr-1fr-par-1fr-par-2franamizuhoboleslawiecommerce-shoppingyeongnamdinhachijohanamakisofukushimaoris-a-conservativegarsheiheijis-a-cparachutingfredrikstadynv6freedesktopazimuthaibinhphuocelotenkawakayamagnetcieszynh-servebeero-stageiseiroumugifuchungbukharag-cloud-championshiphoplixn--30rr7yfreemyiphosteurovisionredumbrellangevagrigentobishimadridvagsoygardenebakkeshibechambagricoharugbydgoszczecin-berlindasdaburfreesitefreetlshimotsukefreisennankokubunjis-a-cubicle-slavellinodeobjectshimotsumafrenchkisshikindleikangerfreseniushinichinanfriuli-v-giuliafriuli-ve-giuliafriuli-vegiuliafriuli-venezia-giuliafriuli-veneziagiuliafriuli-vgiuliafriuliv-giuliafriulive-giuliafriulivegiuliafriulivenezia-giuliafriuliveneziagiuliafriulivgiuliafrlfroganshinjotelulubin-vpncateringebunkyonanaoshimamateramockashiwarafrognfrolandynvpnpluservicesevastopolitiendafrom-akamaized-stagingfrom-alfrom-arfrom-azurewebsiteshikagamiishibuyabukihokuizumobaragusabaerobaticketshinjukuleuvenicefrom-campobassociatest-iserveblogsytenrissadistdlibestadultrentin-sudtirolfrom-coachaseljeducationcillahppiacenzaganfrom-ctrentin-sued-tirolfrom-dcatfooddagestangefrom-decagliarikuzentakataikillfrom-flapymntrentin-suedtirolfrom-gap-east-1from-higashiagatsumagoianiafrom-iafrom-idyroyrvikingulenfrom-ilfrom-in-the-bandairtelebitbridgestonemurorangecloudplatform0from-kshinkamigototalfrom-kyfrom-langsonyantakahamalselveruminamiminowafrom-malvikaufentigerfrom-mdfrom-mein-vigorlicefrom-mifunefrom-mnfrom-modshinshinotsurgeryfrom-mshinshirofrom-mtnfrom-ncatholicurus-4from-ndfrom-nefrom-nhs-heilbronnoysundfrom-njshintokushimafrom-nminamioguni5from-nvalledaostargithubusercontentrentino-a-adigefrom-nycaxiaskvollpagesardegnarutolgaulardalvivanovoldafrom-ohdancefrom-okegawassamukawataris-a-democratrentino-aadigefrom-orfrom-panasonichernovtsykkylvenneslaskerrylogisticsardiniafrom-pratohmamurogawatsonrenderfrom-ris-a-designerimarugame-hostyhostingfrom-schmidtre-gauldalfrom-sdfrom-tnfrom-txn--32vp30hachinoheavyfrom-utsiracusagaeroclubmedecin-addrammenuorodoyerfrom-val-daostavalleyfrom-vtrentino-alto-adigefrom-wafrom-wiardwebthingsjcbnpparibashkiriafrom-wvallee-aosteroyfrom-wyfrosinonefrostabackplaneapplebesbyengerdalp1froyal-commissionfruskydivingfujiiderafujikawaguchikonefujiminokamoenairtrafficplexus-2fujinomiyadapliefujiokazakinkobearalvahkikonaibetsubame-south-1fujisatoshoeshintomikasaharafujisawafujishiroishidakabiratoridediboxn--3bst00minamisanrikubetsupportrentino-altoadigefujitsuruokakamigaharafujiyoshidappnodearthainguyenfukayabeardubaikawagoefukuchiyamadatsunanjoburgfukudomigawafukuis-a-doctorfukumitsubishigakirkeneshinyoshitomiokamisatokamachippubetsuikitchenfukuokakegawafukuroishikariwakunigamigrationfukusakirovogradoyfukuyamagatakaharunusualpersonfunabashiriuchinadattorelayfunagatakahashimamakiryuohkurafunahashikamiamakusatsumasendaisenergyeongginowaniihamatamakinoharafundfunkfeuerfuoiskujukuriyamandalfuosskoczowindowskrakowinefurubirafurudonordreisa-hockeynutwentertainmentrentino-s-tirolfurukawajimangolffanshiojirishirifujiedafusoctrangfussagamiharafutabayamaguchinomihachimanagementrentino-stirolfutboldlygoingnowhere-for-more-og-romsdalfuttsurutashinais-a-financialadvisor-aurdalfuturecmshioyamelhushirahamatonbetsurnadalfuturehostingfuturemailingfvghakuis-a-gurunzenhakusandnessjoenhaldenhalfmoonscalebookinghostedpictetrentino-sud-tirolhalsakakinokiaham-radio-opinbar1hamburghammarfeastasiahamurakamigoris-a-hard-workershiraokamisunagawahanamigawahanawahandavvesiidanangodaddyn-o-saurealestatefarmerseinehandcrafteducatorprojectrentino-sudtirolhangglidinghangoutrentino-sued-tirolhannannestadhannosegawahanoipinkazohanyuzenhappouzshiratakahagianghasamap-northeast-3hasaminami-alpshishikuis-a-hunterhashbanghasudazaifudaigodogadobeioruntimedio-campidano-mediocampidanomediohasura-appinokokamikoaniikappudopaashisogndalhasvikazteleportrentino-suedtirolhatogayahoooshikamagayaitakamoriokakudamatsuehatoyamazakitahiroshimarcheapartmentshisuifuettertdasnetzhatsukaichikaiseiyoichipshitaramahattfjelldalhayashimamotobusells-for-lesshizukuishimoichilloutsystemscloudsitehazuminobushibukawahelplfinancialhelsinkitakamiizumisanofidonnakamurataitogliattinnhemneshizuokamitondabayashiogamagoriziahemsedalhepforgeblockshoujis-a-knightpointtokaizukamaishikshacknetrentinoa-adigehetemlbfanhigashichichibuzentsujiiehigashihiroshimanehigashiizumozakitakatakanabeautychyattorneyagawakkanaioirasebastopoleangaviikadenagahamaroyhigashikagawahigashikagurasoedahigashikawakitaaikitakyushunantankazunovecorebungoonow-dnshowahigashikurumeinforumzhigashimatsushimarnardalhigashimatsuyamakitaakitadaitoigawahigashimurayamamotorcycleshowtimeloyhigashinarusells-for-uhigashinehigashiomitamanoshiroomghigashiosakasayamanakakogawahigashishirakawamatakanezawahigashisumiyoshikawaminamiaikitamihamadahigashitsunospamproxyhigashiurausukitamotosunnydayhigashiyamatokoriyamanashiibaclieu-1higashiyodogawahigashiyoshinogaris-a-landscaperspectakasakitanakagusukumoldeliveryhippyhiraizumisatohokkaidontexistmein-iservschulecznakaniikawatanagurahirakatashinagawahiranais-a-lawyerhirarahiratsukaeruhirayaizuwakamatsubushikusakadogawahitachiomiyaginozawaonsensiositehitachiotaketakaokalmykiahitraeumtgeradegreehjartdalhjelmelandholyhomegoodshwinnersiiitesilkddiamondsimple-urlhomeipioneerhomelinkyard-cloudjiffyresdalhomelinuxn--3ds443ghomeofficehomesecuritymacaparecidahomesecuritypchiryukyuragiizehomesenseeringhomeskleppippugliahomeunixn--3e0b707ehondahonjyoitakarazukaluganskfh-muensterhornindalhorsells-itrentinoaadigehortendofinternet-dnsimplesitehospitalhotelwithflightsirdalhotmailhoyangerhoylandetakasagooglecodespotrentinoalto-adigehungyenhurdalhurumajis-a-liberalhyllestadhyogoris-a-libertarianhyugawarahyundaiwafuneis-very-evillasalleitungsenis-very-goodyearis-very-niceis-very-sweetpepperugiais-with-thebandoomdnstraceisk01isk02jenv-arubacninhbinhdinhktistoryjeonnamegawajetztrentinostiroljevnakerjewelryjgorajlljls-sto1jls-sto2jls-sto3jmpixolinodeusercontentrentinosud-tiroljnjcloud-ver-jpchitosetogitsuliguriajoyokaichibahcavuotnagaivuotnagaokakyotambabymilk3jozis-a-musicianjpnjprsolarvikhersonlanxessolundbeckhmelnitskiyamasoykosaigawakosakaerodromegalloabatobamaceratachikawafaicloudineencoreapigeekoseis-a-painterhostsolutionslupskhakassiakosheroykoshimizumakis-a-patsfankoshughesomakosugekotohiradomainstitutekotourakouhokumakogenkounosupersalevangerkouyamasudakouzushimatrixn--3pxu8khplaystation-cloudyclusterkozagawakozakis-a-personaltrainerkozowiosomnarviklabudhabikinokawachinaganoharamcocottekpnkppspbarcelonagawakepnord-odalwaysdatabaseballangenkainanaejrietisalatinabenogiehtavuoatnaamesjevuemielnombrendlyngen-rootaruibxos3-us-gov-west-1krasnikahokutokonamegatakatoris-a-photographerokussldkrasnodarkredstonekrelliankristiansandcatsoowitdkmpspawnextdirectrentinosudtirolkristiansundkrodsheradkrokstadelvaldaostavangerkropyvnytskyis-a-playershiftcryptonomichinomiyakekryminamiyamashirokawanabelaudnedalnkumamotoyamatsumaebashimofusakatakatsukis-a-republicanonoichinosekigaharakumanowtvaokumatorinokumejimatsumotofukekumenanyokkaichirurgiens-dentistes-en-francekundenkunisakis-a-rockstarachowicekunitachiaraisaijolsterkunitomigusukukis-a-socialistgstagekunneppubtlsopotrentinosued-tirolkuokgroupizzakurgankurobegetmyipirangalluplidlugolekagaminorddalkurogimimozaokinawashirosatochiokinoshimagentositempurlkuroisodegaurakuromatsunais-a-soxfankuronkurotakikawasakis-a-studentalkushirogawakustanais-a-teacherkassyncloudkusuppliesor-odalkutchanelkutnokuzumakis-a-techietipslzkvafjordkvalsundkvamsterdamnserverbaniakvanangenkvinesdalkvinnheradkviteseidatingkvitsoykwpspdnsor-varangermishimatsusakahogirlymisugitokorozawamitakeharamitourismartlabelingmitoyoakemiuramiyazurecontainerdpoliticaobangmiyotamatsukuris-an-actormjondalenmonzabrianzaramonzaebrianzamonzaedellabrianzamordoviamorenapolicemoriyamatsuuramoriyoshiminamiashigaramormonstermoroyamatsuzakis-an-actressmushcdn77-sslingmortgagemoscowithgoogleapiszmoseushimogosenmosjoenmoskenesorreisahayakawakamiichikawamisatottoris-an-anarchistjordalshalsenmossortlandmosviknx-serversusakiyosupabaseminemotegit-reposoruminanomoviemovimientokyotangotembaixadattowebhareidsbergmozilla-iotrentinosuedtirolmtranbytomaridagawalmartrentinsud-tirolmuikaminokawanishiaizubangemukoelnmunakatanemuosattemupkomatsushimassa-carrara-massacarraramassabuzzmurmanskomforbar2murotorcraftranakatombetsumy-gatewaymusashinodesakegawamuseumincomcastoripressorfoldmusicapetownnews-stagingmutsuzawamy-vigormy-wanggoupilemyactivedirectorymyamazeplaymyasustor-elvdalmycdmycloudnsoundcastorjdevcloudfunctionsokndalmydattolocalcertificationmyddnsgeekgalaxymydissentrentinsudtirolmydobissmarterthanyoumydrobofageometre-experts-comptablesowamydspectruminisitemyeffectrentinsued-tirolmyfastly-edgekey-stagingmyfirewalledreplittlestargardmyforuminterecifedextraspace-to-rentalstomakomaibaramyfritzmyftpaccesspeedpartnermyhome-servermyjinomykolaivencloud66mymailermymediapchoseikarugalsacemyokohamamatsudamypeplatformsharis-an-artistockholmestrandmypetsphinxn--41amyphotoshibajddarvodkafjordvaporcloudmypictureshinomypsxn--42c2d9amysecuritycamerakermyshopblockspjelkavikommunalforbundmyshopifymyspreadshopselectrentinsuedtirolmytabitordermythic-beastspydebergmytis-a-anarchistg-buildermytuleap-partnersquaresindevicenzamyvnchoshichikashukudoyamakeuppermywirecipescaracallypoivronpokerpokrovskommunepolkowicepoltavalle-aostavernpomorzeszowithyoutuberspacekitagawaponpesaro-urbino-pesarourbinopesaromasvuotnaritakurashikis-bykleclerchitachinakagawaltervistaipeigersundynamic-dnsarlpordenonepornporsangerporsangugeporsgrunnanpoznanpraxihuanprdprgmrprimetelprincipeprivatelinkomonowruzhgorodeoprivatizehealthinsuranceprofesionalprogressivegasrlpromonza-e-della-brianzaptokuyamatsushigepropertysnesrvarggatrevisogneprotectionprotonetroandindependent-inquest-a-la-masionprudentialpruszkowiwatsukiyonotaireserve-onlineprvcyonabarumbriaprzeworskogpunyufuelpupulawypussycatanzarowixsitepvhachirogatakahatakaishimojis-a-geekautokeinotteroypvtrogstadpwchowderpzqhadanorthwesternmutualqldqotoyohashimotoshimaqponiatowadaqslgbtroitskomorotsukagawaqualifioapplatter-applatterplcube-serverquangngais-certifiedugit-pagespeedmobilizeroticaltanissettailscaleforcequangninhthuanquangtritonoshonais-foundationquickconnectromsakuragawaquicksytestreamlitapplumbingouvaresearchitectesrhtrentoyonakagyokutoyakomakizunokunimimatakasugais-an-engineeringquipelementstrippertuscanytushungrytuvalle-daostamayukis-into-animeiwamizawatuxfamilytuyenquangbinhthuantwmailvestnesuzukis-gonevestre-slidreggio-calabriavestre-totennishiawakuravestvagoyvevelstadvibo-valentiaavibovalentiavideovinhphuchromedicinagatorogerssarufutsunomiyawakasaikaitakokonoevinnicarbonia-iglesias-carboniaiglesiascarboniavinnytsiavipsinaapplurinacionalvirginanmokurennebuvirtual-userveexchangevirtualservervirtualuserveftpodhalevisakurais-into-carsnoasakuholeckodairaviterboliviajessheimmobilienvivianvivoryvixn--45br5cylvlaanderennesoyvladikavkazimierz-dolnyvladimirvlogintoyonezawavmintsorocabalashovhachiojiyahikobierzycevologdanskoninjambylvolvolkswagencyouvolyngdalvoorlopervossevangenvotevotingvotoyonovps-hostrowiechungnamdalseidfjordynathomebuiltwithdarkhangelskypecorittogojomeetoystre-slidrettozawawmemergencyahabackdropalermochizukikirarahkkeravjuwmflabsvalbardunloppadualstackomvuxn--3hcrj9chonanbuskerudynamisches-dnsarpsborgripeeweeklylotterywoodsidellogliastradingworse-thanhphohochiminhadselbuyshouseshirakolobrzegersundongthapmircloudletshiranukamishihorowowloclawekonskowolawawpdevcloudwpenginepoweredwphostedmailwpmucdnipropetrovskygearappodlasiellaknoluoktagajobojis-an-entertainerwpmudevcdnaccessojamparaglidingwritesthisblogoipodzonewroclawmcloudwsseoullensvanguardianwtcp4wtfastlylbanzaicloudappspotagereporthruherecreationinomiyakonojorpelandigickarasjohkameyamatotakadawuozuerichardlillywzmiuwajimaxn--4it797konsulatrobeepsondriobranconagareyamaizuruhrxn--4pvxs4allxn--54b7fta0ccistrondheimpertrixcdn77-secureadymadealstahaugesunderxn--55qw42gxn--55qx5dxn--5dbhl8dxn--5js045dxn--5rtp49citadelhichisochimkentozsdell-ogliastraderxn--5rtq34kontuminamiuonumatsunoxn--5su34j936bgsgxn--5tzm5gxn--6btw5axn--6frz82gxn--6orx2rxn--6qq986b3xlxn--7t0a264citicarrdrobakamaiorigin-stagingmxn--12co0c3b4evalleaostaobaomoriguchiharaffleentrycloudflare-ipfstcgroupaaskimitsubatamibulsan-suedtirolkuszczytnoopscbgrimstadrrxn--80aaa0cvacationsvchoyodobashichinohealth-carereforminamidaitomanaustdalxn--80adxhksveioxn--80ao21axn--80aqecdr1axn--80asehdbarclaycards3-us-west-1xn--80aswgxn--80aukraanghkeliwebpaaskoyabeagleboardxn--8dbq2axn--8ltr62konyvelohmusashimurayamassivegridxn--8pvr4uxn--8y0a063axn--90a1affinitylotterybnikeisencowayxn--90a3academiamicable-modemoneyxn--90aeroportsinfolionetworkangerxn--90aishobaraxn--90amckinseyxn--90azhytomyrxn--9dbq2axn--9et52uxn--9krt00axn--andy-iraxn--aroport-byanagawaxn--asky-iraxn--aurskog-hland-jnbarclays3-us-west-2xn--avery-yuasakurastoragexn--b-5gaxn--b4w605ferdxn--balsan-sdtirol-nsbsvelvikongsbergxn--bck1b9a5dre4civilaviationfabricafederation-webredirectmediatechnologyeongbukashiwazakiyosembokutamamuraxn--bdddj-mrabdxn--bearalvhki-y4axn--berlevg-jxaxn--bhcavuotna-s4axn--bhccavuotna-k7axn--bidr-5nachikatsuuraxn--bievt-0qa2xn--bjarky-fyanaizuxn--bjddar-ptarumizusawaxn--blt-elabcienciamallamaceiobbcn-north-1xn--bmlo-graingerxn--bod-2natalxn--bozen-sdtirol-2obanazawaxn--brnny-wuacademy-firewall-gatewayxn--brnnysund-m8accident-investigation-aptibleadpagesquare7xn--brum-voagatrustkanazawaxn--btsfjord-9zaxn--bulsan-sdtirol-nsbarefootballooningjovikarasjoketokashikiyokawaraxn--c1avgxn--c2br7gxn--c3s14misakis-a-therapistoiaxn--cck2b3baremetalombardyn-vpndns3-website-ap-northeast-1xn--cckwcxetdxn--cesena-forl-mcbremangerxn--cesenaforl-i8axn--cg4bkis-into-cartoonsokamitsuexn--ciqpnxn--clchc0ea0b2g2a9gcdxn--czr694bargainstantcloudfrontdoorestauranthuathienhuebinordre-landiherokuapparochernigovernmentjeldsundiscordsays3-website-ap-southeast-1xn--czrs0trvaroyxn--czru2dxn--czrw28barrel-of-knowledgeapplinziitatebayashijonawatebizenakanojoetsumomodellinglassnillfjordiscordsezgoraxn--d1acj3barrell-of-knowledgecomputermezproxyzgorzeleccoffeedbackanagawarmiastalowa-wolayangroupars3-website-ap-southeast-2xn--d1alfaststacksevenassigdalxn--d1atrysiljanxn--d5qv7z876clanbibaiduckdnsaseboknowsitallxn--davvenjrga-y4axn--djrs72d6uyxn--djty4koobindalxn--dnna-grajewolterskluwerxn--drbak-wuaxn--dyry-iraxn--e1a4cldmail-boxaxn--eckvdtc9dxn--efvn9svn-repostuff-4-salexn--efvy88haebaruericssongdalenviknaklodzkochikushinonsenasakuchinotsuchiurakawaxn--ehqz56nxn--elqq16hagakhanhhoabinhduongxn--eveni-0qa01gaxn--f6qx53axn--fct429kooris-a-nascarfanxn--fhbeiarnxn--finny-yuaxn--fiq228c5hsbcleverappsassarinuyamashinazawaxn--fiq64barsycenterprisecloudcontrolappgafanquangnamasteigenoamishirasatochigifts3-website-eu-west-1xn--fiqs8swidnicaravanylvenetogakushimotoganexn--fiqz9swidnikitagatakkomaganexn--fjord-lraxn--fjq720axn--fl-ziaxn--flor-jraxn--flw351exn--forl-cesena-fcbsswiebodzindependent-commissionxn--forlcesena-c8axn--fpcrj9c3dxn--frde-granexn--frna-woaxn--frya-hraxn--fzc2c9e2clickrisinglesjaguarxn--fzys8d69uvgmailxn--g2xx48clinicasacampinagrandebungotakadaemongolianishitosashimizunaminamiawajikintuitoyotsukaidownloadrudtvsaogoncapooguyxn--gckr3f0fastvps-serveronakanotoddenxn--gecrj9cliniquedaklakasamatsudoesntexisteingeekasserversicherungroks-theatrentin-sud-tirolxn--ggaviika-8ya47hagebostadxn--gildeskl-g0axn--givuotna-8yandexcloudxn--gjvik-wuaxn--gk3at1exn--gls-elacaixaxn--gmq050is-into-gamessinamsosnowieconomiasadojin-dslattuminamitanexn--gmqw5axn--gnstigbestellen-zvbrplsbxn--45brj9churcharterxn--gnstigliefern-wobihirosakikamijimayfirstorfjordxn--h-2failxn--h1ahnxn--h1alizxn--h2breg3eveneswinoujsciencexn--h2brj9c8clothingdustdatadetectrani-andria-barletta-trani-andriaxn--h3cuzk1dienbienxn--hbmer-xqaxn--hcesuolo-7ya35barsyonlinehimejiiyamanouchikujoinvilleirvikarasuyamashikemrevistathellequipmentjmaxxxjavald-aostatics3-website-sa-east-1xn--hebda8basicserversejny-2xn--hery-iraxn--hgebostad-g3axn--hkkinen-5waxn--hmmrfeasta-s4accident-prevention-k3swisstufftoread-booksnestudioxn--hnefoss-q1axn--hobl-iraxn--holtlen-hxaxn--hpmir-xqaxn--hxt814exn--hyanger-q1axn--hylandet-54axn--i1b6b1a6a2exn--imr513nxn--indery-fyaotsusonoxn--io0a7is-leetrentinoaltoadigexn--j1adpohlxn--j1aefauskedsmokorsetagayaseralingenovaraxn--j1ael8basilicataniaxn--j1amhaibarakisosakitahatakamatsukawaxn--j6w193gxn--jlq480n2rgxn--jlster-byasakaiminatoyookananiimiharuxn--jrpeland-54axn--jvr189misasaguris-an-accountantsmolaquilaocais-a-linux-useranishiaritabashikaoizumizakitashiobaraxn--k7yn95exn--karmy-yuaxn--kbrq7oxn--kcrx77d1x4axn--kfjord-iuaxn--klbu-woaxn--klt787dxn--kltp7dxn--kltx9axn--klty5xn--45q11circlerkstagentsasayamaxn--koluokta-7ya57haiduongxn--kprw13dxn--kpry57dxn--kput3is-lostre-toteneis-a-llamarumorimachidaxn--krager-gyasugitlabbvieeexn--kranghke-b0axn--krdsherad-m8axn--krehamn-dxaxn--krjohka-hwab49jdfastly-terrariuminamiiseharaxn--ksnes-uuaxn--kvfjord-nxaxn--kvitsy-fyasuokanmakiwakuratexn--kvnangen-k0axn--l-1fairwindsynology-diskstationxn--l1accentureklamborghinikkofuefukihabororosynology-dsuzakadnsaliastudynaliastrynxn--laheadju-7yatominamibosoftwarendalenugxn--langevg-jxaxn--lcvr32dxn--ldingen-q1axn--leagaviika-52basketballfinanzjaworznoticeableksvikaratsuginamikatagamilanotogawaxn--lesund-huaxn--lgbbat1ad8jejuxn--lgrd-poacctulaspeziaxn--lhppi-xqaxn--linds-pramericanexpresservegame-serverxn--loabt-0qaxn--lrdal-sraxn--lrenskog-54axn--lt-liacn-northwest-1xn--lten-granvindafjordxn--lury-iraxn--m3ch0j3axn--mely-iraxn--merker-kuaxn--mgb2ddesxn--mgb9awbfbsbxn--1qqw23axn--mgba3a3ejtunesuzukamogawaxn--mgba3a4f16axn--mgba3a4fra1-deloittexn--mgba7c0bbn0axn--mgbaakc7dvfsxn--mgbaam7a8haiphongonnakatsugawaxn--mgbab2bdxn--mgbah1a3hjkrdxn--mgbai9a5eva00batsfjordiscountry-snowplowiczeladzlgleezeu-2xn--mgbai9azgqp6jelasticbeanstalkharkovalleeaostexn--mgbayh7gparasitexn--mgbbh1a71exn--mgbc0a9azcgxn--mgbca7dzdoxn--mgbcpq6gpa1axn--mgberp4a5d4a87gxn--mgberp4a5d4arxn--mgbgu82axn--mgbi4ecexposedxn--mgbpl2fhskopervikhmelnytskyivalleedaostexn--mgbqly7c0a67fbcngroks-thisayamanobeatsaudaxn--mgbqly7cvafricargoboavistanbulsan-sudtirolxn--mgbt3dhdxn--mgbtf8flatangerxn--mgbtx2bauhauspostman-echofunatoriginstances3-website-us-east-1xn--mgbx4cd0abkhaziaxn--mix082fbx-osewienxn--mix891fbxosexyxn--mjndalen-64axn--mk0axindependent-inquiryxn--mk1bu44cnpyatigorskjervoyagexn--mkru45is-not-certifiedxn--mlatvuopmi-s4axn--mli-tlavagiskexn--mlselv-iuaxn--moreke-juaxn--mori-qsakuratanxn--mosjen-eyatsukannamihokksundxn--mot-tlavangenxn--mre-og-romsdal-qqbuservecounterstrikexn--msy-ula0hair-surveillancexn--mtta-vrjjat-k7aflakstadaokayamazonaws-cloud9guacuiababybluebiteckidsmynasushiobaracingrok-freeddnsfreebox-osascoli-picenogatabuseating-organicbcgjerdrumcprequalifymelbourneasypanelblagrarq-authgear-stagingjerstadeltaishinomakilovecollegefantasyleaguenoharauthgearappspacehosted-by-previderehabmereitattoolforgerockyombolzano-altoadigeorgeorgiauthordalandroideporteatonamidorivnebetsukubankanumazuryomitanocparmautocodebergamoarekembuchikumagayagawafflecelloisirs3-external-180reggioemiliaromagnarusawaustrheimbalsan-sudtirolivingitpagexlivornobserveregruhostingivestbyglandroverhalladeskjakamaiedge-stagingivingjemnes3-eu-west-2038xn--muost-0qaxn--mxtq1misawaxn--ngbc5azdxn--ngbe9e0axn--ngbrxn--4dbgdty6ciscofreakamaihd-stagingriwataraindroppdalxn--nit225koryokamikawanehonbetsuwanouchikuhokuryugasakis-a-nursellsyourhomeftpiwatexn--nmesjevuemie-tcbalatinord-frontierxn--nnx388axn--nodessakurawebsozais-savedxn--nqv7fs00emaxn--nry-yla5gxn--ntso0iqx3axn--ntsq17gxn--nttery-byaeservehalflifeinsurancexn--nvuotna-hwaxn--nyqy26axn--o1achernivtsicilynxn--4dbrk0cexn--o3cw4hakatanortonkotsunndalxn--o3cyx2axn--od0algardxn--od0aq3beneventodayusuharaxn--ogbpf8fldrvelvetromsohuissier-justicexn--oppegrd-ixaxn--ostery-fyatsushiroxn--osyro-wuaxn--otu796dxn--p1acfedjeezxn--p1ais-slickharkivallee-d-aostexn--pgbs0dhlx3xn--porsgu-sta26fedorainfraclouderaxn--pssu33lxn--pssy2uxn--q7ce6axn--q9jyb4cnsauheradyndns-at-homedepotenzamamicrosoftbankasukabedzin-brbalsfjordietgoryoshiokanravocats3-fips-us-gov-west-1xn--qcka1pmcpenzapposxn--qqqt11misconfusedxn--qxa6axn--qxamunexus-3xn--rady-iraxn--rdal-poaxn--rde-ulazioxn--rdy-0nabaris-uberleetrentinos-tirolxn--rennesy-v1axn--rhkkervju-01afedorapeoplefrakkestadyndns-webhostingujogaszxn--rholt-mragowoltlab-democraciaxn--rhqv96gxn--rht27zxn--rht3dxn--rht61exn--risa-5naturalxn--risr-iraxn--rland-uuaxn--rlingen-mxaxn--rmskog-byawaraxn--rny31hakodatexn--rovu88bentleyusuitatamotorsitestinglitchernihivgubs3-website-us-west-1xn--rros-graphicsxn--rskog-uuaxn--rst-0naturbruksgymnxn--rsta-framercanvasxn--rvc1e0am3exn--ryken-vuaxn--ryrvik-byawatahamaxn--s-1faitheshopwarezzoxn--s9brj9cntraniandriabarlettatraniandriaxn--sandnessjen-ogbentrendhostingliwiceu-3xn--sandy-yuaxn--sdtirol-n2axn--seral-lraxn--ses554gxn--sgne-graphoxn--4gbriminiserverxn--skierv-utazurestaticappspaceusercontentunkongsvingerxn--skjervy-v1axn--skjk-soaxn--sknit-yqaxn--sknland-fxaxn--slat-5navigationxn--slt-elabogadobeaemcloud-fr1xn--smla-hraxn--smna-gratangenxn--snase-nraxn--sndre-land-0cbeppublishproxyuufcfanirasakindependent-panelomonza-brianzaporizhzhedmarkarelianceu-4xn--snes-poaxn--snsa-roaxn--sr-aurdal-l8axn--sr-fron-q1axn--sr-odal-q1axn--sr-varanger-ggbeskidyn-ip24xn--srfold-byaxn--srreisa-q1axn--srum-gratis-a-bloggerxn--stfold-9xaxn--stjrdal-s1axn--stjrdalshalsen-sqbestbuyshoparenagasakikuchikuseihicampinashikiminohostfoldnavyuzawaxn--stre-toten-zcbetainaboxfuselfipartindependent-reviewegroweibolognagasukeu-north-1xn--t60b56axn--tckweddingxn--tiq49xqyjelenia-goraxn--tjme-hraxn--tn0agrocerydxn--tnsberg-q1axn--tor131oxn--trany-yuaxn--trentin-sd-tirol-rzbhzc66xn--trentin-sdtirol-7vbialystokkeymachineu-south-1xn--trentino-sd-tirol-c3bielawakuyachimataharanzanishiazaindielddanuorrindigenamerikawauevje-og-hornnes3-website-us-west-2xn--trentino-sdtirol-szbiella-speziaxn--trentinosd-tirol-rzbieszczadygeyachiyodaeguamfamscompute-1xn--trentinosdtirol-7vbievat-band-campaignieznoorstaplesakyotanabellunordeste-idclkarlsoyxn--trentinsd-tirol-6vbifukagawalbrzycharitydalomzaporizhzhiaxn--trentinsdtirol-nsbigv-infolkebiblegnicalvinklein-butterhcloudiscoursesalangenishigotpantheonsitexn--trgstad-r1axn--trna-woaxn--troms-zuaxn--tysvr-vraxn--uc0atventuresinstagingxn--uc0ay4axn--uist22hakonexn--uisz3gxn--unjrga-rtashkenturindalxn--unup4yxn--uuwu58axn--vads-jraxn--valle-aoste-ebbturystykaneyamazoexn--valle-d-aoste-ehboehringerikexn--valleaoste-e7axn--valledaoste-ebbvadsoccertmgreaterxn--vard-jraxn--vegrshei-c0axn--vermgensberater-ctb-hostingxn--vermgensberatung-pwbiharstadotsubetsugarulezajskiervaksdalondonetskarmoyxn--vestvgy-ixa6oxn--vg-yiabruzzombieidskogasawarackmazerbaijan-mayenbaidarmeniaxn--vgan-qoaxn--vgsy-qoa0jellybeanxn--vgu402coguchikuzenishiwakinvestmentsaveincloudyndns-at-workisboringsakershusrcfdyndns-blogsitexn--vhquvestfoldxn--vler-qoaxn--vre-eiker-k8axn--vrggt-xqadxn--vry-yla5gxn--vuq861bihoronobeokagakikugawalesundiscoverdalondrinaplesknsalon-1xn--w4r85el8fhu5dnraxn--w4rs40lxn--wcvs22dxn--wgbh1communexn--wgbl6axn--xhq521bikedaejeonbuk0xn--xkc2al3hye2axn--xkc2dl3a5ee0hakubackyardshiraois-a-greenxn--y9a3aquarelleasingxn--yer-znavois-very-badxn--yfro4i67oxn--ygarden-p1axn--ygbi2ammxn--4it168dxn--ystre-slidre-ujbiofficialorenskoglobodoes-itcouldbeworldishangrilamdongnairkitapps-audibleasecuritytacticsxn--0trq7p7nnishiharaxn--zbx025dxn--zf0ao64axn--zf0avxlxn--zfr164bipartsaloonishiizunazukindustriaxnbayernxz
\ No newline at end of file
diff --git a/publicsuffix/example_test.go b/publicsuffix/example_test.go
index 3f44dcfe75..c051dac8e0 100644
--- a/publicsuffix/example_test.go
+++ b/publicsuffix/example_test.go
@@ -77,7 +77,7 @@ func ExamplePublicSuffix_manager() {
// > golang.dev dev is ICANN Managed
// > golang.net net is ICANN Managed
// > play.golang.org org is ICANN Managed
- // > gophers.in.space.museum space.museum is ICANN Managed
+ // > gophers.in.space.museum museum is ICANN Managed
// >
// > 0emm.com com is ICANN Managed
// > a.0emm.com a.0emm.com is Privately Managed
diff --git a/publicsuffix/table.go b/publicsuffix/table.go
index 6bdadcc448..78d400fa65 100644
--- a/publicsuffix/table.go
+++ b/publicsuffix/table.go
@@ -4,7 +4,7 @@ package publicsuffix
import _ "embed"
-const version = "publicsuffix.org's public_suffix_list.dat, git revision e248cbc92a527a166454afe9914c4c1b4253893f (2022-11-15T18:02:38Z)"
+const version = "publicsuffix.org's public_suffix_list.dat, git revision 63cbc63d470d7b52c35266aa96c4c98c96ec499c (2023-08-03T10:01:25Z)"
const (
nodesBits = 40
@@ -26,7 +26,7 @@ const (
)
// numTLD is the number of top level domains.
-const numTLD = 1494
+const numTLD = 1474
// text is the combined text of all labels.
//
@@ -63,8 +63,8 @@ var nodes uint40String
//go:embed data/children
var children uint32String
-// max children 718 (capacity 1023)
-// max text offset 32976 (capacity 65535)
-// max text length 36 (capacity 63)
-// max hi 9656 (capacity 16383)
-// max lo 9651 (capacity 16383)
+// max children 743 (capacity 1023)
+// max text offset 30876 (capacity 65535)
+// max text length 31 (capacity 63)
+// max hi 9322 (capacity 16383)
+// max lo 9317 (capacity 16383)
diff --git a/publicsuffix/table_test.go b/publicsuffix/table_test.go
index 99698271a9..a297b3b0dd 100644
--- a/publicsuffix/table_test.go
+++ b/publicsuffix/table_test.go
@@ -2,7 +2,7 @@
package publicsuffix
-const numICANNRules = 7367
+const numICANNRules = 6893
var rules = [...]string{
"ac",
@@ -302,9 +302,26 @@ var rules = [...]string{
"org.bi",
"biz",
"bj",
- "asso.bj",
- "barreau.bj",
- "gouv.bj",
+ "africa.bj",
+ "agro.bj",
+ "architectes.bj",
+ "assur.bj",
+ "avocats.bj",
+ "co.bj",
+ "com.bj",
+ "eco.bj",
+ "econo.bj",
+ "edu.bj",
+ "info.bj",
+ "loisirs.bj",
+ "money.bj",
+ "net.bj",
+ "org.bj",
+ "ote.bj",
+ "resto.bj",
+ "restaurant.bj",
+ "tourism.bj",
+ "univ.bj",
"bm",
"com.bm",
"edu.bm",
@@ -3596,552 +3613,6 @@ var rules = [...]string{
"co.mu",
"or.mu",
"museum",
- "academy.museum",
- "agriculture.museum",
- "air.museum",
- "airguard.museum",
- "alabama.museum",
- "alaska.museum",
- "amber.museum",
- "ambulance.museum",
- "american.museum",
- "americana.museum",
- "americanantiques.museum",
- "americanart.museum",
- "amsterdam.museum",
- "and.museum",
- "annefrank.museum",
- "anthro.museum",
- "anthropology.museum",
- "antiques.museum",
- "aquarium.museum",
- "arboretum.museum",
- "archaeological.museum",
- "archaeology.museum",
- "architecture.museum",
- "art.museum",
- "artanddesign.museum",
- "artcenter.museum",
- "artdeco.museum",
- "arteducation.museum",
- "artgallery.museum",
- "arts.museum",
- "artsandcrafts.museum",
- "asmatart.museum",
- "assassination.museum",
- "assisi.museum",
- "association.museum",
- "astronomy.museum",
- "atlanta.museum",
- "austin.museum",
- "australia.museum",
- "automotive.museum",
- "aviation.museum",
- "axis.museum",
- "badajoz.museum",
- "baghdad.museum",
- "bahn.museum",
- "bale.museum",
- "baltimore.museum",
- "barcelona.museum",
- "baseball.museum",
- "basel.museum",
- "baths.museum",
- "bauern.museum",
- "beauxarts.museum",
- "beeldengeluid.museum",
- "bellevue.museum",
- "bergbau.museum",
- "berkeley.museum",
- "berlin.museum",
- "bern.museum",
- "bible.museum",
- "bilbao.museum",
- "bill.museum",
- "birdart.museum",
- "birthplace.museum",
- "bonn.museum",
- "boston.museum",
- "botanical.museum",
- "botanicalgarden.museum",
- "botanicgarden.museum",
- "botany.museum",
- "brandywinevalley.museum",
- "brasil.museum",
- "bristol.museum",
- "british.museum",
- "britishcolumbia.museum",
- "broadcast.museum",
- "brunel.museum",
- "brussel.museum",
- "brussels.museum",
- "bruxelles.museum",
- "building.museum",
- "burghof.museum",
- "bus.museum",
- "bushey.museum",
- "cadaques.museum",
- "california.museum",
- "cambridge.museum",
- "can.museum",
- "canada.museum",
- "capebreton.museum",
- "carrier.museum",
- "cartoonart.museum",
- "casadelamoneda.museum",
- "castle.museum",
- "castres.museum",
- "celtic.museum",
- "center.museum",
- "chattanooga.museum",
- "cheltenham.museum",
- "chesapeakebay.museum",
- "chicago.museum",
- "children.museum",
- "childrens.museum",
- "childrensgarden.museum",
- "chiropractic.museum",
- "chocolate.museum",
- "christiansburg.museum",
- "cincinnati.museum",
- "cinema.museum",
- "circus.museum",
- "civilisation.museum",
- "civilization.museum",
- "civilwar.museum",
- "clinton.museum",
- "clock.museum",
- "coal.museum",
- "coastaldefence.museum",
- "cody.museum",
- "coldwar.museum",
- "collection.museum",
- "colonialwilliamsburg.museum",
- "coloradoplateau.museum",
- "columbia.museum",
- "columbus.museum",
- "communication.museum",
- "communications.museum",
- "community.museum",
- "computer.museum",
- "computerhistory.museum",
- "xn--comunicaes-v6a2o.museum",
- "contemporary.museum",
- "contemporaryart.museum",
- "convent.museum",
- "copenhagen.museum",
- "corporation.museum",
- "xn--correios-e-telecomunicaes-ghc29a.museum",
- "corvette.museum",
- "costume.museum",
- "countryestate.museum",
- "county.museum",
- "crafts.museum",
- "cranbrook.museum",
- "creation.museum",
- "cultural.museum",
- "culturalcenter.museum",
- "culture.museum",
- "cyber.museum",
- "cymru.museum",
- "dali.museum",
- "dallas.museum",
- "database.museum",
- "ddr.museum",
- "decorativearts.museum",
- "delaware.museum",
- "delmenhorst.museum",
- "denmark.museum",
- "depot.museum",
- "design.museum",
- "detroit.museum",
- "dinosaur.museum",
- "discovery.museum",
- "dolls.museum",
- "donostia.museum",
- "durham.museum",
- "eastafrica.museum",
- "eastcoast.museum",
- "education.museum",
- "educational.museum",
- "egyptian.museum",
- "eisenbahn.museum",
- "elburg.museum",
- "elvendrell.museum",
- "embroidery.museum",
- "encyclopedic.museum",
- "england.museum",
- "entomology.museum",
- "environment.museum",
- "environmentalconservation.museum",
- "epilepsy.museum",
- "essex.museum",
- "estate.museum",
- "ethnology.museum",
- "exeter.museum",
- "exhibition.museum",
- "family.museum",
- "farm.museum",
- "farmequipment.museum",
- "farmers.museum",
- "farmstead.museum",
- "field.museum",
- "figueres.museum",
- "filatelia.museum",
- "film.museum",
- "fineart.museum",
- "finearts.museum",
- "finland.museum",
- "flanders.museum",
- "florida.museum",
- "force.museum",
- "fortmissoula.museum",
- "fortworth.museum",
- "foundation.museum",
- "francaise.museum",
- "frankfurt.museum",
- "franziskaner.museum",
- "freemasonry.museum",
- "freiburg.museum",
- "fribourg.museum",
- "frog.museum",
- "fundacio.museum",
- "furniture.museum",
- "gallery.museum",
- "garden.museum",
- "gateway.museum",
- "geelvinck.museum",
- "gemological.museum",
- "geology.museum",
- "georgia.museum",
- "giessen.museum",
- "glas.museum",
- "glass.museum",
- "gorge.museum",
- "grandrapids.museum",
- "graz.museum",
- "guernsey.museum",
- "halloffame.museum",
- "hamburg.museum",
- "handson.museum",
- "harvestcelebration.museum",
- "hawaii.museum",
- "health.museum",
- "heimatunduhren.museum",
- "hellas.museum",
- "helsinki.museum",
- "hembygdsforbund.museum",
- "heritage.museum",
- "histoire.museum",
- "historical.museum",
- "historicalsociety.museum",
- "historichouses.museum",
- "historisch.museum",
- "historisches.museum",
- "history.museum",
- "historyofscience.museum",
- "horology.museum",
- "house.museum",
- "humanities.museum",
- "illustration.museum",
- "imageandsound.museum",
- "indian.museum",
- "indiana.museum",
- "indianapolis.museum",
- "indianmarket.museum",
- "intelligence.museum",
- "interactive.museum",
- "iraq.museum",
- "iron.museum",
- "isleofman.museum",
- "jamison.museum",
- "jefferson.museum",
- "jerusalem.museum",
- "jewelry.museum",
- "jewish.museum",
- "jewishart.museum",
- "jfk.museum",
- "journalism.museum",
- "judaica.museum",
- "judygarland.museum",
- "juedisches.museum",
- "juif.museum",
- "karate.museum",
- "karikatur.museum",
- "kids.museum",
- "koebenhavn.museum",
- "koeln.museum",
- "kunst.museum",
- "kunstsammlung.museum",
- "kunstunddesign.museum",
- "labor.museum",
- "labour.museum",
- "lajolla.museum",
- "lancashire.museum",
- "landes.museum",
- "lans.museum",
- "xn--lns-qla.museum",
- "larsson.museum",
- "lewismiller.museum",
- "lincoln.museum",
- "linz.museum",
- "living.museum",
- "livinghistory.museum",
- "localhistory.museum",
- "london.museum",
- "losangeles.museum",
- "louvre.museum",
- "loyalist.museum",
- "lucerne.museum",
- "luxembourg.museum",
- "luzern.museum",
- "mad.museum",
- "madrid.museum",
- "mallorca.museum",
- "manchester.museum",
- "mansion.museum",
- "mansions.museum",
- "manx.museum",
- "marburg.museum",
- "maritime.museum",
- "maritimo.museum",
- "maryland.museum",
- "marylhurst.museum",
- "media.museum",
- "medical.museum",
- "medizinhistorisches.museum",
- "meeres.museum",
- "memorial.museum",
- "mesaverde.museum",
- "michigan.museum",
- "midatlantic.museum",
- "military.museum",
- "mill.museum",
- "miners.museum",
- "mining.museum",
- "minnesota.museum",
- "missile.museum",
- "missoula.museum",
- "modern.museum",
- "moma.museum",
- "money.museum",
- "monmouth.museum",
- "monticello.museum",
- "montreal.museum",
- "moscow.museum",
- "motorcycle.museum",
- "muenchen.museum",
- "muenster.museum",
- "mulhouse.museum",
- "muncie.museum",
- "museet.museum",
- "museumcenter.museum",
- "museumvereniging.museum",
- "music.museum",
- "national.museum",
- "nationalfirearms.museum",
- "nationalheritage.museum",
- "nativeamerican.museum",
- "naturalhistory.museum",
- "naturalhistorymuseum.museum",
- "naturalsciences.museum",
- "nature.museum",
- "naturhistorisches.museum",
- "natuurwetenschappen.museum",
- "naumburg.museum",
- "naval.museum",
- "nebraska.museum",
- "neues.museum",
- "newhampshire.museum",
- "newjersey.museum",
- "newmexico.museum",
- "newport.museum",
- "newspaper.museum",
- "newyork.museum",
- "niepce.museum",
- "norfolk.museum",
- "north.museum",
- "nrw.museum",
- "nyc.museum",
- "nyny.museum",
- "oceanographic.museum",
- "oceanographique.museum",
- "omaha.museum",
- "online.museum",
- "ontario.museum",
- "openair.museum",
- "oregon.museum",
- "oregontrail.museum",
- "otago.museum",
- "oxford.museum",
- "pacific.museum",
- "paderborn.museum",
- "palace.museum",
- "paleo.museum",
- "palmsprings.museum",
- "panama.museum",
- "paris.museum",
- "pasadena.museum",
- "pharmacy.museum",
- "philadelphia.museum",
- "philadelphiaarea.museum",
- "philately.museum",
- "phoenix.museum",
- "photography.museum",
- "pilots.museum",
- "pittsburgh.museum",
- "planetarium.museum",
- "plantation.museum",
- "plants.museum",
- "plaza.museum",
- "portal.museum",
- "portland.museum",
- "portlligat.museum",
- "posts-and-telecommunications.museum",
- "preservation.museum",
- "presidio.museum",
- "press.museum",
- "project.museum",
- "public.museum",
- "pubol.museum",
- "quebec.museum",
- "railroad.museum",
- "railway.museum",
- "research.museum",
- "resistance.museum",
- "riodejaneiro.museum",
- "rochester.museum",
- "rockart.museum",
- "roma.museum",
- "russia.museum",
- "saintlouis.museum",
- "salem.museum",
- "salvadordali.museum",
- "salzburg.museum",
- "sandiego.museum",
- "sanfrancisco.museum",
- "santabarbara.museum",
- "santacruz.museum",
- "santafe.museum",
- "saskatchewan.museum",
- "satx.museum",
- "savannahga.museum",
- "schlesisches.museum",
- "schoenbrunn.museum",
- "schokoladen.museum",
- "school.museum",
- "schweiz.museum",
- "science.museum",
- "scienceandhistory.museum",
- "scienceandindustry.museum",
- "sciencecenter.museum",
- "sciencecenters.museum",
- "science-fiction.museum",
- "sciencehistory.museum",
- "sciences.museum",
- "sciencesnaturelles.museum",
- "scotland.museum",
- "seaport.museum",
- "settlement.museum",
- "settlers.museum",
- "shell.museum",
- "sherbrooke.museum",
- "sibenik.museum",
- "silk.museum",
- "ski.museum",
- "skole.museum",
- "society.museum",
- "sologne.museum",
- "soundandvision.museum",
- "southcarolina.museum",
- "southwest.museum",
- "space.museum",
- "spy.museum",
- "square.museum",
- "stadt.museum",
- "stalbans.museum",
- "starnberg.museum",
- "state.museum",
- "stateofdelaware.museum",
- "station.museum",
- "steam.museum",
- "steiermark.museum",
- "stjohn.museum",
- "stockholm.museum",
- "stpetersburg.museum",
- "stuttgart.museum",
- "suisse.museum",
- "surgeonshall.museum",
- "surrey.museum",
- "svizzera.museum",
- "sweden.museum",
- "sydney.museum",
- "tank.museum",
- "tcm.museum",
- "technology.museum",
- "telekommunikation.museum",
- "television.museum",
- "texas.museum",
- "textile.museum",
- "theater.museum",
- "time.museum",
- "timekeeping.museum",
- "topology.museum",
- "torino.museum",
- "touch.museum",
- "town.museum",
- "transport.museum",
- "tree.museum",
- "trolley.museum",
- "trust.museum",
- "trustee.museum",
- "uhren.museum",
- "ulm.museum",
- "undersea.museum",
- "university.museum",
- "usa.museum",
- "usantiques.museum",
- "usarts.museum",
- "uscountryestate.museum",
- "usculture.museum",
- "usdecorativearts.museum",
- "usgarden.museum",
- "ushistory.museum",
- "ushuaia.museum",
- "uslivinghistory.museum",
- "utah.museum",
- "uvic.museum",
- "valley.museum",
- "vantaa.museum",
- "versailles.museum",
- "viking.museum",
- "village.museum",
- "virginia.museum",
- "virtual.museum",
- "virtuel.museum",
- "vlaanderen.museum",
- "volkenkunde.museum",
- "wales.museum",
- "wallonie.museum",
- "war.museum",
- "washingtondc.museum",
- "watchandclock.museum",
- "watch-and-clock.museum",
- "western.museum",
- "westfalen.museum",
- "whaling.museum",
- "wildlife.museum",
- "williamsburg.museum",
- "windmill.museum",
- "workshop.museum",
- "york.museum",
- "yorkshire.museum",
- "yosemite.museum",
- "youth.museum",
- "zoological.museum",
- "zoology.museum",
- "xn--9dbhblg6di.museum",
- "xn--h1aegh.museum",
"mv",
"aero.mv",
"biz.mv",
@@ -5133,52 +4604,60 @@ var rules = [...]string{
"turystyka.pl",
"gov.pl",
"ap.gov.pl",
+ "griw.gov.pl",
"ic.gov.pl",
"is.gov.pl",
- "us.gov.pl",
"kmpsp.gov.pl",
+ "konsulat.gov.pl",
"kppsp.gov.pl",
- "kwpsp.gov.pl",
- "psp.gov.pl",
- "wskr.gov.pl",
"kwp.gov.pl",
+ "kwpsp.gov.pl",
+ "mup.gov.pl",
"mw.gov.pl",
- "ug.gov.pl",
- "um.gov.pl",
- "umig.gov.pl",
- "ugim.gov.pl",
- "upow.gov.pl",
- "uw.gov.pl",
- "starostwo.gov.pl",
+ "oia.gov.pl",
+ "oirm.gov.pl",
+ "oke.gov.pl",
+ "oow.gov.pl",
+ "oschr.gov.pl",
+ "oum.gov.pl",
"pa.gov.pl",
+ "pinb.gov.pl",
+ "piw.gov.pl",
"po.gov.pl",
+ "pr.gov.pl",
+ "psp.gov.pl",
"psse.gov.pl",
"pup.gov.pl",
"rzgw.gov.pl",
"sa.gov.pl",
+ "sdn.gov.pl",
+ "sko.gov.pl",
"so.gov.pl",
"sr.gov.pl",
- "wsa.gov.pl",
- "sko.gov.pl",
+ "starostwo.gov.pl",
+ "ug.gov.pl",
+ "ugim.gov.pl",
+ "um.gov.pl",
+ "umig.gov.pl",
+ "upow.gov.pl",
+ "uppo.gov.pl",
+ "us.gov.pl",
+ "uw.gov.pl",
"uzs.gov.pl",
+ "wif.gov.pl",
"wiih.gov.pl",
"winb.gov.pl",
- "pinb.gov.pl",
"wios.gov.pl",
"witd.gov.pl",
- "wzmiuw.gov.pl",
- "piw.gov.pl",
"wiw.gov.pl",
- "griw.gov.pl",
- "wif.gov.pl",
- "oum.gov.pl",
- "sdn.gov.pl",
- "zp.gov.pl",
- "uppo.gov.pl",
- "mup.gov.pl",
+ "wkz.gov.pl",
+ "wsa.gov.pl",
+ "wskr.gov.pl",
+ "wsse.gov.pl",
"wuoz.gov.pl",
- "konsulat.gov.pl",
- "oirm.gov.pl",
+ "wzmiuw.gov.pl",
+ "zp.gov.pl",
+ "zpisdn.gov.pl",
"augustow.pl",
"babia-gora.pl",
"bedzin.pl",
@@ -5722,6 +5201,7 @@ var rules = [...]string{
"kirovograd.ua",
"km.ua",
"kr.ua",
+ "kropyvnytskyi.ua",
"krym.ua",
"ks.ua",
"kv.ua",
@@ -6063,18 +5543,84 @@ var rules = [...]string{
"net.vi",
"org.vi",
"vn",
+ "ac.vn",
+ "ai.vn",
+ "biz.vn",
"com.vn",
- "net.vn",
- "org.vn",
"edu.vn",
"gov.vn",
- "int.vn",
- "ac.vn",
- "biz.vn",
+ "health.vn",
+ "id.vn",
"info.vn",
+ "int.vn",
+ "io.vn",
"name.vn",
+ "net.vn",
+ "org.vn",
"pro.vn",
- "health.vn",
+ "angiang.vn",
+ "bacgiang.vn",
+ "backan.vn",
+ "baclieu.vn",
+ "bacninh.vn",
+ "baria-vungtau.vn",
+ "bentre.vn",
+ "binhdinh.vn",
+ "binhduong.vn",
+ "binhphuoc.vn",
+ "binhthuan.vn",
+ "camau.vn",
+ "cantho.vn",
+ "caobang.vn",
+ "daklak.vn",
+ "daknong.vn",
+ "danang.vn",
+ "dienbien.vn",
+ "dongnai.vn",
+ "dongthap.vn",
+ "gialai.vn",
+ "hagiang.vn",
+ "haiduong.vn",
+ "haiphong.vn",
+ "hanam.vn",
+ "hanoi.vn",
+ "hatinh.vn",
+ "haugiang.vn",
+ "hoabinh.vn",
+ "hungyen.vn",
+ "khanhhoa.vn",
+ "kiengiang.vn",
+ "kontum.vn",
+ "laichau.vn",
+ "lamdong.vn",
+ "langson.vn",
+ "laocai.vn",
+ "longan.vn",
+ "namdinh.vn",
+ "nghean.vn",
+ "ninhbinh.vn",
+ "ninhthuan.vn",
+ "phutho.vn",
+ "phuyen.vn",
+ "quangbinh.vn",
+ "quangnam.vn",
+ "quangngai.vn",
+ "quangninh.vn",
+ "quangtri.vn",
+ "soctrang.vn",
+ "sonla.vn",
+ "tayninh.vn",
+ "thaibinh.vn",
+ "thainguyen.vn",
+ "thanhhoa.vn",
+ "thanhphohochiminh.vn",
+ "thuathienhue.vn",
+ "tiengiang.vn",
+ "travinh.vn",
+ "tuyenquang.vn",
+ "vinhlong.vn",
+ "vinhphuc.vn",
+ "yenbai.vn",
"vu",
"com.vu",
"edu.vu",
@@ -6221,7 +5767,6 @@ var rules = [...]string{
"org.zw",
"aaa",
"aarp",
- "abarth",
"abb",
"abbott",
"abbvie",
@@ -6235,7 +5780,6 @@ var rules = [...]string{
"accountants",
"aco",
"actor",
- "adac",
"ads",
"adult",
"aeg",
@@ -6249,7 +5793,6 @@ var rules = [...]string{
"airforce",
"airtel",
"akdn",
- "alfaromeo",
"alibaba",
"alipay",
"allfinanz",
@@ -6445,7 +5988,6 @@ var rules = [...]string{
"contact",
"contractors",
"cooking",
- "cookingchannel",
"cool",
"corsica",
"country",
@@ -6554,7 +6096,6 @@ var rules = [...]string{
"feedback",
"ferrari",
"ferrero",
- "fiat",
"fidelity",
"fido",
"film",
@@ -6576,7 +6117,6 @@ var rules = [...]string{
"fly",
"foo",
"food",
- "foodnetwork",
"football",
"ford",
"forex",
@@ -6661,7 +6201,6 @@ var rules = [...]string{
"helsinki",
"here",
"hermes",
- "hgtv",
"hiphop",
"hisamitsu",
"hitachi",
@@ -6680,7 +6219,6 @@ var rules = [...]string{
"host",
"hosting",
"hot",
- "hoteles",
"hotels",
"hotmail",
"house",
@@ -6761,7 +6299,6 @@ var rules = [...]string{
"lamborghini",
"lamer",
"lancaster",
- "lancia",
"land",
"landrover",
"lanxess",
@@ -6789,7 +6326,6 @@ var rules = [...]string{
"limited",
"limo",
"lincoln",
- "linde",
"link",
"lipsy",
"live",
@@ -6800,7 +6336,6 @@ var rules = [...]string{
"loans",
"locker",
"locus",
- "loft",
"lol",
"london",
"lotte",
@@ -6813,7 +6348,6 @@ var rules = [...]string{
"lundbeck",
"luxe",
"luxury",
- "macys",
"madrid",
"maif",
"maison",
@@ -6827,7 +6361,6 @@ var rules = [...]string{
"markets",
"marriott",
"marshalls",
- "maserati",
"mattel",
"mba",
"mckinsey",
@@ -6868,7 +6401,6 @@ var rules = [...]string{
"mtn",
"mtr",
"music",
- "mutual",
"nab",
"nagoya",
"natura",
@@ -6933,7 +6465,6 @@ var rules = [...]string{
"partners",
"parts",
"party",
- "passagens",
"pay",
"pccw",
"pet",
@@ -7063,7 +6594,6 @@ var rules = [...]string{
"select",
"sener",
"services",
- "ses",
"seven",
"sew",
"sex",
@@ -7157,7 +6687,6 @@ var rules = [...]string{
"tiaa",
"tickets",
"tienda",
- "tiffany",
"tips",
"tires",
"tirol",
@@ -7180,7 +6709,6 @@ var rules = [...]string{
"trading",
"training",
"travel",
- "travelchannel",
"travelers",
"travelersinsurance",
"trust",
@@ -7225,7 +6753,6 @@ var rules = [...]string{
"voting",
"voto",
"voyage",
- "vuelos",
"wales",
"walmart",
"walter",
@@ -7316,7 +6843,6 @@ var rules = [...]string{
"xn--io0a7i",
"xn--j1aef",
"xn--jlq480n2rg",
- "xn--jlq61u9w7b",
"xn--jvr189m",
"xn--kcrx77d1x4a",
"xn--kput3i",
@@ -7379,17 +6905,35 @@ var rules = [...]string{
"graphox.us",
"*.devcdnaccesso.com",
"*.on-acorn.io",
+ "activetrail.biz",
"adobeaemcloud.com",
"*.dev.adobeaemcloud.com",
"hlx.live",
"adobeaemcloud.net",
"hlx.page",
"hlx3.page",
+ "adobeio-static.net",
+ "adobeioruntime.net",
"beep.pl",
"airkitapps.com",
"airkitapps-au.com",
"airkitapps.eu",
"aivencloud.com",
+ "akadns.net",
+ "akamai.net",
+ "akamai-staging.net",
+ "akamaiedge.net",
+ "akamaiedge-staging.net",
+ "akamaihd.net",
+ "akamaihd-staging.net",
+ "akamaiorigin.net",
+ "akamaiorigin-staging.net",
+ "akamaized.net",
+ "akamaized-staging.net",
+ "edgekey.net",
+ "edgekey-staging.net",
+ "edgesuite.net",
+ "edgesuite-staging.net",
"barsy.ca",
"*.compute.estate",
"*.alces.network",
@@ -7456,46 +7000,72 @@ var rules = [...]string{
"s3.dualstack.us-east-2.amazonaws.com",
"s3.us-east-2.amazonaws.com",
"s3-website.us-east-2.amazonaws.com",
+ "analytics-gateway.ap-northeast-1.amazonaws.com",
+ "analytics-gateway.eu-west-1.amazonaws.com",
+ "analytics-gateway.us-east-1.amazonaws.com",
+ "analytics-gateway.us-east-2.amazonaws.com",
+ "analytics-gateway.us-west-2.amazonaws.com",
+ "webview-assets.aws-cloud9.af-south-1.amazonaws.com",
"vfs.cloud9.af-south-1.amazonaws.com",
"webview-assets.cloud9.af-south-1.amazonaws.com",
+ "webview-assets.aws-cloud9.ap-east-1.amazonaws.com",
"vfs.cloud9.ap-east-1.amazonaws.com",
"webview-assets.cloud9.ap-east-1.amazonaws.com",
+ "webview-assets.aws-cloud9.ap-northeast-1.amazonaws.com",
"vfs.cloud9.ap-northeast-1.amazonaws.com",
"webview-assets.cloud9.ap-northeast-1.amazonaws.com",
+ "webview-assets.aws-cloud9.ap-northeast-2.amazonaws.com",
"vfs.cloud9.ap-northeast-2.amazonaws.com",
"webview-assets.cloud9.ap-northeast-2.amazonaws.com",
+ "webview-assets.aws-cloud9.ap-northeast-3.amazonaws.com",
"vfs.cloud9.ap-northeast-3.amazonaws.com",
"webview-assets.cloud9.ap-northeast-3.amazonaws.com",
+ "webview-assets.aws-cloud9.ap-south-1.amazonaws.com",
"vfs.cloud9.ap-south-1.amazonaws.com",
"webview-assets.cloud9.ap-south-1.amazonaws.com",
+ "webview-assets.aws-cloud9.ap-southeast-1.amazonaws.com",
"vfs.cloud9.ap-southeast-1.amazonaws.com",
"webview-assets.cloud9.ap-southeast-1.amazonaws.com",
+ "webview-assets.aws-cloud9.ap-southeast-2.amazonaws.com",
"vfs.cloud9.ap-southeast-2.amazonaws.com",
"webview-assets.cloud9.ap-southeast-2.amazonaws.com",
+ "webview-assets.aws-cloud9.ca-central-1.amazonaws.com",
"vfs.cloud9.ca-central-1.amazonaws.com",
"webview-assets.cloud9.ca-central-1.amazonaws.com",
+ "webview-assets.aws-cloud9.eu-central-1.amazonaws.com",
"vfs.cloud9.eu-central-1.amazonaws.com",
"webview-assets.cloud9.eu-central-1.amazonaws.com",
+ "webview-assets.aws-cloud9.eu-north-1.amazonaws.com",
"vfs.cloud9.eu-north-1.amazonaws.com",
"webview-assets.cloud9.eu-north-1.amazonaws.com",
+ "webview-assets.aws-cloud9.eu-south-1.amazonaws.com",
"vfs.cloud9.eu-south-1.amazonaws.com",
"webview-assets.cloud9.eu-south-1.amazonaws.com",
+ "webview-assets.aws-cloud9.eu-west-1.amazonaws.com",
"vfs.cloud9.eu-west-1.amazonaws.com",
"webview-assets.cloud9.eu-west-1.amazonaws.com",
+ "webview-assets.aws-cloud9.eu-west-2.amazonaws.com",
"vfs.cloud9.eu-west-2.amazonaws.com",
"webview-assets.cloud9.eu-west-2.amazonaws.com",
+ "webview-assets.aws-cloud9.eu-west-3.amazonaws.com",
"vfs.cloud9.eu-west-3.amazonaws.com",
"webview-assets.cloud9.eu-west-3.amazonaws.com",
+ "webview-assets.aws-cloud9.me-south-1.amazonaws.com",
"vfs.cloud9.me-south-1.amazonaws.com",
"webview-assets.cloud9.me-south-1.amazonaws.com",
+ "webview-assets.aws-cloud9.sa-east-1.amazonaws.com",
"vfs.cloud9.sa-east-1.amazonaws.com",
"webview-assets.cloud9.sa-east-1.amazonaws.com",
+ "webview-assets.aws-cloud9.us-east-1.amazonaws.com",
"vfs.cloud9.us-east-1.amazonaws.com",
"webview-assets.cloud9.us-east-1.amazonaws.com",
+ "webview-assets.aws-cloud9.us-east-2.amazonaws.com",
"vfs.cloud9.us-east-2.amazonaws.com",
"webview-assets.cloud9.us-east-2.amazonaws.com",
+ "webview-assets.aws-cloud9.us-west-1.amazonaws.com",
"vfs.cloud9.us-west-1.amazonaws.com",
"webview-assets.cloud9.us-west-1.amazonaws.com",
+ "webview-assets.aws-cloud9.us-west-2.amazonaws.com",
"vfs.cloud9.us-west-2.amazonaws.com",
"webview-assets.cloud9.us-west-2.amazonaws.com",
"cn-north-1.eb.amazonaws.com.cn",
@@ -7542,6 +7112,7 @@ var rules = [...]string{
"myasustor.com",
"cdn.prod.atlassian-dev.net",
"translated.page",
+ "autocode.dev",
"myfritz.net",
"onavstack.net",
"*.awdev.ca",
@@ -7588,6 +7159,8 @@ var rules = [...]string{
"vm.bytemark.co.uk",
"cafjs.com",
"mycd.eu",
+ "canva-apps.cn",
+ "canva-apps.com",
"drr.ac",
"uwu.ai",
"carrd.co",
@@ -7653,8 +7226,11 @@ var rules = [...]string{
"cloudcontrolled.com",
"cloudcontrolapp.com",
"*.cloudera.site",
- "pages.dev",
+ "cf-ipfs.com",
+ "cloudflare-ipfs.com",
"trycloudflare.com",
+ "pages.dev",
+ "r2.dev",
"workers.dev",
"wnext.app",
"co.ca",
@@ -8227,6 +7803,7 @@ var rules = [...]string{
"channelsdvr.net",
"u.channelsdvr.net",
"edgecompute.app",
+ "fastly-edge.com",
"fastly-terrarium.com",
"fastlylb.net",
"map.fastlylb.net",
@@ -8566,6 +8143,7 @@ var rules = [...]string{
"ngo.ng",
"edu.scot",
"sch.so",
+ "ie.ua",
"hostyhosting.io",
"xn--hkkinen-5wa.fi",
"*.moonscale.io",
@@ -8633,7 +8211,6 @@ var rules = [...]string{
"iobb.net",
"mel.cloudlets.com.au",
"cloud.interhostsolutions.be",
- "users.scale.virtualcloud.com.br",
"mycloud.by",
"alp1.ae.flow.ch",
"appengine.flow.ch",
@@ -8657,9 +8234,7 @@ var rules = [...]string{
"de.trendhosting.cloud",
"jele.club",
"amscompute.com",
- "clicketcloud.com",
"dopaas.com",
- "hidora.com",
"paas.hosted-by-previder.com",
"rag-cloud.hosteur.com",
"rag-cloud-ch.hosteur.com",
@@ -8834,6 +8409,7 @@ var rules = [...]string{
"azurestaticapps.net",
"1.azurestaticapps.net",
"2.azurestaticapps.net",
+ "3.azurestaticapps.net",
"centralus.azurestaticapps.net",
"eastasia.azurestaticapps.net",
"eastus2.azurestaticapps.net",
@@ -8864,7 +8440,19 @@ var rules = [...]string{
"cloud.nospamproxy.com",
"netlify.app",
"4u.com",
+ "ngrok.app",
+ "ngrok-free.app",
+ "ngrok.dev",
+ "ngrok-free.dev",
"ngrok.io",
+ "ap.ngrok.io",
+ "au.ngrok.io",
+ "eu.ngrok.io",
+ "in.ngrok.io",
+ "jp.ngrok.io",
+ "sa.ngrok.io",
+ "us.ngrok.io",
+ "ngrok.pizza",
"nh-serv.co.uk",
"nfshost.com",
"*.developer.app",
@@ -9084,6 +8672,7 @@ var rules = [...]string{
"eu.pythonanywhere.com",
"qoto.io",
"qualifioapp.com",
+ "ladesk.com",
"qbuser.com",
"cloudsite.builders",
"instances.spawn.cc",
@@ -9132,6 +8721,53 @@ var rules = [...]string{
"xn--h1aliz.xn--p1acf",
"xn--90a1af.xn--p1acf",
"xn--41a.xn--p1acf",
+ "180r.com",
+ "dojin.com",
+ "sakuratan.com",
+ "sakuraweb.com",
+ "x0.com",
+ "2-d.jp",
+ "bona.jp",
+ "crap.jp",
+ "daynight.jp",
+ "eek.jp",
+ "flop.jp",
+ "halfmoon.jp",
+ "jeez.jp",
+ "matrix.jp",
+ "mimoza.jp",
+ "ivory.ne.jp",
+ "mail-box.ne.jp",
+ "mints.ne.jp",
+ "mokuren.ne.jp",
+ "opal.ne.jp",
+ "sakura.ne.jp",
+ "sumomo.ne.jp",
+ "topaz.ne.jp",
+ "netgamers.jp",
+ "nyanta.jp",
+ "o0o0.jp",
+ "rdy.jp",
+ "rgr.jp",
+ "rulez.jp",
+ "s3.isk01.sakurastorage.jp",
+ "s3.isk02.sakurastorage.jp",
+ "saloon.jp",
+ "sblo.jp",
+ "skr.jp",
+ "tank.jp",
+ "uh-oh.jp",
+ "undo.jp",
+ "rs.webaccel.jp",
+ "user.webaccel.jp",
+ "websozai.jp",
+ "xii.jp",
+ "squares.net",
+ "jpn.org",
+ "kirara.st",
+ "x0.to",
+ "from.tv",
+ "sakura.tv",
"*.builder.code.com",
"*.dev-builder.code.com",
"*.stg-builder.code.com",
@@ -9204,6 +8840,9 @@ var rules = [...]string{
"beta.bounty-full.com",
"small-web.org",
"vp4.me",
+ "snowflake.app",
+ "privatelink.snowflake.app",
+ "streamlit.app",
"streamlitapp.com",
"try-snowplow.com",
"srht.site",
@@ -9243,6 +8882,7 @@ var rules = [...]string{
"myspreadshop.se",
"myspreadshop.co.uk",
"api.stdlib.com",
+ "storipress.app",
"storj.farm",
"utwente.io",
"soc.srcf.net",
@@ -9272,6 +8912,8 @@ var rules = [...]string{
"vpnplus.to",
"direct.quickconnect.to",
"tabitorder.co.il",
+ "mytabit.co.il",
+ "mytabit.com",
"taifun-dns.de",
"beta.tailscale.net",
"ts.net",
@@ -9350,6 +8992,7 @@ var rules = [...]string{
"hk.org",
"ltd.hk",
"inc.hk",
+ "it.com",
"name.pm",
"sch.tf",
"biz.wf",
@@ -9472,7 +9115,6 @@ var rules = [...]string{
var nodeLabels = [...]string{
"aaa",
"aarp",
- "abarth",
"abb",
"abbott",
"abbvie",
@@ -9488,7 +9130,6 @@ var nodeLabels = [...]string{
"aco",
"actor",
"ad",
- "adac",
"ads",
"adult",
"ae",
@@ -9508,7 +9149,6 @@ var nodeLabels = [...]string{
"airtel",
"akdn",
"al",
- "alfaromeo",
"alibaba",
"alipay",
"allfinanz",
@@ -9750,7 +9390,6 @@ var nodeLabels = [...]string{
"contact",
"contractors",
"cooking",
- "cookingchannel",
"cool",
"coop",
"corsica",
@@ -9882,7 +9521,6 @@ var nodeLabels = [...]string{
"ferrari",
"ferrero",
"fi",
- "fiat",
"fidelity",
"fido",
"film",
@@ -9908,7 +9546,6 @@ var nodeLabels = [...]string{
"fo",
"foo",
"food",
- "foodnetwork",
"football",
"ford",
"forex",
@@ -10014,7 +9651,6 @@ var nodeLabels = [...]string{
"helsinki",
"here",
"hermes",
- "hgtv",
"hiphop",
"hisamitsu",
"hitachi",
@@ -10036,7 +9672,6 @@ var nodeLabels = [...]string{
"host",
"hosting",
"hot",
- "hoteles",
"hotels",
"hotmail",
"house",
@@ -10149,7 +9784,6 @@ var nodeLabels = [...]string{
"lamborghini",
"lamer",
"lancaster",
- "lancia",
"land",
"landrover",
"lanxess",
@@ -10180,7 +9814,6 @@ var nodeLabels = [...]string{
"limited",
"limo",
"lincoln",
- "linde",
"link",
"lipsy",
"live",
@@ -10192,7 +9825,6 @@ var nodeLabels = [...]string{
"loans",
"locker",
"locus",
- "loft",
"lol",
"london",
"lotte",
@@ -10212,7 +9844,6 @@ var nodeLabels = [...]string{
"lv",
"ly",
"ma",
- "macys",
"madrid",
"maif",
"maison",
@@ -10226,7 +9857,6 @@ var nodeLabels = [...]string{
"markets",
"marriott",
"marshalls",
- "maserati",
"mattel",
"mba",
"mc",
@@ -10286,7 +9916,6 @@ var nodeLabels = [...]string{
"mu",
"museum",
"music",
- "mutual",
"mv",
"mw",
"mx",
@@ -10374,7 +10003,6 @@ var nodeLabels = [...]string{
"partners",
"parts",
"party",
- "passagens",
"pay",
"pccw",
"pe",
@@ -10530,7 +10158,6 @@ var nodeLabels = [...]string{
"select",
"sener",
"services",
- "ses",
"seven",
"sew",
"sex",
@@ -10647,7 +10274,6 @@ var nodeLabels = [...]string{
"tiaa",
"tickets",
"tienda",
- "tiffany",
"tips",
"tires",
"tirol",
@@ -10677,7 +10303,6 @@ var nodeLabels = [...]string{
"trading",
"training",
"travel",
- "travelchannel",
"travelers",
"travelersinsurance",
"trust",
@@ -10739,7 +10364,6 @@ var nodeLabels = [...]string{
"voto",
"voyage",
"vu",
- "vuelos",
"wales",
"walmart",
"walter",
@@ -10856,7 +10480,6 @@ var nodeLabels = [...]string{
"xn--j1amh",
"xn--j6w193g",
"xn--jlq480n2rg",
- "xn--jlq61u9w7b",
"xn--jvr189m",
"xn--kcrx77d1x4a",
"xn--kprw13d",
@@ -11119,18 +10742,24 @@ var nodeLabels = [...]string{
"loginline",
"messerli",
"netlify",
+ "ngrok",
+ "ngrok-free",
"noop",
"northflank",
"ondigitalocean",
"onflashdrive",
"platform0",
"run",
+ "snowflake",
+ "storipress",
+ "streamlit",
"telebit",
"typedream",
"vercel",
"web",
"wnext",
"a",
+ "privatelink",
"bet",
"com",
"coop",
@@ -11316,6 +10945,7 @@ var nodeLabels = [...]string{
"edu",
"or",
"org",
+ "activetrail",
"cloudns",
"dscloud",
"dyndns",
@@ -11330,10 +10960,27 @@ var nodeLabels = [...]string{
"orx",
"selfip",
"webhop",
- "asso",
- "barreau",
+ "africa",
+ "agro",
+ "architectes",
+ "assur",
+ "avocats",
"blogspot",
- "gouv",
+ "co",
+ "com",
+ "eco",
+ "econo",
+ "edu",
+ "info",
+ "loisirs",
+ "money",
+ "net",
+ "org",
+ "ote",
+ "restaurant",
+ "resto",
+ "tourism",
+ "univ",
"com",
"edu",
"gov",
@@ -11529,9 +11176,6 @@ var nodeLabels = [...]string{
"zlg",
"blogspot",
"simplesite",
- "virtualcloud",
- "scale",
- "users",
"ac",
"al",
"am",
@@ -11772,6 +11416,7 @@ var nodeLabels = [...]string{
"ac",
"ah",
"bj",
+ "canva-apps",
"com",
"cq",
"edu",
@@ -11853,6 +11498,7 @@ var nodeLabels = [...]string{
"owo",
"001www",
"0emm",
+ "180r",
"1kapp",
"3utilities",
"4u",
@@ -11888,11 +11534,13 @@ var nodeLabels = [...]string{
"br",
"builtwithdark",
"cafjs",
+ "canva-apps",
"cechire",
+ "cf-ipfs",
"ciscofreak",
- "clicketcloud",
"cloudcontrolapp",
"cloudcontrolled",
+ "cloudflare-ipfs",
"cn",
"co",
"code",
@@ -11919,6 +11567,7 @@ var nodeLabels = [...]string{
"dnsdojo",
"dnsiskinky",
"doesntexist",
+ "dojin",
"dontexist",
"doomdns",
"dopaas",
@@ -11951,6 +11600,7 @@ var nodeLabels = [...]string{
"eu",
"evennode",
"familyds",
+ "fastly-edge",
"fastly-terrarium",
"fastvps-server",
"fbsbx",
@@ -12024,7 +11674,6 @@ var nodeLabels = [...]string{
"health-carereform",
"herokuapp",
"herokussl",
- "hidora",
"hk",
"hobby-site",
"homelinux",
@@ -12098,6 +11747,7 @@ var nodeLabels = [...]string{
"isa-geek",
"isa-hockeynut",
"issmarterthanyou",
+ "it",
"jdevcloud",
"jelastic",
"joyent",
@@ -12107,6 +11757,7 @@ var nodeLabels = [...]string{
"kozow",
"kr",
"ktistory",
+ "ladesk",
"likes-pie",
"likescandy",
"linode",
@@ -12133,6 +11784,7 @@ var nodeLabels = [...]string{
"myshopblocks",
"myshopify",
"myspreadshop",
+ "mytabit",
"mythic-beasts",
"mytuleap",
"myvnc",
@@ -12179,6 +11831,8 @@ var nodeLabels = [...]string{
"rhcloud",
"ru",
"sa",
+ "sakuratan",
+ "sakuraweb",
"saves-the-whales",
"scrysec",
"securitytactics",
@@ -12241,6 +11895,7 @@ var nodeLabels = [...]string{
"wphostedmail",
"wpmucdn",
"writesthisblog",
+ "x0",
"xnbay",
"yolasite",
"za",
@@ -12295,107 +11950,154 @@ var nodeLabels = [...]string{
"us-east-2",
"us-west-1",
"us-west-2",
+ "aws-cloud9",
"cloud9",
+ "webview-assets",
"vfs",
"webview-assets",
+ "aws-cloud9",
"cloud9",
+ "webview-assets",
"vfs",
"webview-assets",
+ "analytics-gateway",
+ "aws-cloud9",
"cloud9",
"dualstack",
+ "webview-assets",
"vfs",
"webview-assets",
"s3",
+ "aws-cloud9",
"cloud9",
"dualstack",
"s3",
"s3-website",
+ "webview-assets",
"vfs",
"webview-assets",
"s3",
+ "aws-cloud9",
"cloud9",
+ "webview-assets",
"vfs",
"webview-assets",
+ "aws-cloud9",
"cloud9",
"dualstack",
"s3",
"s3-website",
+ "webview-assets",
"vfs",
"webview-assets",
"s3",
+ "aws-cloud9",
"cloud9",
"dualstack",
+ "webview-assets",
"vfs",
"webview-assets",
"s3",
+ "aws-cloud9",
"cloud9",
"dualstack",
+ "webview-assets",
"vfs",
"webview-assets",
"s3",
+ "aws-cloud9",
"cloud9",
"dualstack",
"s3",
"s3-website",
+ "webview-assets",
"vfs",
"webview-assets",
"s3",
+ "aws-cloud9",
"cloud9",
"dualstack",
"s3",
"s3-website",
+ "webview-assets",
"vfs",
"webview-assets",
"s3",
+ "aws-cloud9",
"cloud9",
+ "webview-assets",
"vfs",
"webview-assets",
+ "aws-cloud9",
"cloud9",
+ "webview-assets",
"vfs",
"webview-assets",
+ "analytics-gateway",
+ "aws-cloud9",
"cloud9",
"dualstack",
+ "webview-assets",
"vfs",
"webview-assets",
"s3",
+ "aws-cloud9",
"cloud9",
"dualstack",
"s3",
"s3-website",
+ "webview-assets",
"vfs",
"webview-assets",
"s3",
+ "aws-cloud9",
"cloud9",
"dualstack",
"s3",
"s3-website",
+ "webview-assets",
"vfs",
"webview-assets",
"s3",
+ "aws-cloud9",
"cloud9",
+ "webview-assets",
"vfs",
"webview-assets",
+ "aws-cloud9",
"cloud9",
"dualstack",
+ "webview-assets",
"vfs",
"webview-assets",
"s3",
+ "analytics-gateway",
+ "aws-cloud9",
"cloud9",
"dualstack",
+ "webview-assets",
"vfs",
"webview-assets",
"s3",
+ "analytics-gateway",
+ "aws-cloud9",
"cloud9",
"dualstack",
"s3",
"s3-website",
+ "webview-assets",
"vfs",
"webview-assets",
"s3",
+ "aws-cloud9",
"cloud9",
+ "webview-assets",
"vfs",
"webview-assets",
+ "analytics-gateway",
+ "aws-cloud9",
"cloud9",
+ "webview-assets",
"vfs",
"webview-assets",
"r",
@@ -12610,6 +12312,7 @@ var nodeLabels = [...]string{
"pages",
"customer",
"bss",
+ "autocode",
"curv",
"deno",
"deno-staging",
@@ -12623,8 +12326,11 @@ var nodeLabels = [...]string{
"localcert",
"loginline",
"mediatech",
+ "ngrok",
+ "ngrok-free",
"pages",
"platter-app",
+ "r2",
"shiftcrypto",
"stg",
"stgstage",
@@ -13016,6 +12722,7 @@ var nodeLabels = [...]string{
"net",
"org",
"blogspot",
+ "mytabit",
"ravpage",
"tabitorder",
"ac",
@@ -13176,6 +12883,13 @@ var nodeLabels = [...]string{
"dyndns",
"id",
"apps",
+ "ap",
+ "au",
+ "eu",
+ "in",
+ "jp",
+ "sa",
+ "us",
"stage",
"mock",
"sys",
@@ -13649,6 +13363,7 @@ var nodeLabels = [...]string{
"net",
"org",
"sch",
+ "2-d",
"ac",
"ad",
"aichi",
@@ -13662,6 +13377,7 @@ var nodeLabels = [...]string{
"bitter",
"blogspot",
"blush",
+ "bona",
"boo",
"boy",
"boyfriend",
@@ -13682,18 +13398,22 @@ var nodeLabels = [...]string{
"cocotte",
"coolblog",
"cranky",
+ "crap",
"cutegirl",
"daa",
+ "daynight",
"deca",
"deci",
"digick",
"ed",
+ "eek",
"egoism",
"ehime",
"fakefur",
"fashionstore",
"fem",
"flier",
+ "flop",
"floppy",
"fool",
"frenchkiss",
@@ -13710,6 +13430,7 @@ var nodeLabels = [...]string{
"greater",
"gunma",
"hacca",
+ "halfmoon",
"handcrafted",
"heavy",
"her",
@@ -13725,6 +13446,7 @@ var nodeLabels = [...]string{
"ishikawa",
"itigo",
"iwate",
+ "jeez",
"jellybean",
"kagawa",
"kagoshima",
@@ -13748,7 +13470,9 @@ var nodeLabels = [...]string{
"lovepop",
"lovesick",
"main",
+ "matrix",
"mie",
+ "mimoza",
"miyagi",
"miyazaki",
"mods",
@@ -13761,10 +13485,13 @@ var nodeLabels = [...]string{
"namaste",
"nara",
"ne",
+ "netgamers",
"niigata",
"nikita",
"nobushi",
"noor",
+ "nyanta",
+ "o0o0",
"oita",
"okayama",
"okinawa",
@@ -13785,22 +13512,30 @@ var nodeLabels = [...]string{
"pussycat",
"pya",
"raindrop",
+ "rdy",
"readymade",
+ "rgr",
+ "rulez",
"sadist",
"saga",
"saitama",
+ "sakurastorage",
+ "saloon",
"sapporo",
+ "sblo",
"schoolbus",
"secret",
"sendai",
"shiga",
"shimane",
"shizuoka",
+ "skr",
"staba",
"stripper",
"sub",
"sunnyday",
"supersale",
+ "tank",
"theshop",
"thick",
"tochigi",
@@ -13809,7 +13544,9 @@ var nodeLabels = [...]string{
"tonkotsu",
"tottori",
"toyama",
+ "uh-oh",
"under",
+ "undo",
"upper",
"usercontent",
"velvet",
@@ -13818,8 +13555,11 @@ var nodeLabels = [...]string{
"vivian",
"wakayama",
"watson",
+ "webaccel",
"weblike",
+ "websozai",
"whitesnow",
+ "xii",
"xn--0trq7p7nn",
"xn--1ctwo",
"xn--1lqs03n",
@@ -14954,6 +14694,14 @@ var nodeLabels = [...]string{
"yoshino",
"aseinet",
"gehirn",
+ "ivory",
+ "mail-box",
+ "mints",
+ "mokuren",
+ "opal",
+ "sakura",
+ "sumomo",
+ "topaz",
"user",
"aga",
"agano",
@@ -15221,6 +14969,10 @@ var nodeLabels = [...]string{
"yoshida",
"yoshikawa",
"yoshimi",
+ "isk01",
+ "isk02",
+ "s3",
+ "s3",
"city",
"city",
"aisho",
@@ -15476,6 +15228,8 @@ var nodeLabels = [...]string{
"wakayama",
"yuasa",
"yura",
+ "rs",
+ "user",
"asahi",
"funagata",
"higashine",
@@ -15865,552 +15619,6 @@ var nodeLabels = [...]string{
"net",
"or",
"org",
- "academy",
- "agriculture",
- "air",
- "airguard",
- "alabama",
- "alaska",
- "amber",
- "ambulance",
- "american",
- "americana",
- "americanantiques",
- "americanart",
- "amsterdam",
- "and",
- "annefrank",
- "anthro",
- "anthropology",
- "antiques",
- "aquarium",
- "arboretum",
- "archaeological",
- "archaeology",
- "architecture",
- "art",
- "artanddesign",
- "artcenter",
- "artdeco",
- "arteducation",
- "artgallery",
- "arts",
- "artsandcrafts",
- "asmatart",
- "assassination",
- "assisi",
- "association",
- "astronomy",
- "atlanta",
- "austin",
- "australia",
- "automotive",
- "aviation",
- "axis",
- "badajoz",
- "baghdad",
- "bahn",
- "bale",
- "baltimore",
- "barcelona",
- "baseball",
- "basel",
- "baths",
- "bauern",
- "beauxarts",
- "beeldengeluid",
- "bellevue",
- "bergbau",
- "berkeley",
- "berlin",
- "bern",
- "bible",
- "bilbao",
- "bill",
- "birdart",
- "birthplace",
- "bonn",
- "boston",
- "botanical",
- "botanicalgarden",
- "botanicgarden",
- "botany",
- "brandywinevalley",
- "brasil",
- "bristol",
- "british",
- "britishcolumbia",
- "broadcast",
- "brunel",
- "brussel",
- "brussels",
- "bruxelles",
- "building",
- "burghof",
- "bus",
- "bushey",
- "cadaques",
- "california",
- "cambridge",
- "can",
- "canada",
- "capebreton",
- "carrier",
- "cartoonart",
- "casadelamoneda",
- "castle",
- "castres",
- "celtic",
- "center",
- "chattanooga",
- "cheltenham",
- "chesapeakebay",
- "chicago",
- "children",
- "childrens",
- "childrensgarden",
- "chiropractic",
- "chocolate",
- "christiansburg",
- "cincinnati",
- "cinema",
- "circus",
- "civilisation",
- "civilization",
- "civilwar",
- "clinton",
- "clock",
- "coal",
- "coastaldefence",
- "cody",
- "coldwar",
- "collection",
- "colonialwilliamsburg",
- "coloradoplateau",
- "columbia",
- "columbus",
- "communication",
- "communications",
- "community",
- "computer",
- "computerhistory",
- "contemporary",
- "contemporaryart",
- "convent",
- "copenhagen",
- "corporation",
- "corvette",
- "costume",
- "countryestate",
- "county",
- "crafts",
- "cranbrook",
- "creation",
- "cultural",
- "culturalcenter",
- "culture",
- "cyber",
- "cymru",
- "dali",
- "dallas",
- "database",
- "ddr",
- "decorativearts",
- "delaware",
- "delmenhorst",
- "denmark",
- "depot",
- "design",
- "detroit",
- "dinosaur",
- "discovery",
- "dolls",
- "donostia",
- "durham",
- "eastafrica",
- "eastcoast",
- "education",
- "educational",
- "egyptian",
- "eisenbahn",
- "elburg",
- "elvendrell",
- "embroidery",
- "encyclopedic",
- "england",
- "entomology",
- "environment",
- "environmentalconservation",
- "epilepsy",
- "essex",
- "estate",
- "ethnology",
- "exeter",
- "exhibition",
- "family",
- "farm",
- "farmequipment",
- "farmers",
- "farmstead",
- "field",
- "figueres",
- "filatelia",
- "film",
- "fineart",
- "finearts",
- "finland",
- "flanders",
- "florida",
- "force",
- "fortmissoula",
- "fortworth",
- "foundation",
- "francaise",
- "frankfurt",
- "franziskaner",
- "freemasonry",
- "freiburg",
- "fribourg",
- "frog",
- "fundacio",
- "furniture",
- "gallery",
- "garden",
- "gateway",
- "geelvinck",
- "gemological",
- "geology",
- "georgia",
- "giessen",
- "glas",
- "glass",
- "gorge",
- "grandrapids",
- "graz",
- "guernsey",
- "halloffame",
- "hamburg",
- "handson",
- "harvestcelebration",
- "hawaii",
- "health",
- "heimatunduhren",
- "hellas",
- "helsinki",
- "hembygdsforbund",
- "heritage",
- "histoire",
- "historical",
- "historicalsociety",
- "historichouses",
- "historisch",
- "historisches",
- "history",
- "historyofscience",
- "horology",
- "house",
- "humanities",
- "illustration",
- "imageandsound",
- "indian",
- "indiana",
- "indianapolis",
- "indianmarket",
- "intelligence",
- "interactive",
- "iraq",
- "iron",
- "isleofman",
- "jamison",
- "jefferson",
- "jerusalem",
- "jewelry",
- "jewish",
- "jewishart",
- "jfk",
- "journalism",
- "judaica",
- "judygarland",
- "juedisches",
- "juif",
- "karate",
- "karikatur",
- "kids",
- "koebenhavn",
- "koeln",
- "kunst",
- "kunstsammlung",
- "kunstunddesign",
- "labor",
- "labour",
- "lajolla",
- "lancashire",
- "landes",
- "lans",
- "larsson",
- "lewismiller",
- "lincoln",
- "linz",
- "living",
- "livinghistory",
- "localhistory",
- "london",
- "losangeles",
- "louvre",
- "loyalist",
- "lucerne",
- "luxembourg",
- "luzern",
- "mad",
- "madrid",
- "mallorca",
- "manchester",
- "mansion",
- "mansions",
- "manx",
- "marburg",
- "maritime",
- "maritimo",
- "maryland",
- "marylhurst",
- "media",
- "medical",
- "medizinhistorisches",
- "meeres",
- "memorial",
- "mesaverde",
- "michigan",
- "midatlantic",
- "military",
- "mill",
- "miners",
- "mining",
- "minnesota",
- "missile",
- "missoula",
- "modern",
- "moma",
- "money",
- "monmouth",
- "monticello",
- "montreal",
- "moscow",
- "motorcycle",
- "muenchen",
- "muenster",
- "mulhouse",
- "muncie",
- "museet",
- "museumcenter",
- "museumvereniging",
- "music",
- "national",
- "nationalfirearms",
- "nationalheritage",
- "nativeamerican",
- "naturalhistory",
- "naturalhistorymuseum",
- "naturalsciences",
- "nature",
- "naturhistorisches",
- "natuurwetenschappen",
- "naumburg",
- "naval",
- "nebraska",
- "neues",
- "newhampshire",
- "newjersey",
- "newmexico",
- "newport",
- "newspaper",
- "newyork",
- "niepce",
- "norfolk",
- "north",
- "nrw",
- "nyc",
- "nyny",
- "oceanographic",
- "oceanographique",
- "omaha",
- "online",
- "ontario",
- "openair",
- "oregon",
- "oregontrail",
- "otago",
- "oxford",
- "pacific",
- "paderborn",
- "palace",
- "paleo",
- "palmsprings",
- "panama",
- "paris",
- "pasadena",
- "pharmacy",
- "philadelphia",
- "philadelphiaarea",
- "philately",
- "phoenix",
- "photography",
- "pilots",
- "pittsburgh",
- "planetarium",
- "plantation",
- "plants",
- "plaza",
- "portal",
- "portland",
- "portlligat",
- "posts-and-telecommunications",
- "preservation",
- "presidio",
- "press",
- "project",
- "public",
- "pubol",
- "quebec",
- "railroad",
- "railway",
- "research",
- "resistance",
- "riodejaneiro",
- "rochester",
- "rockart",
- "roma",
- "russia",
- "saintlouis",
- "salem",
- "salvadordali",
- "salzburg",
- "sandiego",
- "sanfrancisco",
- "santabarbara",
- "santacruz",
- "santafe",
- "saskatchewan",
- "satx",
- "savannahga",
- "schlesisches",
- "schoenbrunn",
- "schokoladen",
- "school",
- "schweiz",
- "science",
- "science-fiction",
- "scienceandhistory",
- "scienceandindustry",
- "sciencecenter",
- "sciencecenters",
- "sciencehistory",
- "sciences",
- "sciencesnaturelles",
- "scotland",
- "seaport",
- "settlement",
- "settlers",
- "shell",
- "sherbrooke",
- "sibenik",
- "silk",
- "ski",
- "skole",
- "society",
- "sologne",
- "soundandvision",
- "southcarolina",
- "southwest",
- "space",
- "spy",
- "square",
- "stadt",
- "stalbans",
- "starnberg",
- "state",
- "stateofdelaware",
- "station",
- "steam",
- "steiermark",
- "stjohn",
- "stockholm",
- "stpetersburg",
- "stuttgart",
- "suisse",
- "surgeonshall",
- "surrey",
- "svizzera",
- "sweden",
- "sydney",
- "tank",
- "tcm",
- "technology",
- "telekommunikation",
- "television",
- "texas",
- "textile",
- "theater",
- "time",
- "timekeeping",
- "topology",
- "torino",
- "touch",
- "town",
- "transport",
- "tree",
- "trolley",
- "trust",
- "trustee",
- "uhren",
- "ulm",
- "undersea",
- "university",
- "usa",
- "usantiques",
- "usarts",
- "uscountryestate",
- "usculture",
- "usdecorativearts",
- "usgarden",
- "ushistory",
- "ushuaia",
- "uslivinghistory",
- "utah",
- "uvic",
- "valley",
- "vantaa",
- "versailles",
- "viking",
- "village",
- "virginia",
- "virtual",
- "virtuel",
- "vlaanderen",
- "volkenkunde",
- "wales",
- "wallonie",
- "war",
- "washingtondc",
- "watch-and-clock",
- "watchandclock",
- "western",
- "westfalen",
- "whaling",
- "wildlife",
- "williamsburg",
- "windmill",
- "workshop",
- "xn--9dbhblg6di",
- "xn--comunicaes-v6a2o",
- "xn--correios-e-telecomunicaes-ghc29a",
- "xn--h1aegh",
- "xn--lns-qla",
- "york",
- "yorkshire",
- "yosemite",
- "youth",
- "zoological",
- "zoology",
"aero",
"biz",
"com",
@@ -16483,6 +15691,19 @@ var nodeLabels = [...]string{
"asso",
"nom",
"adobeaemcloud",
+ "adobeio-static",
+ "adobeioruntime",
+ "akadns",
+ "akamai",
+ "akamai-staging",
+ "akamaiedge",
+ "akamaiedge-staging",
+ "akamaihd",
+ "akamaihd-staging",
+ "akamaiorigin",
+ "akamaiorigin-staging",
+ "akamaized",
+ "akamaized-staging",
"alwaysdata",
"appudo",
"at-band-camp",
@@ -16532,6 +15753,10 @@ var nodeLabels = [...]string{
"dynv6",
"eating-organic",
"edgeapp",
+ "edgekey",
+ "edgekey-staging",
+ "edgesuite",
+ "edgesuite-staging",
"elastx",
"endofinternet",
"familyds",
@@ -16612,6 +15837,7 @@ var nodeLabels = [...]string{
"shopselect",
"siteleaf",
"square7",
+ "squares",
"srcf",
"static-access",
"supabase",
@@ -16634,6 +15860,7 @@ var nodeLabels = [...]string{
"cdn",
"1",
"2",
+ "3",
"centralus",
"eastasia",
"eastus2",
@@ -17619,6 +16846,7 @@ var nodeLabels = [...]string{
"is-very-nice",
"is-very-sweet",
"isa-geek",
+ "jpn",
"js",
"kicks-ass",
"mayfirst",
@@ -17774,6 +17002,7 @@ var nodeLabels = [...]string{
"org",
"framer",
"1337",
+ "ngrok",
"biz",
"com",
"edu",
@@ -17978,12 +17207,17 @@ var nodeLabels = [...]string{
"kwpsp",
"mup",
"mw",
+ "oia",
"oirm",
+ "oke",
+ "oow",
+ "oschr",
"oum",
"pa",
"pinb",
"piw",
"po",
+ "pr",
"psp",
"psse",
"pup",
@@ -18009,11 +17243,14 @@ var nodeLabels = [...]string{
"wios",
"witd",
"wiw",
+ "wkz",
"wsa",
"wskr",
+ "wsse",
"wuoz",
"wzmiuw",
"zp",
+ "zpisdn",
"co",
"name",
"own",
@@ -18355,6 +17592,7 @@ var nodeLabels = [...]string{
"consulado",
"edu",
"embaixada",
+ "kirara",
"mil",
"net",
"noho",
@@ -18501,6 +17739,7 @@ var nodeLabels = [...]string{
"quickconnect",
"rdv",
"vpnplus",
+ "x0",
"direct",
"prequalifyme",
"now-dns",
@@ -18549,7 +17788,9 @@ var nodeLabels = [...]string{
"travel",
"better-than",
"dyndns",
+ "from",
"on-the-web",
+ "sakura",
"worse-than",
"blogspot",
"club",
@@ -18602,6 +17843,7 @@ var nodeLabels = [...]string{
"dp",
"edu",
"gov",
+ "ie",
"if",
"in",
"inf",
@@ -18616,6 +17858,7 @@ var nodeLabels = [...]string{
"kirovograd",
"km",
"kr",
+ "kropyvnytskyi",
"krym",
"ks",
"kv",
@@ -19010,18 +18253,84 @@ var nodeLabels = [...]string{
"net",
"org",
"ac",
+ "ai",
+ "angiang",
+ "bacgiang",
+ "backan",
+ "baclieu",
+ "bacninh",
+ "baria-vungtau",
+ "bentre",
+ "binhdinh",
+ "binhduong",
+ "binhphuoc",
+ "binhthuan",
"biz",
"blogspot",
+ "camau",
+ "cantho",
+ "caobang",
"com",
+ "daklak",
+ "daknong",
+ "danang",
+ "dienbien",
+ "dongnai",
+ "dongthap",
"edu",
+ "gialai",
"gov",
+ "hagiang",
+ "haiduong",
+ "haiphong",
+ "hanam",
+ "hanoi",
+ "hatinh",
+ "haugiang",
"health",
+ "hoabinh",
+ "hungyen",
+ "id",
"info",
"int",
+ "io",
+ "khanhhoa",
+ "kiengiang",
+ "kontum",
+ "laichau",
+ "lamdong",
+ "langson",
+ "laocai",
+ "longan",
+ "namdinh",
"name",
"net",
+ "nghean",
+ "ninhbinh",
+ "ninhthuan",
"org",
+ "phutho",
+ "phuyen",
"pro",
+ "quangbinh",
+ "quangnam",
+ "quangngai",
+ "quangninh",
+ "quangtri",
+ "soctrang",
+ "sonla",
+ "tayninh",
+ "thaibinh",
+ "thainguyen",
+ "thanhhoa",
+ "thanhphohochiminh",
+ "thuathienhue",
+ "tiengiang",
+ "travinh",
+ "tuyenquang",
+ "vinhlong",
+ "vinhphuc",
+ "yenbai",
"blog",
"cn",
"com",
Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.