From 9d8440667cc216284c4355fac63e6d46b7e51740 Mon Sep 17 00:00:00 2001 From: "Lawrence, Rendall" Date: Mon, 11 Jul 2022 13:32:18 +0300 Subject: [PATCH] (tested) Add `reuseport` feauture. SO_REUSEPORT allows multipple instances to use same port. It might be used for better scalability and performance. * update dependencies * fix data race of http's srv and tlsSrv variables --- dist/example_config.yaml | 5 +++ frontend/http/frontend.go | 72 ++++++++++++++++++++++++++++++--------- frontend/udp/frontend.go | 31 ++++++++++++----- go.mod | 9 ++--- go.sum | 12 +++++++ 5 files changed, 101 insertions(+), 28 deletions(-) diff --git a/dist/example_config.yaml b/dist/example_config.yaml index 7ee528b..60ea754 100644 --- a/dist/example_config.yaml +++ b/dist/example_config.yaml @@ -28,6 +28,8 @@ mochi: # The path to the required files to listen via HTTPS. tls_cert_path: "" tls_key_path: "" + # Enable SO_REUSEPORT to allow starting multiple mochi instances with the same HTTP(S) port. + reuse_port: true # The timeout durations for HTTP requests. read_timeout: 5s @@ -95,6 +97,9 @@ mochi: # BitTorrent traffic. addr: "0.0.0.0:6969" + # Enable SO_REUSEPORT to allow starting multiple mochi instances with the same UDP port. + reuse_port: true + # The leeway for a timestamp on a connection ID. max_clock_skew: 10s diff --git a/frontend/http/frontend.go b/frontend/http/frontend.go index c4ee849..ab6c71b 100644 --- a/frontend/http/frontend.go +++ b/frontend/http/frontend.go @@ -6,11 +6,14 @@ import ( "context" "crypto/tls" "errors" + "net" "net/http" "net/netip" + "sync" "time" "github.com/julienschmidt/httprouter" + "github.com/libp2p/go-reuseport" "github.com/sot-tech/mochi/bittorrent" "github.com/sot-tech/mochi/frontend" @@ -33,6 +36,7 @@ type Config struct { EnableKeepAlive bool `cfg:"enable_keepalive"` TLSCertPath string `cfg:"tls_cert_path"` TLSKeyPath string `cfg:"tls_key_path"` + ReusePort bool `cfg:"reuse_port"` AnnounceRoutes []string `cfg:"announce_routes"` ScrapeRoutes []string `cfg:"scrape_routes"` PingRoutes []string `cfg:"ping_routes"` @@ -92,9 +96,11 @@ func (cfg Config) Validate() Config { // Frontend represents the state of an HTTP BitTorrent Frontend. type Frontend struct { - srv *http.Server - tlsSrv *http.Server - tlsCfg *tls.Config + srv *http.Server + srvMu sync.Mutex + tlsSrv *http.Server + tlsSrvMu sync.Mutex + tlsCfg *tls.Config logic frontend.TrackerLogic Config @@ -110,8 +116,10 @@ func NewFrontend(logic frontend.TrackerLogic, c conf.MapConfig) (*Frontend, erro cfg := provided.Validate() f := &Frontend{ - logic: logic, - Config: cfg, + logic: logic, + Config: cfg, + srvMu: sync.Mutex{}, + tlsSrvMu: sync.Mutex{}, } if cfg.Addr == "" && cfg.HTTPSAddr == "" { @@ -177,12 +185,17 @@ func NewFrontend(logic frontend.TrackerLogic, c conf.MapConfig) (*Frontend, erro func (f *Frontend) Stop() stop.Result { stopGroup := stop.NewGroup() + f.srvMu.Lock() if f.srv != nil { stopGroup.AddFunc(f.makeStopFunc(f.srv)) } + f.srvMu.Unlock() + + f.tlsSrvMu.Lock() if f.tlsSrv != nil { stopGroup.AddFunc(f.makeStopFunc(f.tlsSrv)) } + f.tlsSrvMu.Unlock() return stopGroup.Stop() } @@ -211,20 +224,47 @@ func (f *Frontend) serveHTTP(handler http.Handler, tls bool) error { var err error if tls { - srv.Addr = f.HTTPSAddr - srv.TLSConfig = f.tlsCfg - f.tlsSrv = srv - err = srv.ListenAndServeTLS("", "") + if f.ReusePort { + var ln net.Listener + if ln, err = reuseport.Listen("tcp", f.HTTPSAddr); err == nil { + defer ln.Close() + srv.TLSConfig = f.tlsCfg + f.tlsSrvMu.Lock() + f.tlsSrv = srv + f.tlsSrvMu.Unlock() + err = srv.ServeTLS(ln, "", "") + } + } else { + srv.Addr = f.HTTPSAddr + srv.TLSConfig = f.tlsCfg + f.tlsSrvMu.Lock() + f.tlsSrv = srv + f.tlsSrvMu.Unlock() + err = srv.ListenAndServeTLS("", "") + } } else { - srv.Addr = f.Addr - f.srv = srv - err = srv.ListenAndServe() + if f.ReusePort { + var ln net.Listener + if ln, err = reuseport.Listen("tcp", f.Addr); err == nil { + defer ln.Close() + f.srvMu.Lock() + f.srv = srv + f.srvMu.Unlock() + err = srv.Serve(ln) + } + } else { + srv.Addr = f.Addr + f.srvMu.Lock() + f.srv = srv + f.srvMu.Unlock() + err = srv.ListenAndServe() + } } // Start the HTTP server. - if !errors.Is(err, http.ErrServerClosed) { - return err + if errors.Is(err, http.ErrServerClosed) { + err = nil } - return nil + return err } func injectRouteParamsToContext(ctx context.Context, ps httprouter.Params) context.Context { @@ -308,7 +348,7 @@ func (f *Frontend) scrapeRoute(w http.ResponseWriter, r *http.Request, ps httpro go f.logic.AfterScrape(ctx, req, resp) } -func (f Frontend) ping(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { +func (f *Frontend) ping(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { var err error if r.Method == http.MethodGet { err = f.logic.Ping(context.TODO()) diff --git a/frontend/udp/frontend.go b/frontend/udp/frontend.go index 1ae757f..7b21839 100644 --- a/frontend/udp/frontend.go +++ b/frontend/udp/frontend.go @@ -13,6 +13,8 @@ import ( "sync" "time" + "github.com/libp2p/go-reuseport" + "github.com/sot-tech/mochi/bittorrent" "github.com/sot-tech/mochi/frontend" "github.com/sot-tech/mochi/frontend/udp/bytepool" @@ -23,14 +25,17 @@ import ( "github.com/sot-tech/mochi/pkg/timecache" ) -var logger = log.NewLogger("udp frontend") - -var allowedGeneratedPrivateKeyRunes = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890") +var ( + logger = log.NewLogger("udp frontend") + allowedGeneratedPrivateKeyRunes = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890") + errUnexpectedConnType = errors.New("unexpected connection type (not UDPConn)") +) // Config represents all of the configurable options for a UDP BitTorrent // Tracker. type Config struct { Addr string + ReusePort bool `cfg:"reuse_port"` PrivateKey string `cfg:"private_key"` MaxClockSkew time.Duration `cfg:"max_clock_skew"` EnableRequestTiming bool `cfg:"enable_request_timing"` @@ -130,12 +135,22 @@ func (t *Frontend) Stop() stop.Result { } // listen resolves the address and binds the server socket. -func (t *Frontend) listen() error { - udpAddr, err := net.ResolveUDPAddr("udp", t.Addr) - if err != nil { - return err +func (t *Frontend) listen() (err error) { + if t.ReusePort { + var ln net.PacketConn + if ln, err = reuseport.ListenPacket("udp", t.Addr); err == nil { + var isOk bool + if t.socket, isOk = ln.(*net.UDPConn); !isOk { + err = errUnexpectedConnType + } + } + } else { + var udpAddr *net.UDPAddr + udpAddr, err = net.ResolveUDPAddr("udp", t.Addr) + if err == nil { + t.socket, err = net.ListenUDP("udp", udpAddr) + } } - t.socket, err = net.ListenUDP("udp", udpAddr) return err } diff --git a/go.mod b/go.mod index 51170a1..f4160a8 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.18 require ( code.cloudfoundry.org/go-diodes v0.0.0-20220601181242-ac2da19efd60 github.com/SermoDigital/jose v0.9.2-0.20180104203859-803625baeddc - github.com/anacrolix/torrent v1.45.0 + github.com/anacrolix/torrent v1.46.0 github.com/go-redis/redis/v8 v8.11.5 github.com/jackc/pgx/v4 v4.16.1 github.com/julienschmidt/httprouter v1.3.0 @@ -16,12 +16,13 @@ require ( github.com/prometheus/client_golang v1.12.2 github.com/rs/zerolog v1.27.0 github.com/stretchr/testify v1.8.0 + github.com/libp2p/go-reuseport v0.2.0 // indirect gopkg.in/yaml.v3 v3.0.1 ) require ( - github.com/anacrolix/dht/v2 v2.17.0 // indirect - github.com/anacrolix/log v0.13.1 // indirect + github.com/anacrolix/dht/v2 v2.18.0 // indirect + github.com/anacrolix/log v0.13.2-0.20220426014722-7b7d13a55d55 // indirect github.com/anacrolix/missinggo v1.3.0 // indirect github.com/anacrolix/missinggo/v2 v2.7.0 // indirect github.com/beorn7/perks v1.0.1 // indirect @@ -49,7 +50,7 @@ require ( github.com/prometheus/common v0.35.0 // indirect github.com/prometheus/procfs v0.7.3 // indirect golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d // indirect - golang.org/x/sys v0.0.0-20220622161953-175b2fd9d664 // indirect + golang.org/x/sys v0.0.0-20220708085239-5a0f0661e09d // indirect golang.org/x/text v0.3.7 // indirect google.golang.org/protobuf v1.28.0 // indirect ) diff --git a/go.sum b/go.sum index 6be96f1..54414ff 100644 --- a/go.sum +++ b/go.sum @@ -52,6 +52,8 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/anacrolix/dht/v2 v2.17.0 h1:MrAS6XqVCqNyyskTwxZB2sqhU/GGUdecb2TNe2b2QjE= github.com/anacrolix/dht/v2 v2.17.0/go.mod h1:osiyaNrMLG9dw7wUtVMaII/NdCjlXeHjUcYzXnmop68= +github.com/anacrolix/dht/v2 v2.18.0 h1:btjVjzjKqO5nKGbJHJ2UmuwiRx+EgX3e+OCHC9+WRz8= +github.com/anacrolix/dht/v2 v2.18.0/go.mod h1:mxrSeP/LIY429SgWMO9o6UdjBjB8ZjBh6HHCmd8Ly1g= github.com/anacrolix/envpprof v0.0.0-20180404065416-323002cec2fa/go.mod h1:KgHhUaQMc8cC0+cEflSgCFNFbKwi5h54gqtVn8yhP7c= github.com/anacrolix/envpprof v1.0.0/go.mod h1:KgHhUaQMc8cC0+cEflSgCFNFbKwi5h54gqtVn8yhP7c= github.com/anacrolix/envpprof v1.1.0/go.mod h1:My7T5oSqVfEn4MD4Meczkw/f5lSIndGAKu/0SM/rkf4= @@ -60,6 +62,8 @@ github.com/anacrolix/log v0.3.0/go.mod h1:lWvLTqzAnCWPJA08T2HCstZi0L1y2Wyvm3FJgw github.com/anacrolix/log v0.6.0/go.mod h1:lWvLTqzAnCWPJA08T2HCstZi0L1y2Wyvm3FJgwU9jwU= github.com/anacrolix/log v0.13.1 h1:BmVwTdxHd5VcNrLylgKwph4P4wf+5VvPgOK4yi91fTY= github.com/anacrolix/log v0.13.1/go.mod h1:D4+CvN8SnruK6zIFS/xPoRJmtvtnxs+CSfDQ+BFxZ68= +github.com/anacrolix/log v0.13.2-0.20220426014722-7b7d13a55d55 h1:22o7jDD18WtIYUa2I983X58o6t8e9GVbcdYOKoy+Y4g= +github.com/anacrolix/log v0.13.2-0.20220426014722-7b7d13a55d55/go.mod h1:D4+CvN8SnruK6zIFS/xPoRJmtvtnxs+CSfDQ+BFxZ68= github.com/anacrolix/missinggo v1.1.0/go.mod h1:MBJu3Sk/k3ZfGYcS7z18gwfu72Ey/xopPFJJbTi5yIo= github.com/anacrolix/missinggo v1.1.2-0.20190815015349-b888af804467/go.mod h1:MBJu3Sk/k3ZfGYcS7z18gwfu72Ey/xopPFJJbTi5yIo= github.com/anacrolix/missinggo v1.2.1/go.mod h1:J5cMhif8jPmFoC3+Uvob3OXXNIhOUikzMt+uUjeM21Y= @@ -76,6 +80,8 @@ github.com/anacrolix/tagflag v1.0.0/go.mod h1:1m2U/K6ZT+JZG0+bdMK6qauP49QT4wE5pm github.com/anacrolix/tagflag v1.1.0/go.mod h1:Scxs9CV10NQatSmbyjqmqmeQNwGzlNe0CMUMIxqHIG8= github.com/anacrolix/torrent v1.45.0 h1:dmRfw+kSl+UtBk2gAOpJgdnu3OzWsSmE6sMTunifpdw= github.com/anacrolix/torrent v1.45.0/go.mod h1:511SlHgyD0LAeN5CIJw8CSfS4BaBVDtfVkRSMHPjfAQ= +github.com/anacrolix/torrent v1.46.0 h1:m5SGlW4p0dJkqrIh4bCqzcKbKFFmfJorf9LSpSM5dEc= +github.com/anacrolix/torrent v1.46.0/go.mod h1:3DE+VA4AgyfKDPjZcIo70D3VFZRo3bfdEBn70CGjca4= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/benbjohnson/immutable v0.2.0/go.mod h1:uc6OHo6PN2++n98KHLxW8ef4W42ylHiQSENghE1ezxI= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -217,6 +223,7 @@ github.com/huandu/xstrings v1.3.1/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq github.com/huandu/xstrings v1.3.2 h1:L18LIDzqlW6xN2rEkpdV8+oL/IXWJ1APd+vsdYy4Wdw= github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/jackc/chunkreader v1.0.0 h1:4s39bBR8ByfqH+DKm8rQA3E1LHZWB9XWcrz8fqaZbe0= github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= @@ -237,6 +244,7 @@ github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65 h1:DadwsjnMwFjfWc9y5W github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65/go.mod h1:5R2h2EEX+qri8jOWMbJCtaPWkrrNc7OHwsp2TCqp7ak= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgproto3 v1.1.0 h1:FYYE4yRw+AgI8wXIinMlNjBbp/UitDJwfj5LqqewP1A= github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg= @@ -299,6 +307,8 @@ github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.10.2 h1:AqzbZs4ZoCBp+GtejcpCpcxM3zlSMx29dXbUSeVtJb8= github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/libp2p/go-reuseport v0.2.0 h1:18PRvIMlpY6ZK85nIAicSBuXXvrYoSw3dsBAR7zc560= +github.com/libp2p/go-reuseport v0.2.0/go.mod h1:bvVho6eLMm6Bz5hmU0LYN3ixd3nPPvtIlaURZZgOY4k= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= @@ -583,6 +593,8 @@ golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220622161953-175b2fd9d664 h1:wEZYwx+kK+KlZ0hpvP2Ls1Xr4+RWnlzGFwPP0aiDjIU= golang.org/x/sys v0.0.0-20220622161953-175b2fd9d664/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220708085239-5a0f0661e09d h1:/m5NbqQelATgoSPVC2Z23sR4kVNokFwDDyWh/3rGY+I= +golang.org/x/sys v0.0.0-20220708085239-5a0f0661e09d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=