add filter_private_ips option to discard private IPs.

Might be used when tracker is behind reverse proxy and one of provided
addresses in `real_ip_header` is private/local address.

Additional changes:

* check if provided address is not multicast/broadcast
* configure `http.Server.ReadHeaderTimeout` with `http.ReadTimeout` to mitigate Slowloris
* update dependencies
* minor docs fixes
This commit is contained in:
Lawrence, Rendall
2022-07-23 15:30:12 +03:00
parent 3e36ad7cbf
commit 96653c45a3
13 changed files with 125 additions and 63 deletions
+16 -12
View File
@@ -15,9 +15,12 @@ type RequestAddress struct {
Provided bool
}
// Validate checks if netip.Addr is valid and not unspecified (0.0.0.0)
func (a RequestAddress) Validate() bool {
return a.IsValid() && !a.IsUnspecified()
// Note: there is no IPv6 broadcast address
var globalBroadcastIPv4 = netip.AddrFrom4([4]byte{255, 255, 255, 255})
// IsValid checks if netip.Addr is valid, not unspecified and not multicast
func (a RequestAddress) IsValid() bool {
return a.Addr.IsValid() && !(a.IsUnspecified() || a.IsMulticast() || a.Addr == globalBroadcastIPv4)
}
// MarshalZerologObject writes fields into zerolog event
@@ -50,29 +53,30 @@ func (aa RequestAddresses) Swap(i, j int) {
// Add checks if provided RequestAddress is valid and adds unmapped
// netip.Addr to array
func (aa *RequestAddresses) Add(a RequestAddress) {
if a.Validate() {
if a.IsValid() {
a.Addr = a.Unmap()
*aa = append(*aa, a)
}
}
// Validate checks if array is not empty and at least one RequestAddress is valid,
// then make them unique and sorts with Less rule
func (aa *RequestAddresses) Validate() bool {
// Sanitize checks if array is not empty and at least one RequestAddress is valid,
// then make them unique and sorts with RequestAddresses.Less rule.
// If ignorePrivate set to true, function will preserve only global unicast and
// non-private (see netip.IsGlobalUnicast and netip.IsPrivate) addresses.
// If there are no valid and global (if ignorePrivate checked) addresses in array,
// function returns false and empty receiver.
func (aa *RequestAddresses) Sanitize(ignorePrivate bool) bool {
if len(*aa) == 0 {
return false
}
uniqueAddresses := make(map[netip.Addr]bool, len(*aa))
for _, a := range *aa {
if a.Validate() {
if a.IsValid() && (!ignorePrivate || a.IsGlobalUnicast() && !a.IsPrivate()) {
if provided, found := uniqueAddresses[a.Addr]; !found || !provided && a.Provided {
uniqueAddresses[a.Addr] = a.Provided
}
}
}
if len(uniqueAddresses) == 0 {
return false
}
*aa = make(RequestAddresses, 0, len(uniqueAddresses))
for a, p := range uniqueAddresses {
*aa = append(*aa, RequestAddress{a, p})
@@ -80,7 +84,7 @@ func (aa *RequestAddresses) Validate() bool {
if len(*aa) > 1 {
sort.Sort(*aa)
}
return true
return len(uniqueAddresses) > 0
}
// GetFirst returns first address from array
+55 -16
View File
@@ -7,25 +7,64 @@ import (
"github.com/stretchr/testify/require"
)
// TestRequestAddresses_Validate test fix issue RM#5617
func TestRequestAddresses_Validate(t *testing.T) {
ra := make(RequestAddresses, 0, 3)
ra = append(ra, RequestAddress{
Addr: netip.MustParseAddr("1.2.3.4"),
var addresses = RequestAddresses{
RequestAddress{
Addr: netip.MustParseAddr("1.2.3.4"), // valid global
Provided: false,
})
ra = append(ra, RequestAddress{
Addr: netip.MustParseAddr("1.2.3.4"),
}, RequestAddress{
Addr: netip.MustParseAddr("1.2.3.4"), // valid global (duplicated)
Provided: true,
})
ra = append(ra, RequestAddress{
Addr: netip.MustParseAddr("4.3.2.1"),
}, RequestAddress{
Addr: netip.MustParseAddr("4.3.2.1"), // valid global
Provided: false,
})
ra = append(ra, RequestAddress{
Addr: netip.MustParseAddr("4.3.2.1"),
}, RequestAddress{
Addr: netip.MustParseAddr("4.3.2.1"), // valid global (duplicated)
Provided: false,
})
require.True(t, ra.Validate())
}, RequestAddress{
Addr: netip.MustParseAddr("10.0.0.1"), // valid local
Provided: false,
}, RequestAddress{
Addr: netip.MustParseAddr("172.16.0.1"), // valid local
Provided: true,
}, RequestAddress{
Addr: netip.MustParseAddr("192.168.0.1"), // valid local
Provided: false,
}, RequestAddress{
Addr: netip.MustParseAddr("127.0.0.1"), // valid loopback
Provided: true,
}, RequestAddress{
Addr: netip.MustParseAddr("224.0.0.1"), // invalid (multicast)
Provided: true,
}, RequestAddress{
Addr: netip.MustParseAddr("233.252.0.1"), // invalid (multicast)
Provided: true,
}, RequestAddress{
Addr: netip.MustParseAddr("255.255.255.255"), // invalid (broadcast)
Provided: true,
}, RequestAddress{
Addr: netip.MustParseAddr("169.254.0.1"), // valid link-local
Provided: true,
}, RequestAddress{
Addr: netip.MustParseAddr("ff01::1"), // invalid (multicast)
Provided: true,
}, RequestAddress{
Addr: netip.MustParseAddr("fe80::1"), // valid link-local
Provided: true,
},
}
// TestRequestAddresses_SanitizeWPrivate test fix issue RM#5617
func TestRequestAddresses_SanitizeWPrivate(t *testing.T) {
ra := make(RequestAddresses, len(addresses))
copy(ra, addresses)
require.True(t, ra.Sanitize(false))
require.Equal(t, 8, len(ra))
}
// TestRequestAddresses_SanitizeAll test fix issue RM#5783
func TestRequestAddresses_SanitizeWOPrivate(t *testing.T) {
ra := make(RequestAddresses, len(addresses))
copy(ra, addresses)
require.True(t, ra.Sanitize(true))
require.Equal(t, 2, len(ra))
}
+4 -4
View File
@@ -15,13 +15,13 @@ var (
// SanitizeAnnounce enforces a max and default NumWant and coerces the peer's
// IP address into the proper format.
func SanitizeAnnounce(r *AnnounceRequest, maxNumWant, defaultNumWant uint32) error {
func SanitizeAnnounce(r *AnnounceRequest, maxNumWant, defaultNumWant uint32, filterPrivate bool) error {
logger.Trace().Object("request", r).Msg("source announce")
if r.Port == 0 {
return ErrInvalidPort
}
if !r.Validate() {
if !r.Sanitize(filterPrivate) {
return ErrInvalidIP
}
@@ -37,13 +37,13 @@ func SanitizeAnnounce(r *AnnounceRequest, maxNumWant, defaultNumWant uint32) err
// SanitizeScrape enforces a max number of infohashes for a single scrape
// request and checks if addresses are valid.
func SanitizeScrape(r *ScrapeRequest, maxScrapeInfoHashes uint32) error {
func SanitizeScrape(r *ScrapeRequest, maxScrapeInfoHashes uint32, filterPrivate bool) error {
logger.Trace().Object("request", r).Msg("source scrape")
if len(r.InfoHashes) > int(maxScrapeInfoHashes) {
r.InfoHashes = r.InfoHashes[:maxScrapeInfoHashes]
}
if !r.Validate() {
if !r.Sanitize(filterPrivate) {
return ErrInvalidIP
}
+12 -4
View File
@@ -73,10 +73,14 @@ mochi:
ping_routes:
- "/ping"
# When enabled, the IP address used to connect to the tracker will not
# override the value clients advertise as their IP address.
# When not enabled, tracker will use only address from which client connected to tracker.
# When enabled, the IP address that clients advertise as their IP address will
# be appended as announce candidate.
allow_ip_spoofing: false
# When enabled, IPs from private, local and loopback subnets will be ignored
filter_private_ips: false
# The HTTP Header containing the IP address of the client.
# This is only necessary if using a reverse proxy.
real_ip_header: "x-real-ip"
@@ -110,10 +114,14 @@ mochi:
# Disabling this should increase performance/decrease load.
enable_request_timing: false
# When enabled, the IP address used to connect to the tracker will not
# override the value clients advertise as their IP address.
# When not enabled, tracker will use only address from which client connected to tracker.
# When enabled, the IP address that clients advertise as their IP address will
# be appended as announce candidate.
allow_ip_spoofing: false
# When enabled, IPs from private, local and loopback subnets will be ignored
filter_private_ips: false
# The maximum number of peers returned for an individual request.
max_numwant: 100
+1 -1
View File
@@ -50,7 +50,7 @@ The typical control flow of handling announces, in more detail, is:
#### Configuration
The frontend must be configurable using a single, exported struct. The struct must have YAML annotations. The struct
must implement `log.Fielder` to be logged on startup.
must implement `zerolog.LogObjectMarshaler` to be logged on startup.
#### Metrics
+1 -1
View File
@@ -116,7 +116,7 @@ storage:
graduate_query: UPDATE mo_peers SET is_seeder=TRUE WHERE info_hash=$1 AND peer_id=$2 AND address=$3 AND port=$4 AND NOT is_seeder
# Query to get count of peers.
# Used both for statistics and for scrape (with clause suffix, see next).
# Only first returned row values used.
# Only first returned row value used.
count_query: SELECT COUNT(1) FILTER (WHERE is_seeder) AS seeders, COUNT(1) FILTER (WHERE NOT is_seeder) AS leechers FROM mo_peers
# Predicate part of `count_query` for get count of peers by info hash
by_info_hash_clause: WHERE info_hash = $1
+5 -4
View File
@@ -214,10 +214,11 @@ func (f *Frontend) makeStopFunc(stopSrv *http.Server) stop.Func {
// requests until Stop() is called or an error is returned.
func (f *Frontend) serveHTTP(handler http.Handler, tls bool) error {
srv := &http.Server{
Handler: handler,
ReadTimeout: f.ReadTimeout,
WriteTimeout: f.WriteTimeout,
IdleTimeout: f.IdleTimeout,
Handler: handler,
ReadTimeout: f.ReadTimeout,
ReadHeaderTimeout: f.ReadTimeout,
WriteTimeout: f.WriteTimeout,
IdleTimeout: f.IdleTimeout,
}
srv.SetKeepAlivesEnabled(f.EnableKeepAlive)
+4 -5
View File
@@ -110,7 +110,7 @@ func ParseAnnounce(r *http.Request, opts ParseOptions) (*bittorrent.AnnounceRequ
// Parse the IP address where the client is listening.
request.RequestAddresses = requestedIPs(r, qp, opts)
if err = bittorrent.SanitizeAnnounce(request, opts.MaxNumWant, opts.DefaultNumWant); err != nil {
if err = bittorrent.SanitizeAnnounce(request, opts.MaxNumWant, opts.DefaultNumWant, opts.FilterPrivateIPs); err != nil {
request = nil
}
@@ -135,7 +135,7 @@ func ParseScrape(r *http.Request, opts ParseOptions) (*bittorrent.ScrapeRequest,
RequestAddresses: requestedIPs(r, qp, opts),
}
err = bittorrent.SanitizeScrape(request, opts.MaxScrapeInfoHashes)
err = bittorrent.SanitizeScrape(request, opts.MaxScrapeInfoHashes, opts.FilterPrivateIPs)
return request, err
}
@@ -169,9 +169,8 @@ func requestedIPs(r *http.Request, p bittorrent.Params, opts ParseOptions) (addr
}
func parseRequestAddress(s string, provided bool) (ra bittorrent.RequestAddress) {
a, e := netip.ParseAddr(s)
if e == nil {
ra.Addr, ra.Provided = a, provided
if addr, err := netip.ParseAddr(s); err == nil {
ra.Addr, ra.Provided = addr, provided
}
return
}
+1
View File
@@ -9,6 +9,7 @@ var logger = log.NewLogger("frontend configurator")
// If AllowIPSpoofing is true, IPs provided via params will be used.
type ParseOptions struct {
AllowIPSpoofing bool `cfg:"allow_ip_spoofing"`
FilterPrivateIPs bool `cfg:"filter_private_ips"`
MaxNumWant uint32 `cfg:"max_numwant"`
DefaultNumWant uint32 `cfg:"default_numwant"`
MaxScrapeInfoHashes uint32 `cfg:"max_scrape_infohashes"`
+2 -2
View File
@@ -104,7 +104,7 @@ func ParseAnnounce(r Request, v6Action bool, opts frontend.ParseOptions) (*bitto
return nil, err
}
if err = bittorrent.SanitizeAnnounce(request, opts.MaxNumWant, opts.DefaultNumWant); err != nil {
if err = bittorrent.SanitizeAnnounce(request, opts.MaxNumWant, opts.DefaultNumWant, opts.FilterPrivateIPs); err != nil {
request = nil
}
@@ -207,7 +207,7 @@ func ParseScrape(r Request, opts frontend.ParseOptions) (*bittorrent.ScrapeReque
RequestAddresses: bittorrent.RequestAddresses{bittorrent.RequestAddress{Addr: r.IP}},
}
err = bittorrent.SanitizeScrape(request, opts.MaxScrapeInfoHashes)
err = bittorrent.SanitizeScrape(request, opts.MaxScrapeInfoHashes, opts.FilterPrivateIPs)
}
return request, err
+4 -4
View File
@@ -41,16 +41,16 @@ require (
github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect
github.com/jackc/pgtype v1.11.0 // indirect
github.com/jackc/puddle v1.2.1 // indirect
github.com/klauspost/cpuid/v2 v2.0.14 // indirect
github.com/klauspost/cpuid/v2 v2.1.0 // indirect
github.com/mattn/go-colorable v0.1.12 // indirect
github.com/mattn/go-isatty v0.0.14 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_model v0.2.0 // indirect
github.com/prometheus/common v0.35.0 // indirect
github.com/prometheus/common v0.37.0 // indirect
github.com/prometheus/procfs v0.7.3 // indirect
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d // indirect
golang.org/x/sys v0.0.0-20220708085239-5a0f0661e09d // indirect
golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa // indirect
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f // indirect
golang.org/x/text v0.3.7 // indirect
google.golang.org/protobuf v1.28.0 // indirect
)
+9 -8
View File
@@ -280,8 +280,8 @@ github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4d
github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.0.14 h1:QRqdp6bb9M9S5yyKeYteXKuoKE4p0tGlra81fKOpWH8=
github.com/klauspost/cpuid/v2 v2.0.14/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c=
github.com/klauspost/cpuid/v2 v2.1.0 h1:eyi1Ad2aNJMW95zcSbmGg7Cg6cq3ADwLpMAP96d8rF0=
github.com/klauspost/cpuid/v2 v2.1.0/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
@@ -364,8 +364,8 @@ github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8b
github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
github.com/prometheus/common v0.35.0 h1:Eyr+Pw2VymWejHqCugNaQXkAi6KayVNxaHeu6khmFBE=
github.com/prometheus/common v0.35.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA=
github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE=
github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA=
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
@@ -450,8 +450,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh
golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d h1:sK3txAijHtOK88l68nt020reeT1ZdKLIYetKl95FzVY=
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa h1:zuSxTR4o9y82ebqCUJYNGJbGPo6sKVl54f/TVDObg1c=
golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -583,8 +583,9 @@ golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220708085239-5a0f0661e09d h1:/m5NbqQelATgoSPVC2Z23sR4kVNokFwDDyWh/3rGY+I=
golang.org/x/sys v0.0.0-20220708085239-5a0f0661e09d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f h1:v4INt8xihDGvnrfjMDVXGxw9wrfxYyCjk0KbXjhR55s=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+11 -2
View File
@@ -9,6 +9,7 @@ import (
"net/http/pprof"
"net/netip"
"sync/atomic"
"time"
"github.com/prometheus/client_golang/prometheus/promhttp"
@@ -16,6 +17,11 @@ import (
"github.com/sot-tech/mochi/pkg/stop"
)
const (
readTimeout = 5 * time.Second
writeTimeout = readTimeout * 2
)
var (
logger = log.NewLogger("metrics")
serverCounter = new(int32)
@@ -68,8 +74,11 @@ func NewServer(addr string) *Server {
s := &Server{
srv: &http.Server{
Addr: addr,
Handler: mux,
Addr: addr,
Handler: mux,
ReadTimeout: readTimeout,
ReadHeaderTimeout: readTimeout,
WriteTimeout: writeTimeout,
},
}