diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index f03d8fb..f218839 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -82,13 +82,10 @@ jobs: run: | go install ./cmd/mochi go install ./cmd/mochi-e2e - curl -LO https://github.com/jzelinskie/faq/releases/download/0.0.6/faq-linux-amd64 - chmod +x faq-linux-amd64 - ./faq-linux-amd64 '.mochi.storage = {"config":{"gc_interval":"3m","peer_lifetime":"31m","prometheus_reporting_interval":"1s","connect_timeout":"15s","read_timeout":"15s","write_timeout":"15s"},"name":"redis"}' ./dist/example_config.yaml > ./dist/example_redis_config.yaml - cat ./dist/example_redis_config.yaml + cat ./dist/example_config_redis.yaml - name: "Run end-to-end tests" run: | - mochi --config=./dist/example_redis_config.yaml --logLevel debug --logPretty & + mochi --config=./dist/example_config_redis.yaml --logLevel debug --logPretty & pid=$! sleep 2 mochi-e2e diff --git a/bittorrent/bittorrent.go b/bittorrent/peer.go similarity index 91% rename from bittorrent/bittorrent.go rename to bittorrent/peer.go index 8a62a29..701413b 100644 --- a/bittorrent/bittorrent.go +++ b/bittorrent/peer.go @@ -117,22 +117,6 @@ func (i InfoHash) RawString() string { return string(i) } -// Scrape represents the state of a swarm that is returned in a scrape response. -type Scrape struct { - InfoHash InfoHash - Snatches uint32 - Complete uint32 - Incomplete uint32 -} - -// MarshalZerologObject writes fields into zerolog event -func (s Scrape) MarshalZerologObject(e *zerolog.Event) { - e.Stringer("infoHash", s.InfoHash). - Uint32("snatches", s.Snatches). - Uint32("complete", s.Complete). - Uint32("incomplete", s.Incomplete) -} - // Peer represents the connection details of a peer that is returned in an // announce response. type Peer struct { diff --git a/bittorrent/request.go b/bittorrent/request.go index 9ea096d..b5cd4a0 100644 --- a/bittorrent/request.go +++ b/bittorrent/request.go @@ -35,19 +35,19 @@ func (a RequestAddress) MarshalZerologObject(e *zerolog.Event) { // connection information about peer type RequestAddresses []RequestAddress -func (aa RequestAddresses) Len() int { - return len(aa) +func (aa *RequestAddresses) Len() int { + return len(*aa) } // Less returns true only if i-th RequestAddress is marked as // RequestAddress.Provided and j-th is not (provided address has // higher priority) -func (aa RequestAddresses) Less(i, j int) bool { - return aa[i].Provided && !aa[j].Provided +func (aa *RequestAddresses) Less(i, j int) bool { + return (*aa)[i].Provided && !(*aa)[j].Provided } -func (aa RequestAddresses) Swap(i, j int) { - aa[i], aa[j] = aa[j], aa[i] +func (aa *RequestAddresses) Swap(i, j int) { + (*aa)[i], (*aa)[j] = (*aa)[j], (*aa)[i] } // Add checks if provided RequestAddress is valid and adds unmapped @@ -82,25 +82,27 @@ func (aa *RequestAddresses) Sanitize(ignorePrivate bool) bool { *aa = append(*aa, RequestAddress{a, p}) } if len(*aa) > 1 { - sort.Sort(*aa) + sort.Sort(aa) } return len(uniqueAddresses) > 0 } // GetFirst returns first address from array // or empty netip.Addr if array is empty -func (aa RequestAddresses) GetFirst() netip.Addr { +func (aa *RequestAddresses) GetFirst() netip.Addr { var a netip.Addr - if len(aa) > 0 { - a = aa[0].Addr + if len(*aa) > 0 { + a = (*aa)[0].Addr } return a } // MarshalZerologArray writes array elements to zerolog event -func (aa RequestAddresses) MarshalZerologArray(a *zerolog.Array) { - for _, addr := range aa { - a.Object(addr) +func (aa *RequestAddresses) MarshalZerologArray(a *zerolog.Array) { + if aa != nil { + for _, addr := range *aa { + a.Object(addr) + } } } @@ -137,7 +139,7 @@ func (rp RequestPeer) Peers() (peers Peers) { // MarshalZerologObject writes fields into zerolog event func (rp RequestPeer) MarshalZerologObject(e *zerolog.Event) { e.Stringer("id", rp.ID). - Array("addresses", rp.RequestAddresses). + Array("addresses", &rp.RequestAddresses). Uint16("port", rp.Port) } @@ -216,18 +218,48 @@ type ScrapeRequest struct { // MarshalZerologObject writes fields into zerolog event func (r ScrapeRequest) MarshalZerologObject(e *zerolog.Event) { - e.Array("addresses", r.RequestAddresses). + e.Array("addresses", &r.RequestAddresses). Array("infoHashes", r.InfoHashes). Object("params", r.Params) } +// Scrape represents the state of a swarm that is returned in a scrape response. +type Scrape struct { + InfoHash InfoHash + Snatches uint32 + Complete uint32 + Incomplete uint32 +} + +// MarshalZerologObject writes fields into zerolog event +func (s Scrape) MarshalZerologObject(e *zerolog.Event) { + e.Stringer("infoHash", s.InfoHash). + Uint32("snatches", s.Snatches). + Uint32("complete", s.Complete). + Uint32("incomplete", s.Incomplete) +} + // Scrapes wrapper of array of Scrape-s type Scrapes []Scrape +func (s *Scrapes) Len() int { + return len(*s) +} + +func (s *Scrapes) Less(i, j int) bool { + return (*s)[i].InfoHash < (*s)[j].InfoHash +} + +func (s *Scrapes) Swap(i, j int) { + (*s)[i], (*s)[j] = (*s)[j], (*s)[i] +} + // MarshalZerologArray writes array elements to zerolog event -func (s Scrapes) MarshalZerologArray(a *zerolog.Array) { - for _, scrape := range s { - a.Object(scrape) +func (s *Scrapes) MarshalZerologArray(a *zerolog.Array) { + if s != nil { + for _, scrape := range *s { + a.Object(scrape) + } } } @@ -241,5 +273,5 @@ type ScrapeResponse struct { // MarshalZerologObject writes fields into zerolog event func (sr ScrapeResponse) MarshalZerologObject(e *zerolog.Event) { - e.Array("scrapes", sr.Files) + e.Array("scrapes", &sr.Files) } diff --git a/bittorrent/sanitize.go b/bittorrent/sanitize.go index fa1a482..c14a4a5 100644 --- a/bittorrent/sanitize.go +++ b/bittorrent/sanitize.go @@ -5,7 +5,7 @@ import ( ) var ( - logger = log.NewLogger("request sanitizer") + logger = log.NewLogger("bittorrent/sanitize") // ErrInvalidIP indicates an invalid IP for an Announce. ErrInvalidIP = ClientError("invalid IP") diff --git a/cmd/mochi/config.go b/cmd/mochi/config.go index f665f5c..b32385e 100644 --- a/cmd/mochi/config.go +++ b/cmd/mochi/config.go @@ -9,7 +9,11 @@ import ( "github.com/sot-tech/mochi/pkg/conf" - // Imports to register middleware drivers. + // Imports to register frontends + _ "github.com/sot-tech/mochi/frontend/http" + _ "github.com/sot-tech/mochi/frontend/udp" + + // Imports to register middleware hooks. _ "github.com/sot-tech/mochi/middleware/clientapproval" _ "github.com/sot-tech/mochi/middleware/jwt" _ "github.com/sot-tech/mochi/middleware/torrentapproval" @@ -29,29 +33,20 @@ type Config struct { // We can make Conf extensible enough that you can program a new response // generator at the cost of making it possible for users to create config that // won't compose a functional tracker. - AnnounceInterval time.Duration `yaml:"announce_interval"` - MinAnnounceInterval time.Duration `yaml:"min_announce_interval"` - MetricsAddr string `yaml:"metrics_addr"` - HTTPConfig conf.MapConfig `yaml:"http"` - UDPConfig conf.MapConfig `yaml:"udp"` - Storage struct { - Name string `yaml:"name"` - Config conf.MapConfig `yaml:"config"` - } `yaml:"storage"` - PreHooks []conf.MapConfig `yaml:"prehooks"` - PostHooks []conf.MapConfig `yaml:"posthooks"` + AnnounceInterval time.Duration `yaml:"announce_interval"` + MinAnnounceInterval time.Duration `yaml:"min_announce_interval"` + MetricsAddr string `yaml:"metrics_addr"` + Frontends []conf.NamedMapConfig `yaml:"frontends"` + Storage conf.NamedMapConfig `yaml:"storage"` + PreHooks []conf.NamedMapConfig `yaml:"prehooks"` + PostHooks []conf.NamedMapConfig `yaml:"posthooks"` } -// ConfigFile represents a namespaced YAML configation file. -type ConfigFile struct { - Conf Config `yaml:"mochi"` -} - -// ParseConfigFile returns a new ConfigFile given the path to a YAML +// ParseConfigFile returns a new Config given the path to a YAML // configuration file. // // It supports relative and absolute paths and environment variables. -func ParseConfigFile(path string) (*ConfigFile, error) { +func ParseConfigFile(path string) (*Config, error) { if path == "" { return nil, errors.New("no config path specified") } @@ -59,7 +54,7 @@ func ParseConfigFile(path string) (*ConfigFile, error) { f, err := os.Open(os.ExpandEnv(path)) if err == nil { defer f.Close() - cfgFile := new(ConfigFile) + cfgFile := new(Config) err = yaml.NewDecoder(f).Decode(cfgFile) return cfgFile, err } diff --git a/cmd/mochi/server.go b/cmd/mochi/server.go index 7f42248..d3dc243 100644 --- a/cmd/mochi/server.go +++ b/cmd/mochi/server.go @@ -4,8 +4,7 @@ import ( "errors" "fmt" - "github.com/sot-tech/mochi/frontend/http" - "github.com/sot-tech/mochi/frontend/udp" + "github.com/sot-tech/mochi/frontend" "github.com/sot-tech/mochi/middleware" "github.com/sot-tech/mochi/pkg/log" "github.com/sot-tech/mochi/pkg/metrics" @@ -24,11 +23,10 @@ type Server struct { // It is optional to provide an instance of the peer store to avoid the // creation of a new one. func (r *Server) Run(configFilePath string) error { - configFile, err := ParseConfigFile(configFilePath) + cfg, err := ParseConfigFile(configFilePath) if err != nil { return fmt.Errorf("failed to read config: %w", err) } - cfg := configFile.Conf r.sg = stop.NewGroup() @@ -39,51 +37,35 @@ func (r *Server) Run(configFilePath string) error { log.Info().Msg("metrics disabled because of empty address") } - log.Info().Str("name", cfg.Storage.Name).Msg("starting storage") - r.storage, err = storage.NewStorage(cfg.Storage.Name, cfg.Storage.Config) + r.storage, err = storage.NewStorage(cfg.Storage) if err != nil { return fmt.Errorf("failed to create storage: %w", err) } - log.Info().Object("config", r.storage).Msg("started storage") preHooks, err := middleware.NewHooks(cfg.PreHooks, r.storage) if err != nil { - return fmt.Errorf("failed to validate hook config: %w", err) + return fmt.Errorf("failed to configure pre-hooks: %w", err) } postHooks, err := middleware.NewHooks(cfg.PostHooks, r.storage) if err != nil { - return fmt.Errorf("failed to validate hook config: %w", err) + return fmt.Errorf("failed to configure post-hooks: %w", err) } - r.logic = middleware.NewLogic(cfg.AnnounceInterval, cfg.MinAnnounceInterval, r.storage, preHooks, postHooks) - - var started bool - if len(cfg.HTTPConfig) > 0 { - log.Info().Object("config", cfg.HTTPConfig).Msg("starting HTTP frontend") - httpFE, err := http.NewFrontend(r.logic, cfg.HTTPConfig) - if err == nil { - r.sg.Add(httpFE) - started = true + if len(cfg.Frontends) > 0 { + var fs []frontend.Frontend + r.logic = middleware.NewLogic(cfg.AnnounceInterval, cfg.MinAnnounceInterval, r.storage, preHooks, postHooks) + if fs, err = frontend.NewFrontends(cfg.Frontends, r.logic); err == nil { + for _, f := range fs { + r.sg.Add(f) + } } else { - return err + err = fmt.Errorf("failed to configure frontends: %w", err) } + } else { + err = errors.New("no frontends configured") } - if len(cfg.UDPConfig) > 0 { - log.Info().Object("config", cfg.UDPConfig).Msg("starting UDP frontend") - udpFE, err := udp.NewFrontend(r.logic, cfg.UDPConfig) - if err == nil { - r.sg.Add(udpFE) - started = true - } else { - return err - } - } - if !started { - return errors.New("no frontends configured") - } - - return nil + return err } // Dispose shuts down an instance of Server. diff --git a/dist/example_config.yaml b/dist/example_config.yaml index ed059b6..3713af9 100644 --- a/dist/example_config.yaml +++ b/dist/example_config.yaml @@ -1,303 +1,201 @@ # @formatter:off -mochi: - # The interval communicated with BitTorrent clients informing them how - # frequently they should announce in between client events. - announce_interval: 30m +# The interval communicated with BitTorrent clients informing them how +# frequently they should announce in between client events. +announce_interval: 30m - # The interval communicated with BitTorrent clients informing them of the - # minimal duration between announces. - min_announce_interval: 15m +# The interval communicated with BitTorrent clients informing them of the +# minimal duration between announces. +min_announce_interval: 15m - # The network interface that will bind to an HTTP endpoint that can be - # scraped by programs collecting metrics. - # - # /metrics serves metrics in the Prometheus format - # /debug/pprof/{cmdline,profile,symbol,trace} serves profiles in the pprof format - metrics_addr: "0.0.0.0:6880" +# The network interface that will bind to an HTTP endpoint that can be +# scraped by programs collecting metrics. +# +# /metrics serves metrics in the Prometheus format +# /debug/pprof/{cmdline,profile,symbol,trace} serves profiles in the pprof format +metrics_addr: "0.0.0.0:6880" +frontends: # This block defines configuration for the tracker's HTTP interface. # If you do not wish to run this, delete this section. - http: - # The network interface that will bind to an HTTP server for serving - # BitTorrent traffic. Remove this to disable the non-TLS listener. - addr: "0.0.0.0:6969" - - # The network interface that will bind to an HTTPS server for serving - # BitTorrent traffic. If set, tls_cert_path and tls_key_path are required. - https_addr: "" - - # The path to the required files to listen via HTTPS. - tls_cert_path: "" - tls_key_path: "" - # Enable SO_REUSEPORT to allow starting multiple mochi instances with the same HTTP(S) port. - reuse_port: true - - # The timeout durations for HTTP requests. - read_timeout: 5s - write_timeout: 5s - - # When true, persistent connections will be allowed. Generally this is not - # useful for a public tracker, but helps performance in some cases (use of - # a reverse proxy, or when there are few clients issuing many requests). - enable_keepalive: false - idle_timeout: 30s - - # Whether to time requests. - # Disabling this should increase performance/decrease load. - enable_request_timing: false - - # An array of routes to listen on for announce requests. This is an option - # to support trackers that do not listen for /announce or need to listen - # on multiple routes. - # - # This supports named parameters and catch-all parameters as described at - # https://github.com/julienschmidt/httprouter#named-parameters - announce_routes: - - "/announce" - # - "/announce.php" - - # An array of routes to listen on for scrape requests. This is an option - # to support trackers that do not listen for /scrape or need to listen - # on multiple routes. - # - # This supports named parameters and catch-all parameters as described at - # https://github.com/julienschmidt/httprouter#named-parameters - scrape_routes: - - "/scrape" - # - "/scrape.php" - - # An array of routes to listen ping requests. - # Used just to ensure if server is operational. Returns nothing, - # just HTTP 200 without body. Listens both GET and HEAD HTTP methods. - # HEAD method just checks http server, GET checks all hooks, - # which support ping - ping_routes: - - "/ping" - - # When not enabled, tracker will use only address from which client connected to tracker. - # When enabled, the IP address that clients advertise as their IP address will - # be appended as announce candidate. - allow_ip_spoofing: false - - # When enabled, IPs from private, local and loopback subnets will be ignored - filter_private_ips: false - - # The HTTP Header containing the IP address of the client. - # This is only necessary if using a reverse proxy. - real_ip_header: "x-real-ip" - - # The maximum number of peers returned for an individual request. - max_numwant: 100 - - # The default number of peers returned for an individual request. - default_numwant: 50 - - # The maximum number of infohashes that can be scraped in one request. - max_scrape_infohashes: 50 - - # This block defines configuration for the tracker's UDP interface. - # If you do not wish to run this, delete this section. - udp: - # The network interface that will bind to a UDP server for serving - # BitTorrent traffic. - addr: "0.0.0.0:6969" - - # Enable SO_REUSEPORT to allow starting multiple mochi instances with the same UDP port. - reuse_port: true - - # The leeway for a timestamp on a connection ID. - max_clock_skew: 10s - - # The key used to encrypt connection IDs. - private_key: "paste a random string here that will be used to hmac connection IDs" - - # Whether to time requests. - # Disabling this should increase performance/decrease load. - enable_request_timing: false - - # When not enabled, tracker will use only address from which client connected to tracker. - # When enabled, the IP address that clients advertise as their IP address will - # be appended as announce candidate. - allow_ip_spoofing: false - - # When enabled, IPs from private, local and loopback subnets will be ignored - filter_private_ips: false - - # The maximum number of peers returned for an individual request. - max_numwant: 100 - - # The default number of peers returned for an individual request. - default_numwant: 50 - - # The maximum number of infohashes that can be scraped in one request. - max_scrape_infohashes: 50 - - - # This block defines configuration used for the storage of peer data. - storage: - name: memory + - name: http config: - # The frequency which stale peers are removed. - # This balances between - # - collecting garbage more often, potentially using more CPU time, but potentially using less memory (lower value) - # - collecting garbage less frequently, saving CPU time, but keeping old peers long, thus using more memory (higher value). - gc_interval: 3m + # The network interface that will bind to an HTTP server for serving + # BitTorrent traffic. Remove this to disable the non-TLS listener. + addr: "0.0.0.0:6969" - # The amount of time until a peer is considered stale. - # To avoid churn, keep this slightly larger than `announce_interval` - peer_lifetime: 31m + # Mark this frontend as HTTPS server for serving + # BitTorrent traffic. If set, tls_cert_path and tls_key_path are required. + tls: false - # The number of partitions data will be divided into in order to provide a - # higher degree of parallelism. - shard_count: 1024 + # The path to the required files to listen via HTTPS. + tls_cert_path: "" + tls_key_path: "" - # The interval at which metrics about the number of infohashes and peers - # are collected and posted to Prometheus. - prometheus_reporting_interval: 1s + # Enable SO_REUSEPORT to allow starting multiple mochi instances with the same HTTP(S) port. + reuse_port: true - # This block defines configuration used for redis storage. - #storage: - #name: redis - #config: - # The frequency which stale peers are removed. - # This balances between - # - collecting garbage more often, potentially using more CPU time, but potentially using less memory (lower value) - # - collecting garbage less frequently, saving CPU time, but keeping old peers long, thus using more memory (higher value). - #gc_interval: 3m + # The timeout durations for HTTP requests. + read_timeout: 5s + write_timeout: 5s - # The interval at which metrics about the number of infohashes and peers - # are collected and posted to Prometheus. - #prometheus_reporting_interval: 1s + # When true, persistent connections will be allowed. Generally this is not + # useful for a public tracker, but helps performance in some cases (use of + # a reverse proxy, or when there are few clients issuing many requests). + enable_keepalive: false + idle_timeout: 30s - # The amount of time until a peer is considered stale. - # To avoid churn, keep this slightly larger than `announce_interval` - #peer_lifetime: 31m + # Whether to time requests. + # Disabling this should increase performance/decrease load. + enable_request_timing: false - # The addresses of redis storage. - # If neither sentinel not cluster switched, - # only first address used - #addresses: ["127.0.0.1:6379"] + # An array of routes to listen on for announce requests. This is an option + # to support trackers that do not listen for /announce or need to listen + # on multiple routes. + # + # This supports named parameters and catch-all parameters as described at + # https://github.com/julienschmidt/httprouter#named-parameters + announce_routes: + - "/announce" + # - "/announce.php" - # Database to be selected after connecting to the server. - #db: 0 + # An array of routes to listen on for scrape requests. This is an option + # to support trackers that do not listen for /scrape or need to listen + # on multiple routes. + # + # This supports named parameters and catch-all parameters as described at + # https://github.com/julienschmidt/httprouter#named-parameters + scrape_routes: + - "/scrape" + # - "/scrape.php" - # Maximum number of socket connections, default is 10 per CPU - #pool_size: 10 + # An array of routes to listen ping requests. + # Used just to ensure if server is operational. Returns nothing, + # just HTTP 200 without body. Listens both GET and HEAD HTTP methods. + # HEAD method just checks http server, GET checks all hooks, + # which support ping + ping_routes: + - "/ping" - # Use the specified login/username to authenticate the current connection - #login: "" + # When not enabled, tracker will use only address from which client connected to tracker. + # When enabled, the IP address that clients advertise as their IP address will + # be appended as announce candidate. + allow_ip_spoofing: false - # Optional password - #password: "" + # When enabled, IPs from private, local and loopback subnets will be ignored + filter_private_ips: false - # Connect to sentinel nodes - #sentinel: false + # The HTTP Header containing the IP address of the client. + # This is only necessary if using a reverse proxy. + real_ip_header: "x-real-ip" - # The master name - #sentinel_master: "" + # The maximum number of peers returned for an individual request. + max_numwant: 100 - # Connect to the redis cluster - #cluster: false + # The default number of peers returned for an individual request. + default_numwant: 50 - # The timeout for reading a command reply from redis. - #read_timeout: 15s + # The maximum number of infohashes that can be scraped in one request. + max_scrape_infohashes: 50 - # The timeout for writing a command to redis. - #write_timeout: 15s + # This block defines configuration for the tracker's UDP interface. + # If you do not wish to run this, delete this section. + - name: udp + config: + # The network interface that will bind to a UDP server for serving + # BitTorrent traffic. + addr: "0.0.0.0:6969" - # Dial timeout for establishing new connections. - #connect_timeout: 15s + # Enable SO_REUSEPORT to allow starting multiple mochi instances with the same UDP port. + reuse_port: true - # This block defines configuration used for PostgreSQL storage. - # example `mo_peers` table structure: - # - info_hash bytea - # - peer_id bytea - # - address inet or bytea - # - port int4 - # - is_seeder bool - # - is_v6 bool - # - created timestamp - #storage: - #name: pg - #config: - # connection string to pg storage. may be URL (postgres://...) or DSN (host=... port=...) - #connection_string: host=127.0.0.1 database=test user=postgres pool_max_conns=50 - # query and parameters for announce operation - #announce: - #query: SELECT peer_id, address, port FROM mo_peers WHERE info_hash=$1 AND is_seeder=$2 AND is_v6=$3 LIMIT $4 - #peer_id_column: peer_id - #address_column: address - #port_column: port - #peer: - # expected parameters: 1 - info hash (bytea), 2 - peer id (bytea), 3 - ip address (bytea/inet) - # 4 - port (int), 5 - is seeder (bool), 6 - is IPv6 (bool), 7 - create date and time (timestamp) - #add_query: INSERT INTO mo_peers VALUES($1, $2, $3, $4, $5, $6, $7) ON CONFLICT (info_hash, peer_id, address, port) DO UPDATE SET created = EXCLUDED.created, is_seeder = EXCLUDED.is_seeder - #del_query: DELETE FROM mo_peers WHERE info_hash=$1 AND peer_id=$2 AND address=$3 AND port=$4 AND is_seeder=$5 - #graduate_query: UPDATE mo_peers SET is_seeder=TRUE WHERE info_hash=$1 AND peer_id=$2 AND address=$3 AND port=$4 AND NOT is_seeder - #count_query: SELECT COUNT(1) FILTER (WHERE is_seeder) AS seeders, COUNT(1) FILTER (WHERE NOT is_seeder) AS leechers FROM mo_peers - # predicate part of `count_query` for get count of peers by info hash - #by_info_hash_clause: WHERE info_hash = $1 - #count_seeders_column: seeders - #count_leechers_column: leechers - # queries for KV-store - #data: - # expected parameters: 1 - context (varchar), 2 - name (bytea), 3 - value (bytea) - #add_query: INSERT INTO mo_kv VALUES($1, $2, $3) ON CONFLICT (context, name) DO NOTHING - #del_query: DELETE FROM mo_kv WHERE context=$1 AND name=$2 - #get_query: SELECT value FROM mo_kv WHERE context=$1 AND name=$2 - # query for check if database is alive - #ping_query: SELECT 1 - # query for garbage collection, expected parameter is timestamp - #gc_query: DELETE FROM mo_peers WHERE created <= $1 - # The amount of time until a peer is considered stale. - # To avoid churn, keep this slightly larger than `announce_interval` - #peer_lifetime: 31m - # The frequency which stale peers are removed. - #gc_interval: 3m - # query for info hash statistics - #info_hash_count_query: SELECT COUNT(DISTINCT info_hash) as info_hashes FROM mo_peers - # The interval at which metrics about the number of info hashes and peers - # are collected and posted to Prometheus. - #prometheus_reporting_interval: 1s + # The leeway for a timestamp on a connection ID. + max_clock_skew: 10s - # This block defines configuration used for middleware executed before a - # response has been returned to a BitTorrent client. - prehooks: - # - name: jwt - # options: - # header: "authorization" - # issuer: "https://issuer.com" - # audience: "https://some.issuer.com" - # jwk_set_url: "https://issuer.com/keys" - # jwk_set_update_interval: 5m - # handle_announce: true - # handle_scrape: false - # - # - name: client approval - # options: - # whitelist: - # - "OP1011" - # blacklist: - # - "OP1012" - # - # - name: interval variation - # options: - # modify_response_probability: 0.2 - # max_increase_delta: 60 - # modify_min_interval: true - # - # This block defines configuration used for torrent approval, it requires to be given - # hashes for whitelist or for blacklist. Hashes are hexadecimal-encoaded. - # - name: torrent approval - # options: - # initial_source: list - # Save data provided by source in storage above - # preserve: false - # configuration: - # hash_list: - # - "a1b2c3d4e5a1b2c3d4e5a1b2c3d4e5a1b2c3d4e5" - # true - whitelist mode, false - blacklist - # invert: false - # Name of storage context where store hash list - # storage_ctx: APPROVED_HASH - posthooks: + # The key used to encrypt connection IDs. + private_key: "paste a random string here that will be used to hmac connection IDs" + + # Whether to time requests. + # Disabling this should increase performance/decrease load. + enable_request_timing: false + + # When not enabled, tracker will use only address from which client connected to tracker. + # When enabled, the IP address that clients advertise as their IP address will + # be appended as announce candidate. + allow_ip_spoofing: false + + # When enabled, IPs from private, local and loopback subnets will be ignored + filter_private_ips: false + + # The maximum number of peers returned for an individual request. + max_numwant: 100 + + # The default number of peers returned for an individual request. + default_numwant: 50 + + # The maximum number of infohashes that can be scraped in one request. + max_scrape_infohashes: 50 + + +# This block defines configuration used for the storage of peer data. +storage: + name: memory + config: + # The frequency which stale peers are removed. + # This balances between + # - collecting garbage more often, potentially using more CPU time, but potentially using less memory (lower value) + # - collecting garbage less frequently, saving CPU time, but keeping old peers long, thus using more memory (higher value). + gc_interval: 3m + + # The amount of time until a peer is considered stale. + # To avoid churn, keep this slightly larger than `announce_interval` + peer_lifetime: 31m + + # The number of partitions data will be divided into in order to provide a + # higher degree of parallelism. + shard_count: 1024 + + # The interval at which metrics about the number of infohashes and peers + # are collected and posted to Prometheus. + prometheus_reporting_interval: 1s + +# This block defines configuration used for middleware executed before a +# response has been returned to a BitTorrent client. +posthooks: [] +prehooks: +# - name: jwt +# config: +# header: "authorization" +# issuer: "https://issuer.com" +# audience: "https://some.issuer.com" +# jwk_set_url: "https://issuer.com/keys" +# jwk_set_update_interval: 5m +# handle_announce: true +# handle_scrape: false +# +# - name: client approval +# config: +# whitelist: +# - "OP1011" +# blacklist: +# - "OP1012" +# +# - name: interval variation +# config: +# modify_response_probability: 0.2 +# max_increase_delta: 60 +# modify_min_interval: true +# +# This block defines configuration used for torrent approval, it requires to be given +# hashes for whitelist or for blacklist. Hashes are hexadecimal-encoaded. +# - name: torrent approval +# config: +# initial_source: list +# Save data provided by source in storage above +# preserve: false +# configuration: +# hash_list: +# - "a1b2c3d4e5a1b2c3d4e5a1b2c3d4e5a1b2c3d4e5" +# true - whitelist mode, false - blacklist +# invert: false +# Name of storage context where store hash list +# storage_ctx: APPROVED_HASH diff --git a/dist/example_config_pg.yaml b/dist/example_config_pg.yaml new file mode 100644 index 0000000..26647f7 --- /dev/null +++ b/dist/example_config_pg.yaml @@ -0,0 +1,116 @@ +# @formatter:off +# Note: see `example_config.yaml` for `frontends` and `*hooks` config description + + +announce_interval: 30m +min_announce_interval: 15m +metrics_addr: "" + +frontends: + - name: http + config: + addr: "0.0.0.0:6969" + tls: false + tls_cert_path: "" + tls_key_path: "" + reuse_port: true + read_timeout: 5s + write_timeout: 5s + enable_keepalive: false + idle_timeout: 30s + enable_request_timing: false + announce_routes: + - "/announce" + scrape_routes: + - "/scrape" + ping_routes: + - "/ping" + allow_ip_spoofing: false + filter_private_ips: false + real_ip_header: "x-real-ip" + max_numwant: 100 + default_numwant: 50 + max_scrape_infohashes: 50 + + - name: udp + config: + addr: "0.0.0.0:6969" + reuse_port: true + max_clock_skew: 10s + private_key: "paste a random string here that will be used to hmac connection IDs" + enable_request_timing: false + allow_ip_spoofing: false + filter_private_ips: false + max_numwant: 100 + default_numwant: 50 + max_scrape_infohashes: 50 + +# This block defines configuration used for PostgreSQL storage. +# example peers table structure: +# - info_hash bytea +# - peer_id bytea +# - address inet or bytea +# - port int4 +# - is_seeder bool +# - is_v6 bool +# - created timestamp +# example downloads table structure: +# - info_hash bytea +# - downloads int +storage: + name: pg + config: + # connection string to pg storage. may be URL (postgres://...) or DSN (host=... port=...) + connection_string: host=127.0.0.1 database=test user=postgres pool_max_conns=50 + # query and parameters for announce operation + announce: + query: SELECT peer_id, address, port FROM mo_peers WHERE info_hash=@info_hash AND is_seeder=@is_seeder AND is_v6=@is_v6 LIMIT @count + peer_id_column: peer_id + address_column: address + port_column: port + + # queries to get/increment 'snatched' (downloaded) count + downloads: + get_query: SELECT downloads FROM mo_downloads where info_hash=@info_hash + inc_query: INSERT INTO mo_downloads VALUES(@info_hash) ON CONFLICT(info_hash) DO UPDATE SET downloads = mo_downloads.downloads + 1 + + # queries and parameters for add/delete/count peers operations + peer: + add_query: INSERT INTO mo_peers VALUES(@info_hash, @peer_id, @address, @port, @is_seeder, @is_v6, @created) ON CONFLICT (info_hash, peer_id, address, port) DO UPDATE SET created = EXCLUDED.created, is_seeder = EXCLUDED.is_seeder + del_query: DELETE FROM mo_peers WHERE info_hash=@info_hash AND peer_id=@peer_id AND address=@address AND port=@port AND is_seeder=@is_seeder + graduate_query: UPDATE mo_peers SET is_seeder=TRUE WHERE info_hash=@info_hash AND peer_id=peer_id AND address=@address AND port=@port AND NOT is_seeder + count_query: SELECT COUNT(1) FILTER (WHERE is_seeder) AS seeders, COUNT(1) FILTER (WHERE NOT is_seeder) AS leechers FROM mo_peers + # predicate part of `count_query` to get count of peers by info hash + by_info_hash_clause: WHERE info_hash = @info_hash + count_seeders_column: seeders + count_leechers_column: leechers + + # queries for KV-store + data: + add_query: INSERT INTO mo_kv VALUES(@context, @key, @value) ON CONFLICT (context, name) DO NOTHING + # Note: in del_query @key parameter is array, NOT single value + del_query: DELETE FROM mo_kv WHERE context=@context AND name = ANY(@key) + get_query: SELECT value FROM mo_kv WHERE context=@context AND name=@key + + # query for check if database is alive + ping_query: SELECT 1 + + # query for garbage collection, expected parameter is timestamp + gc_query: DELETE FROM mo_peers WHERE created <= @created + + # The amount of time until a peer is considered stale. + # To avoid churn, keep this slightly larger than `announce_interval` + peer_lifetime: 31m + + # The frequency which stale peers are removed. + gc_interval: 3m + + # query for info hash statistics + info_hash_count_query: SELECT COUNT(DISTINCT info_hash) as info_hashes FROM mo_peers + + # The interval at which metrics about the number of info hashes and peers + # are collected and posted to Prometheus. + prometheus_reporting_interval: 1s + +posthooks: [] +prehooks: [] diff --git a/dist/example_config_redis.yaml b/dist/example_config_redis.yaml new file mode 100644 index 0000000..792e047 --- /dev/null +++ b/dist/example_config_redis.yaml @@ -0,0 +1,103 @@ +# @formatter:off +# Note: see `example_config.yaml` for `frontends` and `*hooks` config description + + +announce_interval: 30m +min_announce_interval: 15m +metrics_addr: "" + +frontends: + - name: http + config: + addr: "0.0.0.0:6969" + tls: false + tls_cert_path: "" + tls_key_path: "" + reuse_port: true + read_timeout: 5s + write_timeout: 5s + enable_keepalive: false + idle_timeout: 30s + enable_request_timing: false + announce_routes: + - "/announce" + scrape_routes: + - "/scrape" + ping_routes: + - "/ping" + allow_ip_spoofing: false + filter_private_ips: false + real_ip_header: "x-real-ip" + max_numwant: 100 + default_numwant: 50 + max_scrape_infohashes: 50 + + - name: udp + config: + addr: "0.0.0.0:6969" + reuse_port: true + max_clock_skew: 10s + private_key: "paste a random string here that will be used to hmac connection IDs" + enable_request_timing: false + allow_ip_spoofing: false + filter_private_ips: false + max_numwant: 100 + default_numwant: 50 + max_scrape_infohashes: 50 + +# This block defines configuration used for redis storage. +storage: + # If used keydb fork, set `keydb` name + name: redis + config: + # The frequency which stale peers are removed. + # This balances between + # - collecting garbage more often, potentially using more CPU time, but potentially using less memory (lower value) + # - collecting garbage less frequently, saving CPU time, but keeping old peers long, thus using more memory (higher value). + gc_interval: 3m + + # The interval at which metrics about the number of infohashes and peers + # are collected and posted to Prometheus. + prometheus_reporting_interval: 1s + + # The amount of time until a peer is considered stale. + # To avoid churn, keep this slightly larger than `announce_interval` + peer_lifetime: 31m + + # The addresses of redis storage. + # If neither sentinel not cluster switched, + # only first address used + addresses: ["127.0.0.1:6379"] + + # Database to be selected after connecting to the server. + db: 0 + + # Maximum number of socket connections, default is 10 per CPU + pool_size: 10 + + # Use the specified login/username to authenticate the current connection + login: "" + + # Optional password + password: "" + + # Connect to sentinel nodes + sentinel: false + + # The master name + sentinel_master: "" + + # Connect to the redis cluster + cluster: false + + # The timeout for reading a command reply from redis. + read_timeout: 15s + + # The timeout for writing a command to redis. + write_timeout: 15s + + # Dial timeout for establishing new connections. + connect_timeout: 15s + +posthooks: [] +prehooks: [] \ No newline at end of file diff --git a/docs/storage/postgres.md b/docs/storage/postgres.md index 143baec..06411fe 100644 --- a/docs/storage/postgres.md +++ b/docs/storage/postgres.md @@ -34,17 +34,20 @@ you provide in configuration. Implementation expects next data types: * Table for peers: - * info hash - byte array (`bytea`) - * peer ID - byte array (`bytea`) - * peer address - `inet` or byte array (`bytea`) - * peer port - integer or derivative type (`int4`, `integer`) - * is seeder - boolean (`bool`) - * is IPv6 - boolean (`bool`) - * peer creation date and time - `timestamp` + * info hash - byte array (`bytea`) + * peer ID - byte array (`bytea`) + * peer address - `inet` or byte array (`bytea`) + * peer port - integer or derivative type (`int4`, `integer`) + * is seeder - boolean (`bool`) + * is IPv6 - boolean (`bool`) + * peer creation date and time - `timestamp` +* Table of download counts (optional) + * info hash - byte array (`bytea`) + * count - or derivative type (`int4`, `integer`) * Table for arbitrary data (KV store): - * context - string (`varchar`, `character varying`) - * name - byte array (`bytea`)* - * value - byte array (`bytea`) + * context - string (`varchar`, `character varying`) + * name - byte array (`bytea`)* + * value - byte array (`bytea`) (*) in KV table `name` present as byte array because of possibility to place hash as _raw_ string, which is not supported by PostgreSQL. @@ -67,6 +70,11 @@ CREATE TABLE mo_peers CREATE INDEX mo_peers_created_idx ON mo_peers (created); CREATE INDEX mo_peers_announce_idx ON mo_peers (info_hash, is_seeder, is_v6); +CREATE TABLE mo_downloads ( + info_hash bytea PRIMARY KEY NOT NULL, + downloads int NOT NULL DEFAULT 1 +); + CREATE TABLE mo_kv ( context varchar NOT NULL, @@ -98,49 +106,54 @@ storage: peer: # Query to add peer info. # Expected arguments: - # 1 - info hash (bytea), - # 2 - peer id (bytea), - # 3 - ip address (bytea/inet) + # 1 - info_hash (bytea), + # 2 - peer_id (bytea), + # 3 - address (bytea/inet) # 4 - port (int4), - # 5 - is seeder (bool), - # 6 - is IPv6 (bool), - # 7 - create date and time (timestamp) + # 5 - is_seeder (bool), + # 6 - is_v6 (bool), + # 7 - created (timestamp) # Query MUST handle situations when tuple # `info hash - peer ID - address - port` is already # exists in table - add_query: INSERT INTO mo_peers VALUES($1, $2, $3, $4, $5, $6, $7) ON CONFLICT (info_hash, peer_id, address, port) DO UPDATE SET created = EXCLUDED.created, is_seeder = EXCLUDED.is_seeder + add_query: INSERT INTO mo_peers VALUES(@info_hash, @peer_id, @address, @port, @is_seeder, @is_v6, @created) ON CONFLICT (info_hash, peer_id, address, port) DO UPDATE SET created = EXCLUDED.created, is_seeder = EXCLUDED.is_seeder # Query to delete peer info. - # Query SHOULD take into account value of `is seeder` flag - del_query: DELETE FROM mo_peers WHERE info_hash=$1 AND peer_id=$2 AND address=$3 AND port=$4 AND is_seeder=$5 + # Query SHOULD take into account value of `is_seeder` flag + del_query: DELETE FROM mo_peers WHERE info_hash=@info_hash AND peer_id=@peer_id AND address=@address AND port=@port AND is_seeder=@is_seeder # Query to update leecher to seeder - graduate_query: UPDATE mo_peers SET is_seeder=TRUE WHERE info_hash=$1 AND peer_id=$2 AND address=$3 AND port=$4 AND NOT is_seeder + graduate_query: UPDATE mo_peers SET is_seeder=TRUE WHERE info_hash=@info_hash AND peer_id=peer_id AND address=@address AND port=@port AND NOT is_seeder # Query to get count of peers. # Used both for statistics and for scrape (with clause suffix, see next). # Only first returned row value used. count_query: SELECT COUNT(1) FILTER (WHERE is_seeder) AS seeders, COUNT(1) FILTER (WHERE NOT is_seeder) AS leechers FROM mo_peers # Predicate part of `count_query` for get count of peers by info hash - by_info_hash_clause: WHERE info_hash = $1 + by_info_hash_clause: WHERE info_hash = @info_hash # Column name of seeders count in `count_query` (case-insensitive). count_seeders_column: seeders # Column name of leechers count in `count_query` (case-insensitive). count_leechers_column: leechers + # Queries to get/increment 'snatched' (downloaded) count + downloads: + get_query: SELECT downloads FROM mo_downloads where info_hash=@info_hash + inc_query: INSERT INTO mo_downloads VALUES(@info_hash) ON CONFLICT(info_hash) DO UPDATE SET downloads = mo_downloads.downloads + 1 # Queries for KV-store data: # Query to add data. # Expected arguments: # 1 - context (varchar), - # 2 - name (bytea), + # 2 - key (bytea), # 3 - value (bytea) - add_query: INSERT INTO mo_kv VALUES($1, $2, $3) ON CONFLICT (context, name) DO NOTHING + add_query: INSERT INTO mo_kv VALUES(@context, @key, @value) ON CONFLICT (context, name) DO NOTHING # Query to delete data. - del_query: DELETE FROM mo_kv WHERE context=$1 AND name=$2 + # Note: @key parameter is array, NOT single value + del_query: DELETE FROM mo_kv WHERE context=@context AND name = ANY(@key) # Query to get data. # Only first returned row and column value used. - get_query: SELECT value FROM mo_kv WHERE context=$1 AND name=$2 + get_query: SELECT value FROM mo_kv WHERE context=@context AND name=@key # Query for check if database is alive (can be omitted) ping_query: SELECT 1 # Query to delete stale peers (peers, which timestamp older than provided argument) - gc_query: DELETE FROM mo_peers WHERE created <= $1 + gc_query: DELETE FROM mo_peers WHERE created <= @created # The frequency which stale peers are removed. gc_interval: 3m # Query to get all info hash count (used for statistics). diff --git a/docs/storage/redis.md b/docs/storage/redis.md index 9a9fd3c..908a097 100644 --- a/docs/storage/redis.md +++ b/docs/storage/redis.md @@ -69,6 +69,8 @@ Here is an example: - : - CHI_L4_ - : +- CHI_D (hash type) + - : ... ``` diff --git a/frontend/frontend.go b/frontend/frontend.go index e1d5b82..e871d96 100644 --- a/frontend/frontend.go +++ b/frontend/frontend.go @@ -3,34 +3,71 @@ package frontend import ( - "context" + "fmt" + "sync" - "github.com/sot-tech/mochi/bittorrent" + "github.com/sot-tech/mochi/middleware" + "github.com/sot-tech/mochi/pkg/conf" + "github.com/sot-tech/mochi/pkg/log" + "github.com/sot-tech/mochi/pkg/stop" ) -// TrackerLogic is the interface used by a frontend in order to: (1) generate a -// response from a parsed request, and (2) asynchronously observe anything -// after the response has been delivered to the client. -type TrackerLogic interface { - // HandleAnnounce generates a response for an Announce. - // - // Returns the updated context, the generated AnnounceResponse and no error - // on success; nil and error on failure. - HandleAnnounce(context.Context, *bittorrent.AnnounceRequest) (context.Context, *bittorrent.AnnounceResponse, error) +var ( + logger = log.NewLogger("frontend") + buildersMU sync.RWMutex + builders = make(map[string]Builder) +) - // AfterAnnounce does something with the results of an Announce after it - // has been completed. - AfterAnnounce(context.Context, *bittorrent.AnnounceRequest, *bittorrent.AnnounceResponse) +// Builder is the function used to initialize a new Frontend +// with provided configuration. +type Builder func(conf.MapConfig, *middleware.Logic) (Frontend, error) - // HandleScrape generates a response for a Scrape. - // - // Returns the updated context, the generated AnnounceResponse and no error - // on success; nil and error on failure. - HandleScrape(context.Context, *bittorrent.ScrapeRequest) (context.Context, *bittorrent.ScrapeResponse, error) +// RegisterBuilder makes a Builder available by the provided name. +// +// If called twice with the same name, the name is blank, or if the provided +// Builder is nil, this function panics. +func RegisterBuilder(name string, b Builder) { + if name == "" { + panic("frontend: could not register Builder with an empty name") + } + if b == nil { + panic("frontend: could not register a nil Builder") + } - // AfterScrape does something with the results of a Scrape after it has been completed. - AfterScrape(context.Context, *bittorrent.ScrapeRequest, *bittorrent.ScrapeResponse) + buildersMU.Lock() + defer buildersMU.Unlock() - // Ping executes checks if all hooks are operational - Ping(context.Context) error + if _, dup := builders[name]; dup { + panic("frontend: RegisterBuilder called twice for " + name) + } + + builders[name] = b +} + +// Frontend dummy interface for bittorrent frontends +type Frontend interface { + stop.Stopper +} + +// NewFrontends is a utility function for initializing Frontend-s in bulk. +// Returns nil hook and error if frontend with name provided in config +// does not exists. +func NewFrontends(configs []conf.NamedMapConfig, logic *middleware.Logic) (fs []Frontend, err error) { + buildersMU.RLock() + defer buildersMU.RUnlock() + for _, c := range configs { + logger.Debug().Object("frontend", c).Msg("starting frontend") + newFrontend, ok := builders[c.Name] + if !ok { + err = fmt.Errorf("hook with name '%s' does not exists", c.Name) + break + } + var f Frontend + if f, err = newFrontend(c.Config, logic); err != nil { + break + } + fs = append(fs, f) + logger.Info().Str("name", c.Name).Msg("frontend started") + } + return } diff --git a/frontend/http/frontend.go b/frontend/http/frontend.go index 3519ad6..be35a39 100644 --- a/frontend/http/frontend.go +++ b/frontend/http/frontend.go @@ -6,266 +6,159 @@ import ( "context" "crypto/tls" "errors" - "net" "net/http" "net/netip" - "sync" "time" "github.com/julienschmidt/httprouter" - "github.com/libp2p/go-reuseport" "github.com/sot-tech/mochi/bittorrent" "github.com/sot-tech/mochi/frontend" + "github.com/sot-tech/mochi/middleware" "github.com/sot-tech/mochi/pkg/conf" "github.com/sot-tech/mochi/pkg/log" "github.com/sot-tech/mochi/pkg/metrics" "github.com/sot-tech/mochi/pkg/stop" ) -var logger = log.NewLogger("http frontend") +var ( + logger = log.NewLogger("frontend/http") + errTLSNotProvided = errors.New("tls certificate/key not provided") + errRoutesNotProvided = errors.New("routes not provided") +) -// Config represents all of the configurable options for an HTTP BitTorrent -// Frontend. +func init() { + frontend.RegisterBuilder("http", NewFrontend) +} + +// Config represents all configurable options for an HTTP BitTorrent Frontend type Config struct { - Addr string - HTTPSAddr string `cfg:"https_addr"` - ReadTimeout time.Duration `cfg:"read_timeout"` - WriteTimeout time.Duration `cfg:"write_timeout"` - IdleTimeout time.Duration `cfg:"idle_timeout"` - EnableKeepAlive bool `cfg:"enable_keepalive"` - TLSCertPath string `cfg:"tls_cert_path"` - TLSKeyPath string `cfg:"tls_key_path"` - ReusePort bool `cfg:"reuse_port"` - AnnounceRoutes []string `cfg:"announce_routes"` - ScrapeRoutes []string `cfg:"scrape_routes"` - PingRoutes []string `cfg:"ping_routes"` - EnableRequestTiming bool `cfg:"enable_request_timing"` + frontend.ListenOptions + IdleTimeout time.Duration `cfg:"idle_timeout"` + EnableKeepAlive bool `cfg:"enable_keepalive"` + UseTLS bool `cfg:"tls"` + TLSCertPath string `cfg:"tls_cert_path"` + TLSKeyPath string `cfg:"tls_key_path"` + AnnounceRoutes []string `cfg:"announce_routes"` + ScrapeRoutes []string `cfg:"scrape_routes"` + PingRoutes []string `cfg:"ping_routes"` ParseOptions } -// Default config constants. -const ( - defaultReadTimeout = 2 * time.Second - defaultWriteTimeout = 2 * time.Second - defaultIdleTimeout = 30 * time.Second -) +const defaultIdleTimeout = 30 * time.Second // Validate sanity checks values set in a config and returns a new config with // default values replacing anything that is invalid. -// -// This function warns to the logger when a value is changed. -func (cfg Config) Validate() Config { - validcfg := cfg - - if cfg.ReadTimeout <= 0 { - validcfg.ReadTimeout = defaultReadTimeout - logger.Warn(). - Str("name", "http.ReadTimeout"). - Dur("provided", cfg.ReadTimeout). - Dur("default", validcfg.ReadTimeout). - Msg("falling back to default configuration") +func (cfg Config) Validate() (validCfg Config, err error) { + validCfg = cfg + if validCfg.ListenOptions, err = cfg.ListenOptions.Validate(); err != nil { + return } - - if cfg.WriteTimeout <= 0 { - validcfg.WriteTimeout = defaultWriteTimeout - logger.Warn(). - Str("name", "http.WriteTimeout"). - Dur("provided", cfg.WriteTimeout). - Dur("default", validcfg.WriteTimeout). - Msg("falling back to default configuration") + if cfg.UseTLS && (len(cfg.TLSCertPath) == 0 || len(cfg.TLSKeyPath) == 0) { + err = errTLSNotProvided + return } - if cfg.IdleTimeout <= 0 { - validcfg.IdleTimeout = defaultIdleTimeout - + validCfg.IdleTimeout = defaultIdleTimeout if cfg.EnableKeepAlive { // If keepalive is disabled, this configuration isn't used anyway. logger.Warn(). - Str("name", "http.IdleTimeout"). + Str("name", "IdleTimeout"). Dur("provided", cfg.IdleTimeout). - Dur("default", validcfg.IdleTimeout). + Dur("default", validCfg.IdleTimeout). Msg("falling back to default configuration") } } - - validcfg.ParseOptions.ParseOptions = cfg.ParseOptions.ParseOptions.Validate() - - return validcfg + validCfg.ParseOptions.ParseOptions = cfg.ParseOptions.ParseOptions.Validate() + return } -// Frontend represents the state of an HTTP BitTorrent Frontend. -type Frontend struct { - srv *http.Server - srvMu sync.Mutex - tlsSrv *http.Server - tlsSrvMu sync.Mutex - tlsCfg *tls.Config - - logic frontend.TrackerLogic - Config +type httpFE struct { + srv *http.Server + logic *middleware.Logic + collectTimings bool + ParseOptions } -// NewFrontend creates a new instance of an HTTP Frontend that asynchronously -// serves requests. -func NewFrontend(logic frontend.TrackerLogic, c conf.MapConfig) (*Frontend, error) { - var provided Config - if err := c.Unmarshal(&provided); err != nil { - return nil, err +// NewFrontend builds and starts http bittorrent frontend from provided configuration +func NewFrontend(c conf.MapConfig, logic *middleware.Logic) (_ frontend.Frontend, err error) { + var cfg Config + if err = c.Unmarshal(&cfg); err != nil { + return } - cfg := provided.Validate() - - f := &Frontend{ - logic: logic, - Config: cfg, - srvMu: sync.Mutex{}, - tlsSrvMu: sync.Mutex{}, + if cfg, err = cfg.Validate(); err != nil { + return } - - if cfg.Addr == "" && cfg.HTTPSAddr == "" { - return nil, errors.New("must specify addr or https_addr or both") - } - if len(cfg.AnnounceRoutes) < 1 || len(cfg.ScrapeRoutes) < 1 { - return nil, errors.New("must specify routes") + err = errRoutesNotProvided + return + } + + f := &httpFE{ + logic: logic, + srv: &http.Server{ + ReadTimeout: cfg.ReadTimeout, + ReadHeaderTimeout: cfg.ReadTimeout, + WriteTimeout: cfg.WriteTimeout, + IdleTimeout: cfg.IdleTimeout, + }, + collectTimings: cfg.EnableRequestTiming, + ParseOptions: cfg.ParseOptions, } // If TLS is enabled, create a key pair. - if cfg.TLSCertPath != "" && cfg.TLSKeyPath != "" { - var err error - f.tlsCfg = &tls.Config{ - Certificates: make([]tls.Certificate, 1), + if cfg.UseTLS { + var cert tls.Certificate + if cert, err = tls.LoadX509KeyPair(cfg.TLSCertPath, cfg.TLSKeyPath); err != nil { + return + } + f.srv.TLSConfig = &tls.Config{ + Certificates: []tls.Certificate{cert}, MinVersion: tls.VersionTLS12, } - f.tlsCfg.Certificates[0], err = tls.LoadX509KeyPair(cfg.TLSCertPath, cfg.TLSKeyPath) - if err != nil { - return nil, err - } - } - - if (cfg.HTTPSAddr == "") != (f.tlsCfg == nil) { - return nil, errors.New("must specify both https_addr, tls_cert_path and tls_key_path") } router := httprouter.New() - for _, route := range f.AnnounceRoutes { + for _, route := range cfg.AnnounceRoutes { router.GET(route, f.announceRoute) } - for _, route := range f.ScrapeRoutes { + for _, route := range cfg.ScrapeRoutes { router.GET(route, f.scrapeRoute) } + for _, route := range cfg.PingRoutes { + router.GET(route, f.ping) + router.HEAD(route, f.ping) + } + f.srv.Handler = router - if len(f.PingRoutes) > 0 { - for _, route := range f.PingRoutes { - router.GET(route, f.ping) - router.HEAD(route, f.ping) + f.srv.SetKeepAlivesEnabled(cfg.EnableKeepAlive) + + go func() { + ln, err := cfg.ListenTCP() + if err == nil { + if f.srv.TLSConfig == nil { + err = f.srv.Serve(ln) + } else { + err = f.srv.ServeTLS(ln, "", "") + } } - } - - if cfg.Addr != "" { - go func() { - if err := f.serveHTTP(router, false); err != nil { - logger.Fatal().Err(err).Str("proto", "http").Msg("failed while serving") - } - }() - } - - if cfg.HTTPSAddr != "" { - go func() { - if err := f.serveHTTP(router, true); err != nil { - logger.Fatal().Err(err).Str("proto", "https").Msg("failed while serving") - } - }() - } + if !errors.Is(err, http.ErrServerClosed) { + logger.Fatal().Err(err).Msg("server failed") + } + }() return f, nil } // Stop provides a thread-safe way to shut down a currently running Frontend. -func (f *Frontend) Stop() stop.Result { - stopGroup := stop.NewGroup() - - f.srvMu.Lock() +func (f *httpFE) Stop() stop.Result { + c := make(stop.Channel) if f.srv != nil { - stopGroup.AddFunc(f.makeStopFunc(f.srv)) - } - f.srvMu.Unlock() - - f.tlsSrvMu.Lock() - if f.tlsSrv != nil { - stopGroup.AddFunc(f.makeStopFunc(f.tlsSrv)) - } - f.tlsSrvMu.Unlock() - - return stopGroup.Stop() -} - -func (f *Frontend) makeStopFunc(stopSrv *http.Server) stop.Func { - return func() stop.Result { - c := make(stop.Channel) go func() { - c.Done(stopSrv.Shutdown(context.Background())) + c.Done(f.srv.Shutdown(context.Background())) }() - return c.Result() } -} - -// serveHTTP blocks while listening and serving non-TLS HTTP BitTorrent -// requests until Stop() is called or an error is returned. -func (f *Frontend) serveHTTP(handler http.Handler, tls bool) error { - srv := &http.Server{ - Handler: handler, - ReadTimeout: f.ReadTimeout, - ReadHeaderTimeout: f.ReadTimeout, - WriteTimeout: f.WriteTimeout, - IdleTimeout: f.IdleTimeout, - } - - srv.SetKeepAlivesEnabled(f.EnableKeepAlive) - - var err error - if tls { - if f.ReusePort { - var ln net.Listener - if ln, err = reuseport.Listen("tcp", f.HTTPSAddr); err == nil { - defer ln.Close() - srv.TLSConfig = f.tlsCfg - f.tlsSrvMu.Lock() - f.tlsSrv = srv - f.tlsSrvMu.Unlock() - err = srv.ServeTLS(ln, "", "") - } - } else { - srv.Addr = f.HTTPSAddr - srv.TLSConfig = f.tlsCfg - f.tlsSrvMu.Lock() - f.tlsSrv = srv - f.tlsSrvMu.Unlock() - err = srv.ListenAndServeTLS("", "") - } - } else { - if f.ReusePort { - var ln net.Listener - if ln, err = reuseport.Listen("tcp", f.Addr); err == nil { - defer ln.Close() - f.srvMu.Lock() - f.srv = srv - f.srvMu.Unlock() - err = srv.Serve(ln) - } - } else { - srv.Addr = f.Addr - f.srvMu.Lock() - f.srv = srv - f.srvMu.Unlock() - err = srv.ListenAndServe() - } - } - // Start the HTTP server. - if errors.Is(err, http.ErrServerClosed) { - err = nil - } - return err + return c.Result() } func injectRouteParamsToContext(ctx context.Context, ps httprouter.Params) context.Context { @@ -277,12 +170,12 @@ func injectRouteParamsToContext(ctx context.Context, ps httprouter.Params) conte } // announceRoute parses and responds to an Announce. -func (f *Frontend) announceRoute(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { +func (f *httpFE) announceRoute(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { var err error var start time.Time var addr netip.Addr var req *bittorrent.AnnounceRequest - if f.EnableRequestTiming && metrics.Enabled() { + if f.collectTimings && metrics.Enabled() { start = time.Now() defer func() { recordResponseDuration("announce", addr, err, time.Since(start)) @@ -314,11 +207,11 @@ func (f *Frontend) announceRoute(w http.ResponseWriter, r *http.Request, ps http } // scrapeRoute parses and responds to a Scrape. -func (f *Frontend) scrapeRoute(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { +func (f *httpFE) scrapeRoute(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { var err error var start time.Time var addr netip.Addr - if f.EnableRequestTiming && metrics.Enabled() { + if f.collectTimings && metrics.Enabled() { start = time.Now() defer func() { recordResponseDuration("scrape", addr, err, time.Since(start)) @@ -349,7 +242,7 @@ func (f *Frontend) scrapeRoute(w http.ResponseWriter, r *http.Request, ps httpro go f.logic.AfterScrape(ctx, req, resp) } -func (f *Frontend) ping(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { +func (f *httpFE) ping(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { var err error if r.Method == http.MethodGet { err = f.logic.Ping(context.TODO()) diff --git a/frontend/http/writer.go b/frontend/http/writer.go index 8b194be..3b6641c 100644 --- a/frontend/http/writer.go +++ b/frontend/http/writer.go @@ -1,16 +1,19 @@ package http import ( + "bytes" "errors" "net" "net/http" + "strconv" "time" - "github.com/anacrolix/torrent/bencode" - "github.com/sot-tech/mochi/bittorrent" + "github.com/sot-tech/mochi/pkg/bytepool" ) +var respBufferPool = bytepool.NewBufferPool() + // WriteError communicates an error to a BitTorrent client over HTTP. func WriteError(w http.ResponseWriter, err error) { message := "internal server error" @@ -20,93 +23,106 @@ func WriteError(w http.ResponseWriter, err error) { } else { logger.Error().Err(err).Msg("http: internal error") } - - if err = bencode.NewEncoder(w).Encode(map[string]any{ - "failure reason": message, - }); err != nil { - logger.Error().Err(err).Msg("unable to encode message") - } + _, _ = w.Write([]byte("d14:failure reason" + strconv.Itoa(len(message)) + ":" + message + "e")) } // WriteAnnounceResponse communicates the results of an Announce to a // BitTorrent client over HTTP. func WriteAnnounceResponse(w http.ResponseWriter, resp *bittorrent.AnnounceResponse) error { + bb := respBufferPool.Get() + defer respBufferPool.Put(bb) + if resp.Interval > 0 { resp.Interval /= time.Second } - if resp.Interval > 0 { resp.MinInterval /= time.Second } - bdict := map[string]any{ - "complete": resp.Complete, - "incomplete": resp.Incomplete, - "interval": resp.Interval, - "min interval": resp.MinInterval, - } + bb.WriteString("d8:completei") + bb.WriteString(strconv.FormatUint(uint64(resp.Complete), 10)) + bb.WriteString("e10:incompletei") + bb.WriteString(strconv.FormatUint(uint64(resp.Incomplete), 10)) + bb.WriteString("e8:intervali") + bb.WriteString(strconv.FormatUint(uint64(resp.Interval), 10)) + bb.WriteString("e12:min intervali") + bb.WriteString(strconv.FormatUint(uint64(resp.MinInterval), 10)) + bb.WriteByte('e') // Add the peers to the dictionary in the compact format. if resp.Compact { // Add the IPv4 peers to the dictionary. - compactAddresses := make([]byte, 0, (net.IPv4len+2)*len(resp.IPv4Peers)) - for _, peer := range resp.IPv4Peers { - compactAddresses = append(compactAddresses, compactAddress(peer)...) - } - if len(compactAddresses) > 0 { - bdict["peers"] = compactAddresses - } - + compactAddresses(bb, resp.IPv4Peers, false) // Add the IPv6 peers to the dictionary. - compactAddresses = make([]byte, 0, (net.IPv6len+2)*len(resp.IPv6Peers)) // IP + port - for _, peer := range resp.IPv6Peers { - compactAddresses = append(compactAddresses, compactAddress(peer)...) - } - if len(compactAddresses) > 0 { - bdict["peers6"] = compactAddresses - } + compactAddresses(bb, resp.IPv6Peers, true) } else { // Add the peers to the dictionary. - peers := make([]map[string]any, 0, len(resp.IPv4Peers)+len(resp.IPv6Peers)) // IP + port + bb.WriteString("5:peersl") for _, peer := range resp.IPv4Peers { - peers = append(peers, dict(peer)) + dictAddress(bb, peer) } for _, peer := range resp.IPv6Peers { - peers = append(peers, dict(peer)) + dictAddress(bb, peer) } - bdict["peers"] = peers + bb.WriteByte('e') } + bb.WriteByte('e') - return bencode.NewEncoder(w).Encode(bdict) + _, err := bb.WriteTo(w) + return err +} + +func compactAddresses(bb *bytes.Buffer, peers bittorrent.Peers, v6 bool) { + l := len(peers) + if l > 0 { + key, al := "5:peers", net.IPv4len + if v6 { + key, al = "6:peers6", net.IPv6len + } + bb.WriteString(key) + bb.WriteString(strconv.Itoa((al + 2) * l)) + bb.WriteByte(':') + for _, peer := range peers { + bb.Write(peer.Addr().AsSlice()) + port := peer.Port() + bb.WriteByte(byte(port >> 8)) + bb.WriteByte(byte(port)) + } + } +} + +func dictAddress(bb *bytes.Buffer, peer bittorrent.Peer) { + bb.WriteString("d2:ip") + addr := peer.Addr().String() + bb.WriteString(strconv.Itoa(len(addr))) + bb.WriteByte(':') + bb.WriteString(addr) + bb.WriteString("7:peer id20:") + bb.Write(peer.ID[:]) + bb.WriteString("4:porti") + bb.WriteString(strconv.FormatUint(uint64(peer.Port()), 10)) + bb.WriteString("ee") } // WriteScrapeResponse communicates the results of a Scrape to a BitTorrent // client over HTTP. func WriteScrapeResponse(w http.ResponseWriter, resp *bittorrent.ScrapeResponse) error { - filesDict := make(map[string]any, len(resp.Files)) + bb := respBufferPool.Get() + defer respBufferPool.Put(bb) + bb.WriteString("d5:filesd") for _, scrape := range resp.Files { - filesDict[string(scrape.InfoHash[:])] = map[string]any{ - "complete": scrape.Complete, - "incomplete": scrape.Incomplete, - } - } - - return bencode.NewEncoder(w).Encode(map[string]any{ - "files": filesDict, - }) -} - -func compactAddress(peer bittorrent.Peer) (buf []byte) { - buf = append(buf, peer.Addr().AsSlice()...) - port := peer.Port() - buf = append(buf, byte(port>>8), byte(port)) - return -} - -func dict(peer bittorrent.Peer) map[string]any { - return map[string]any{ - "peer id": peer.ID.RawString(), - "ip": peer.Addr(), - "port": peer.Port(), + bb.WriteString(strconv.Itoa(len(scrape.InfoHash))) + bb.WriteByte(':') + bb.Write([]byte(scrape.InfoHash)) + bb.WriteString("d8:completei") + bb.WriteString(strconv.FormatUint(uint64(scrape.Complete), 10)) + bb.WriteString("e10:downloadedi") + bb.WriteString(strconv.FormatUint(uint64(scrape.Snatches), 10)) + bb.WriteString("e10:incompletei") + bb.WriteString(strconv.FormatUint(uint64(scrape.Incomplete), 10)) + bb.WriteString("ee") } + bb.WriteString("ee") + _, err := bb.WriteTo(w) + return err } diff --git a/frontend/options.go b/frontend/options.go index d8941ca..c104853 100644 --- a/frontend/options.go +++ b/frontend/options.go @@ -1,8 +1,102 @@ package frontend -import "github.com/sot-tech/mochi/pkg/log" +import ( + "errors" + "net" + "time" -var logger = log.NewLogger("frontend configurator") + "github.com/libp2p/go-reuseport" +) + +const ( + defaultReadTimeout = 2 * time.Second + defaultWriteTimeout = 2 * time.Second +) + +var ( + // ErrAddressNotProvided returned if listen address not provided in configuration + ErrAddressNotProvided = errors.New("address not provided") + errUnexpectedListenerType = errors.New("unexpected listener type") +) + +// ListenOptions is the base configuration which may be used in net listeners +type ListenOptions struct { + Addr string + ReusePort bool `cfg:"reuse_port"` + ReadTimeout time.Duration `cfg:"read_timeout"` + WriteTimeout time.Duration `cfg:"write_timeout"` + EnableRequestTiming bool `cfg:"enable_request_timing"` +} + +// Validate checks if listen address provided and sets default +// timeout options if needed +func (lo ListenOptions) Validate() (validOptions ListenOptions, err error) { + validOptions = lo + if len(lo.Addr) == 0 { + err = ErrAddressNotProvided + } else { + if lo.ReadTimeout <= 0 { + validOptions.ReadTimeout = defaultReadTimeout + logger.Warn(). + Str("name", "ReadTimeout"). + Dur("provided", lo.ReadTimeout). + Dur("default", validOptions.ReadTimeout). + Msg("falling back to default configuration") + } + + if lo.WriteTimeout <= 0 { + validOptions.WriteTimeout = defaultWriteTimeout + logger.Warn(). + Str("name", "WriteTimeout"). + Dur("provided", lo.WriteTimeout). + Dur("default", validOptions.WriteTimeout). + Msg("falling back to default configuration") + } + } + return +} + +// ListenTCP listens at the given TCP Addr +// with SO_REUSEPORT and SO_REUSEADDR options enabled if +// ReusePort set to true +func (lo ListenOptions) ListenTCP() (conn *net.TCPListener, err error) { + if lo.ReusePort { + var ln net.Listener + if ln, err = reuseport.Listen("tcp", lo.Addr); err == nil { + var ok bool + if conn, ok = ln.(*net.TCPListener); !ok { + err = errUnexpectedListenerType + } + } + } else { + var addr *net.TCPAddr + if addr, err = net.ResolveTCPAddr("tcp", lo.Addr); err == nil { + conn, err = net.ListenTCP("tcp", addr) + } + } + return +} + +// ListenUDP listens at the given UDP Addr +// with SO_REUSEPORT and SO_REUSEADDR options enabled if +// ReusePort set to true +func (lo ListenOptions) ListenUDP() (conn *net.UDPConn, err error) { + if lo.ReusePort { + var ln net.PacketConn + if ln, err = reuseport.ListenPacket("udp", lo.Addr); err == nil { + var ok bool + if conn, ok = ln.(*net.UDPConn); !ok { + err = errUnexpectedListenerType + } + } + } else { + var addr *net.UDPAddr + if addr, err = net.ResolveUDPAddr("udp", lo.Addr); err == nil { + conn, err = net.ListenUDP("udp", addr) + } + } + return +} // ParseOptions is the configuration used to parse an Announce Request. // diff --git a/frontend/udp/frontend.go b/frontend/udp/frontend.go index 7b21839..6dadd61 100644 --- a/frontend/udp/frontend.go +++ b/frontend/udp/frontend.go @@ -13,11 +13,10 @@ import ( "sync" "time" - "github.com/libp2p/go-reuseport" - "github.com/sot-tech/mochi/bittorrent" "github.com/sot-tech/mochi/frontend" - "github.com/sot-tech/mochi/frontend/udp/bytepool" + "github.com/sot-tech/mochi/middleware" + "github.com/sot-tech/mochi/pkg/bytepool" "github.com/sot-tech/mochi/pkg/conf" "github.com/sot-tech/mochi/pkg/log" "github.com/sot-tech/mochi/pkg/metrics" @@ -26,28 +25,32 @@ import ( ) var ( - logger = log.NewLogger("udp frontend") + logger = log.NewLogger("frontend/udp") allowedGeneratedPrivateKeyRunes = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890") - errUnexpectedConnType = errors.New("unexpected connection type (not UDPConn)") ) -// Config represents all of the configurable options for a UDP BitTorrent +func init() { + frontend.RegisterBuilder("udp", NewFrontend) +} + +// Config represents all the configurable options for a UDP BitTorrent // Tracker. type Config struct { - Addr string - ReusePort bool `cfg:"reuse_port"` - PrivateKey string `cfg:"private_key"` - MaxClockSkew time.Duration `cfg:"max_clock_skew"` - EnableRequestTiming bool `cfg:"enable_request_timing"` + frontend.ListenOptions + PrivateKey string `cfg:"private_key"` + MaxClockSkew time.Duration `cfg:"max_clock_skew"` frontend.ParseOptions } // Validate sanity checks values set in a config and returns a new config with // default values replacing anything that is invalid. -// -// This function warns to the logger when a value is changed. -func (cfg Config) Validate() Config { - validcfg := cfg +func (cfg Config) Validate() (validCfg Config, err error) { + if len(cfg.Addr) == 0 { + err = frontend.ErrAddressNotProvided + return + } + + validCfg = cfg // Generate a private key if one isn't provided by the user. if cfg.PrivateKey == "" { @@ -55,45 +58,49 @@ func (cfg Config) Validate() Config { for i := range pkeyRunes { pkeyRunes[i] = allowedGeneratedPrivateKeyRunes[rand.Intn(len(allowedGeneratedPrivateKeyRunes))] } - validcfg.PrivateKey = string(pkeyRunes) + validCfg.PrivateKey = string(pkeyRunes) logger.Warn(). - Str("name", "UDP.PrivateKey"). + Str("name", "PrivateKey"). Str("provided", ""). - Str("key", validcfg.PrivateKey). + Str("key", validCfg.PrivateKey). Msg("falling back to default configuration") } - validcfg.ParseOptions = cfg.ParseOptions.Validate() + validCfg.ParseOptions = cfg.ParseOptions.Validate() - return validcfg + return } -// Frontend holds the state of a UDP BitTorrent Frontend. -type Frontend struct { - socket *net.UDPConn - closing chan struct{} - wg sync.WaitGroup - - genPool *sync.Pool - - logic frontend.TrackerLogic - Config +// udpFE holds the state of a UDP BitTorrent Frontend. +type udpFE struct { + socket *net.UDPConn + closing chan any + wg sync.WaitGroup + genPool *sync.Pool + logic *middleware.Logic + maxClockSkew time.Duration + collectTimings bool + frontend.ParseOptions } -// NewFrontend creates a new instance of an UDP Frontend that asynchronously -// serves requests. -func NewFrontend(logic frontend.TrackerLogic, c conf.MapConfig) (*Frontend, error) { - var provided Config - if err := c.Unmarshal(&provided); err != nil { +// NewFrontend builds and starts udp bittorrent frontend from provided configuration +func NewFrontend(c conf.MapConfig, logic *middleware.Logic) (frontend.Frontend, error) { + var err error + var cfg Config + if err = c.Unmarshal(&cfg); err != nil { + return nil, err + } + if cfg, err = cfg.Validate(); err != nil { return nil, err } - cfg := provided.Validate() - f := &Frontend{ - closing: make(chan struct{}), - logic: logic, - Config: cfg, + f := &udpFE{ + closing: make(chan any), + logic: logic, + maxClockSkew: cfg.MaxClockSkew, + collectTimings: cfg.EnableRequestTiming, + ParseOptions: cfg.ParseOptions, genPool: &sync.Pool{ New: func() any { return NewConnectionIDGenerator(cfg.PrivateKey) @@ -101,22 +108,20 @@ func NewFrontend(logic frontend.TrackerLogic, c conf.MapConfig) (*Frontend, erro }, } - if err := f.listen(); err != nil { - return nil, err + if f.socket, err = cfg.ListenUDP(); err == nil { + f.wg.Add(1) + go func() { + if err := f.serve(); err != nil { + logger.Fatal().Err(err).Msg("server failed") + } + }() } - f.wg.Add(1) - go func() { - if err := f.serve(); err != nil { - logger.Fatal().Err(err).Str("proto", "udp").Msg("failed while serving") - } - }() - - return f, nil + return f, err } // Stop provides a thread-safe way to shut down a currently running Frontend. -func (t *Frontend) Stop() stop.Result { +func (t *udpFE) Stop() stop.Result { select { case <-t.closing: return stop.AlreadyStopped @@ -126,42 +131,26 @@ func (t *Frontend) Stop() stop.Result { c := make(stop.Channel) go func() { close(t.closing) - _ = t.socket.SetReadDeadline(time.Now()) - t.wg.Wait() - c.Done(t.socket.Close()) + var err error + if t.socket != nil { + _ = t.socket.SetReadDeadline(time.Now()) + t.wg.Wait() + err = t.socket.Close() + } + c.Done(err) }() return c.Result() } -// listen resolves the address and binds the server socket. -func (t *Frontend) listen() (err error) { - if t.ReusePort { - var ln net.PacketConn - if ln, err = reuseport.ListenPacket("udp", t.Addr); err == nil { - var isOk bool - if t.socket, isOk = ln.(*net.UDPConn); !isOk { - err = errUnexpectedConnType - } - } - } else { - var udpAddr *net.UDPAddr - udpAddr, err = net.ResolveUDPAddr("udp", t.Addr) - if err == nil { - t.socket, err = net.ListenUDP("udp", udpAddr) - } - } - return err -} - // serve blocks while listening and serving UDP BitTorrent requests // until Stop() is called or an error is returned. -func (t *Frontend) serve() error { - pool := bytepool.New(2048) +func (t *udpFE) serve() error { + pool := bytepool.NewBytePool(2048) defer t.wg.Done() for { - // Check to see if we need to shutdown. + // Check to see if we need shutdown. select { case <-t.closing: log.Debug().Msg("serve received shutdown signal") @@ -196,14 +185,14 @@ func (t *Frontend) serve() error { // Handle the request. addr := addrPort.Addr().Unmap() var start time.Time - if t.EnableRequestTiming && metrics.Enabled() { + if t.collectTimings && metrics.Enabled() { start = time.Now() } action, err := t.handleRequest( Request{(*buffer)[:n], addr}, ResponseWriter{t.socket, addrPort}, ) - if t.EnableRequestTiming && metrics.Enabled() { + if t.collectTimings && metrics.Enabled() { recordResponseDuration(action, addr, err, time.Since(start)) } }() @@ -229,7 +218,7 @@ func (w ResponseWriter) Write(b []byte) (int, error) { } // handleRequest parses and responds to a UDP Request. -func (t *Frontend) handleRequest(r Request, w ResponseWriter) (actionName string, err error) { +func (t *udpFE) handleRequest(r Request, w ResponseWriter) (actionName string, err error) { if len(r.Packet) < 16 { // Malformed, no client packets are less than 16 bytes. // We explicitly return nothing in case this is a DoS attempt. @@ -248,7 +237,7 @@ func (t *Frontend) handleRequest(r Request, w ResponseWriter) (actionName string // If this isn't requesting a new connection ID and the connection ID is // invalid, then fail. - if actionID != connectActionID && !gen.Validate(connID, r.IP, timecache.Now(), t.MaxClockSkew) { + if actionID != connectActionID && !gen.Validate(connID, r.IP, timecache.Now(), t.maxClockSkew) { err = errBadConnectionID WriteError(w, txID, err) return diff --git a/frontend/udp/frontend_test.go b/frontend/udp/frontend_test.go index 86b8c6f..8ae8b8c 100644 --- a/frontend/udp/frontend_test.go +++ b/frontend/udp/frontend_test.go @@ -17,12 +17,15 @@ func init() { } func TestStartStopRaceIssue437(t *testing.T) { - ps, err := storage.NewStorage("memory", conf.MapConfig{}) + ps, err := storage.NewStorage(conf.NamedMapConfig{ + Name: "memory", + Config: conf.MapConfig{}, + }) if err != nil { t.Fatal(err) } lgc := middleware.NewLogic(0, 0, ps, nil, nil) - fe, err := udp.NewFrontend(lgc, conf.MapConfig{"addr": "127.0.0.1:0"}) + fe, err := udp.NewFrontend(conf.MapConfig{"addr": "127.0.0.1:0"}, lgc) if err != nil { t.Fatal(err) } diff --git a/frontend/udp/parser.go b/frontend/udp/parser.go index f3c0aaf..91e8571 100644 --- a/frontend/udp/parser.go +++ b/frontend/udp/parser.go @@ -1,15 +1,14 @@ package udp import ( - "bytes" "encoding/binary" "fmt" "net" "net/netip" - "sync" "github.com/sot-tech/mochi/bittorrent" "github.com/sot-tech/mochi/frontend" + "github.com/sot-tech/mochi/pkg/bytepool" ) const ( @@ -48,6 +47,8 @@ var ( errUnknownOptionType = bittorrent.ClientError("unknown option type") errInvalidInfoHash = bittorrent.ClientError("invalid info hash") errInvalidPeerID = bittorrent.ClientError("invalid info hash") + + reqRespBufferPool = bytepool.NewBufferPool() ) // ParseAnnounce parses an AnnounceRequest from a UDP request. @@ -111,23 +112,6 @@ func ParseAnnounce(r Request, v6Action bool, opts frontend.ParseOptions) (*bitto return request, err } -type buffer struct { - bytes.Buffer -} - -var bufferFree = sync.Pool{ - New: func() any { return new(buffer) }, -} - -func newBuffer() *buffer { - return bufferFree.Get().(*buffer) -} - -func (b *buffer) free() { - b.Reset() - bufferFree.Put(b) -} - // handleOptionalParameters parses the optional parameters as described in BEP // 41 and updates an announce with the values parsed. func handleOptionalParameters(packet []byte) (bittorrent.Params, error) { @@ -135,8 +119,8 @@ func handleOptionalParameters(packet []byte) (bittorrent.Params, error) { return bittorrent.ParseURLData("") } - buf := newBuffer() - defer buf.free() + buf := reqRespBufferPool.Get() + defer reqRespBufferPool.Put(buf) for i := 0; i < len(packet); { option := packet[i] diff --git a/frontend/udp/writer.go b/frontend/udp/writer.go index fe860d3..c6d8e34 100644 --- a/frontend/udp/writer.go +++ b/frontend/udp/writer.go @@ -18,8 +18,8 @@ func WriteError(w io.Writer, txID []byte, err error) { err = fmt.Errorf("internal error occurred: %w", err) } - buf := newBuffer() - defer buf.free() + buf := reqRespBufferPool.Get() + defer reqRespBufferPool.Put(buf) writeHeader(buf, txID, errorActionID) _, _ = buf.WriteString(err.Error()) _, _ = buf.WriteRune('\000') @@ -32,8 +32,8 @@ func WriteError(w io.Writer, txID []byte, err error) { // If v6Action is set, the action will be 4, according to // https://web.archive.org/web/20170503181830/http://opentracker.blog.h3q.com/2007/12/28/the-ipv6-situation/ func WriteAnnounce(w io.Writer, txID []byte, resp *bittorrent.AnnounceResponse, v6Action, v6Peers bool) { - buf := newBuffer() - defer buf.free() + buf := reqRespBufferPool.Get() + defer reqRespBufferPool.Put(buf) if v6Action { writeHeader(buf, txID, announceV6ActionID) @@ -59,8 +59,8 @@ func WriteAnnounce(w io.Writer, txID []byte, resp *bittorrent.AnnounceResponse, // WriteScrape encodes a scrape response according to BEP 15. func WriteScrape(w io.Writer, txID []byte, resp *bittorrent.ScrapeResponse) { - buf := newBuffer() - defer buf.free() + buf := reqRespBufferPool.Get() + defer reqRespBufferPool.Put(buf) writeHeader(buf, txID, scrapeActionID) @@ -75,8 +75,8 @@ func WriteScrape(w io.Writer, txID []byte, resp *bittorrent.ScrapeResponse) { // WriteConnectionID encodes a new connection response according to BEP 15. func WriteConnectionID(w io.Writer, txID, connID []byte) { - buf := newBuffer() - defer buf.free() + buf := reqRespBufferPool.Get() + defer reqRespBufferPool.Put(buf) writeHeader(buf, txID, connectActionID) _, _ = buf.Write(connID) diff --git a/go.mod b/go.mod index b9260c4..bfe6302 100644 --- a/go.mod +++ b/go.mod @@ -3,48 +3,48 @@ module github.com/sot-tech/mochi go 1.18 require ( - code.cloudfoundry.org/go-diodes v0.0.0-20220927211948-2985a72b297c - github.com/MicahParks/keyfunc v1.4.0 + code.cloudfoundry.org/go-diodes v0.0.0-20221017175818-728392b37655 + github.com/MicahParks/keyfunc v1.5.1 github.com/anacrolix/torrent v1.47.0 github.com/go-redis/redis/v8 v8.11.5 github.com/golang-jwt/jwt/v4 v4.4.2 - github.com/jackc/pgx/v5 v5.0.1 + github.com/jackc/pgx/v5 v5.0.4 github.com/julienschmidt/httprouter v1.3.0 github.com/libp2p/go-reuseport v0.2.0 github.com/minio/sha256-simd v1.0.0 github.com/mitchellh/mapstructure v1.5.0 github.com/prometheus/client_golang v1.13.0 github.com/rs/zerolog v1.28.0 - github.com/stretchr/testify v1.8.0 + github.com/stretchr/testify v1.8.1 gopkg.in/yaml.v3 v3.0.1 ) require ( - github.com/anacrolix/dht/v2 v2.19.0 // indirect + github.com/anacrolix/dht/v2 v2.19.1 // indirect github.com/anacrolix/log v0.13.2-0.20220711050817-613cb738ef30 // indirect github.com/anacrolix/missinggo v1.3.0 // indirect - github.com/anacrolix/missinggo/v2 v2.7.0 // indirect + github.com/anacrolix/missinggo/v2 v2.7.1 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect - github.com/fsnotify/fsnotify v1.5.4 // indirect + github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/huandu/xstrings v1.3.2 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect github.com/jackc/puddle/v2 v2.0.0 // indirect - github.com/klauspost/cpuid/v2 v2.1.1 // indirect + github.com/klauspost/cpuid/v2 v2.1.2 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.16 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.2 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_model v0.2.0 // indirect + github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/common v0.37.0 // indirect github.com/prometheus/procfs v0.8.0 // indirect - golang.org/x/crypto v0.0.0-20220926161630-eccd6366d1be // indirect - golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec // indirect - golang.org/x/text v0.3.7 // indirect + golang.org/x/crypto v0.1.0 // indirect + golang.org/x/sys v0.1.0 // indirect + golang.org/x/text v0.4.0 // indirect google.golang.org/protobuf v1.28.1 // indirect ) diff --git a/go.sum b/go.sum index ad69847..bb1a7e7 100644 --- a/go.sum +++ b/go.sum @@ -30,15 +30,15 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -code.cloudfoundry.org/go-diodes v0.0.0-20220927211948-2985a72b297c h1:6Gb9bHc7g8thQ4AWxwdA6qxhtDNcqNw2T1+Z+w37kOw= -code.cloudfoundry.org/go-diodes v0.0.0-20220927211948-2985a72b297c/go.mod h1:v30sK4Ipg4Ng0pRG22iLfeMRfYUHQNee9f4NkE5t1yc= +code.cloudfoundry.org/go-diodes v0.0.0-20221017175818-728392b37655 h1:3IlYr3dIyXkemR1GoOITnBWnWBHG27wnC6mpV9OLD7c= +code.cloudfoundry.org/go-diodes v0.0.0-20221017175818-728392b37655/go.mod h1:T+qZrTfw8xwdJgVRD5pSvVyx4CeqsP30YQooBYFADRc= crawshaw.io/iox v0.0.0-20181124134642-c51c3df30797/go.mod h1:sXBiorCo8c46JlQV3oXPKINnZ8mcqnye1EkVkqsectk= crawshaw.io/sqlite v0.3.2/go.mod h1:igAO5JulrQ1DbdZdtVq48mnZUBAPOeFzer7VhDWNtW4= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/MicahParks/keyfunc v1.4.0 h1:/qTnDiQJSM0pkUvKRVb82jvvm0SAOffAqKSIrgUqwdI= -github.com/MicahParks/keyfunc v1.4.0/go.mod h1:IdnCilugA0O/99dW+/MkvlyrsX8+L8+x95xuVNtM5jw= +github.com/MicahParks/keyfunc v1.5.1 h1:RlyyYgKQI/adkIw1yXYtPvTAOb7hBhSX42aH23d8N0Q= +github.com/MicahParks/keyfunc v1.5.1/go.mod h1:IdnCilugA0O/99dW+/MkvlyrsX8+L8+x95xuVNtM5jw= github.com/RoaringBitmap/roaring v0.4.7/go.mod h1:8khRDP4HmeXns4xIj9oGrKSz7XTQiJx2zgh7AcNke4w= github.com/RoaringBitmap/roaring v0.4.17/go.mod h1:D3qVegWTmfCaX4Bl5CrBE9hfrSrrXIr8KVNvRsDi1NI= github.com/RoaringBitmap/roaring v0.4.23/go.mod h1:D0gp8kJQgE1A4LQ5wFLggQEyvDi06Mq5mKs52e1TwOo= @@ -49,8 +49,8 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= -github.com/anacrolix/dht/v2 v2.19.0 h1:A9oMHWRGbLmCyx1JlYzg79bDrur8V60+0ts8ZwEVYt4= -github.com/anacrolix/dht/v2 v2.19.0/go.mod h1:0h83KnnAQ2AUYhpQ/CkoZP45K41pjDAlPR9zGHgFjQE= +github.com/anacrolix/dht/v2 v2.19.1 h1:V/UUGBASGYqYkSnmHJwX8uQmzkyhbgwE6jqcHKnNTD8= +github.com/anacrolix/dht/v2 v2.19.1/go.mod h1:3TU93c1s/oA8I/VH4m3CNP/BeKsiOGmo6HwfZBMTKUs= github.com/anacrolix/envpprof v0.0.0-20180404065416-323002cec2fa/go.mod h1:KgHhUaQMc8cC0+cEflSgCFNFbKwi5h54gqtVn8yhP7c= github.com/anacrolix/envpprof v1.0.0/go.mod h1:KgHhUaQMc8cC0+cEflSgCFNFbKwi5h54gqtVn8yhP7c= github.com/anacrolix/envpprof v1.1.0/go.mod h1:My7T5oSqVfEn4MD4Meczkw/f5lSIndGAKu/0SM/rkf4= @@ -67,8 +67,8 @@ github.com/anacrolix/missinggo v1.3.0/go.mod h1:bqHm8cE8xr+15uVfMG3BFui/TxyB6//H github.com/anacrolix/missinggo/perf v1.0.0/go.mod h1:ljAFWkBuzkO12MQclXzZrosP5urunoLS0Cbvb4V0uMQ= github.com/anacrolix/missinggo/v2 v2.2.0/go.mod h1:o0jgJoYOyaoYQ4E2ZMISVa9c88BbUBVQQW4QeRkNCGY= github.com/anacrolix/missinggo/v2 v2.5.1/go.mod h1:WEjqh2rmKECd0t1VhQkLGTdIWXO6f6NLjp5GlMZ+6FA= -github.com/anacrolix/missinggo/v2 v2.7.0 h1:4fzOAAn/VCvfWGviLmh64MPMttrlYew81JdPO7nSHvI= -github.com/anacrolix/missinggo/v2 v2.7.0/go.mod h1:2IZIvmRTizALNYFYXsPR7ofXPzJgyBpKZ4kMqMEICkI= +github.com/anacrolix/missinggo/v2 v2.7.1 h1:Y+wL0JC6D2icpwhDpcrRM4THQB/uFcPNYUtZMbYvQgI= +github.com/anacrolix/missinggo/v2 v2.7.1/go.mod h1:2IZIvmRTizALNYFYXsPR7ofXPzJgyBpKZ4kMqMEICkI= github.com/anacrolix/stm v0.2.0/go.mod h1:zoVQRvSiGjGoTmbM0vSLIiaKjWtNPeTvXUSdJQA4hsg= github.com/anacrolix/tagflag v0.0.0-20180109131632-2146c8d41bf0/go.mod h1:1m2U/K6ZT+JZG0+bdMK6qauP49QT4wE5pmhJXOKKCHw= github.com/anacrolix/tagflag v1.0.0/go.mod h1:1m2U/K6ZT+JZG0+bdMK6qauP49QT4wE5pmhJXOKKCHw= @@ -113,8 +113,8 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= -github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= +github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= +github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= github.com/glycerine/go-unsnap-stream v0.0.0-20190901134440-81cf024a9e0a/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= @@ -215,8 +215,8 @@ github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsI github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg= github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= -github.com/jackc/pgx/v5 v5.0.1 h1:JZu9othr7l8so2JMDAGeDUMXqERAuZpovyfl4H50tdg= -github.com/jackc/pgx/v5 v5.0.1/go.mod h1:JBbvW3Hdw77jKl9uJrEDATUZIFM2VFPzRq4RWIhkF4o= +github.com/jackc/pgx/v5 v5.0.4 h1:r5O6y84qHX/z/HZV40JBdx2obsHz7/uRj5b+CcYEdeY= +github.com/jackc/pgx/v5 v5.0.4/go.mod h1:U0ynklHtgg43fue9Ly30w3OCSTDPlXjig9ghrNGaguQ= github.com/jackc/puddle/v2 v2.0.0 h1:Kwk/AlLigcnZsDssc3Zun1dk1tAtQNPaBBxBHWn0Mjc= github.com/jackc/puddle/v2 v2.0.0/go.mod h1:itE7ZJY8xnoo0JqJEpSMprN0f+NQkMCuEV/N9j8h0oc= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= @@ -234,8 +234,8 @@ github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4d github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.1.1 h1:t0wUqjowdm8ezddV5k0tLWVklVuvLJpoHeb4WBdydm0= -github.com/klauspost/cpuid/v2 v2.1.1/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= +github.com/klauspost/cpuid/v2 v2.1.2 h1:XhdX4fqAJUA0yj+kUwMavO0hHrSPAecYdYf1ZmxHvak= +github.com/klauspost/cpuid/v2 v2.1.2/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= @@ -255,8 +255,8 @@ github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27k github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/matttproud/golang_protobuf_extensions v1.0.2 h1:hAHbPm5IJGijwng3PWk09JkG9WeqChjprR5s9bBZ+OM= -github.com/matttproud/golang_protobuf_extensions v1.0.2/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= +github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= @@ -275,7 +275,7 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.20.2 h1:8uQq0zMgLEfa0vRrrBgaJF2gyW9Da9BmfGV+OyUzfkY= +github.com/onsi/gomega v1.22.1 h1:pY8O4lBfsHKZHM/6nrxkhVPUznOlIu3quZcKP/M20KI= github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= @@ -298,8 +298,9 @@ github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1: github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= +github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= @@ -337,6 +338,7 @@ github.com/smartystreets/goconvey v0.0.0-20190306220146-200a235640ff/go.mod h1:K github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -344,8 +346,9 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/tinylib/msgp v1.1.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= @@ -367,8 +370,8 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20220926161630-eccd6366d1be h1:fmw3UbQh+nxngCAHrDCCztao/kbYFnWjoqop8dHx05A= -golang.org/x/crypto v0.0.0-20220926161630-eccd6366d1be/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU= +golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -432,7 +435,7 @@ golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b h1:PxfKdU9lEEDYjdIzOtC4qFWgkU2rGHdKlKowJSMN9h0= +golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -492,11 +495,11 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec h1:BkDtF2Ih9xZ7le9ndzTA7KJow28VbQW3odyk/8drmuI= -golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -505,8 +508,9 @@ golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/middleware/clientapproval/clientapproval.go b/middleware/clientapproval/clientapproval.go index 644e8ed..8c272dd 100644 --- a/middleware/clientapproval/clientapproval.go +++ b/middleware/clientapproval/clientapproval.go @@ -39,10 +39,10 @@ type hook struct { unapproved map[ClientID]struct{} } -func build(options conf.MapConfig, _ storage.PeerStorage) (middleware.Hook, error) { +func build(config conf.MapConfig, _ storage.PeerStorage) (middleware.Hook, error) { var cfg Config - if err := options.Unmarshal(&cfg); err != nil { + if err := config.Unmarshal(&cfg); err != nil { return nil, fmt.Errorf("middleware %s: %w", Name, err) } diff --git a/middleware/hooks.go b/middleware/hooks.go index 2b1d77c..444969b 100644 --- a/middleware/hooks.go +++ b/middleware/hooks.go @@ -19,7 +19,7 @@ type Hook interface { } // Pinger is an optional interface that may be implemented by a pre Hook -// to check if it is operational. Used in frontend.TrackerLogic. +// to check if it is operational. Used in frontend.Logic. // // It may be useful in cases when Hook performs foreign requests to // some external resources (i.e. storage) and `ping` request should diff --git a/middleware/jwt/jwt.go b/middleware/jwt/jwt.go index fe57633..34556ae 100644 --- a/middleware/jwt/jwt.go +++ b/middleware/jwt/jwt.go @@ -23,19 +23,17 @@ import ( "github.com/sot-tech/mochi/storage" ) -// Name is the name by which this middleware is registered with Conf. const ( - Name = "jwt" authorizationHeader = "authorization" bearerAuthPrefix = "bearer " ) func init() { - middleware.RegisterBuilder(Name, build) + middleware.RegisterBuilder("jwt", build) } var ( - logger = log.NewLogger(Name) + logger = log.NewLogger("middleware/jwt") // ErrMissingJWT is returned when a JWT is missing from a request. ErrMissingJWT = bittorrent.ClientError("unapproved request: missing jwt") @@ -70,13 +68,11 @@ type hook struct { jwks *keyfunc.JWKS } -func build(options conf.MapConfig, _ storage.PeerStorage) (h middleware.Hook, err error) { +func build(config conf.MapConfig, _ storage.PeerStorage) (h middleware.Hook, err error) { var cfg Config - logger.Debug().Object("options", options).Msg("creating new JWT middleware") - - if err = options.Unmarshal(&cfg); err != nil { - return nil, fmt.Errorf("middleware %s: %w", Name, err) + if err = config.Unmarshal(&cfg); err != nil { + return nil, fmt.Errorf("unable to deserialise configuration: %w", err) } if len(cfg.JWKSetURL) > 0 && len(cfg.Issuer) > 0 && len(cfg.Audience) > 0 { @@ -206,7 +202,7 @@ func (h *hook) HandleScrape(ctx context.Context, req *bittorrent.ScrapeRequest, if errs := h.validateBaseJWT(jwtParam, claims); len(errs) > 0 { logger.Info(). Errs("errors", errs). - Array("source", req.RequestAddresses). + Array("source", &req.RequestAddresses). Msg("JWT validation failed") err = ErrInvalidJWT } else { @@ -217,7 +213,7 @@ func (h *hook) HandleScrape(ctx context.Context, req *bittorrent.ScrapeRequest, } else { logger.Info(). Err(err). - Array("addresses", req.RequestAddresses). + Array("addresses", &req.RequestAddresses). Msg("'infohashes' claim parse failed") } } @@ -241,7 +237,7 @@ func (h *hook) HandleScrape(ctx context.Context, req *bittorrent.ScrapeRequest, logger.Info(). Array("claimInfoHashes", claimIHs). Array("requestInfoHashes", req.InfoHashes). - Array("addresses", req.RequestAddresses). + Array("addresses", &req.RequestAddresses). Msg("unequal 'infohashes' claim when validating JWT") err = ErrInvalidJWT } diff --git a/middleware/logic.go b/middleware/logic.go index 93320de..458c156 100644 --- a/middleware/logic.go +++ b/middleware/logic.go @@ -2,21 +2,26 @@ package middleware import ( "context" + "sort" "time" "github.com/sot-tech/mochi/bittorrent" - "github.com/sot-tech/mochi/frontend" - "github.com/sot-tech/mochi/pkg/log" "github.com/sot-tech/mochi/pkg/stop" "github.com/sot-tech/mochi/storage" ) -var ( - logger = log.NewLogger("middleware") - _ frontend.TrackerLogic = &Logic{} -) +// Logic used by a frontend in order to: (1) generate a +// response from a parsed request, and (2) asynchronously observe anything +// after the response has been delivered to the client. +type Logic struct { + announceInterval time.Duration + minAnnounceInterval time.Duration + preHooks []Hook + postHooks []Hook + pingers []Pinger +} -// NewLogic creates a new instance of a TrackerLogic that executes the provided +// NewLogic creates a new instance of a Logic that executes the provided // middleware hooks. func NewLogic(annInterval, minAnnInterval time.Duration, peerStore storage.PeerStorage, preHooks, postHooks []Hook) *Logic { l := &Logic{ @@ -34,17 +39,10 @@ func NewLogic(annInterval, minAnnInterval time.Duration, peerStore storage.PeerS return l } -// Logic is an implementation of the TrackerLogic that functions by -// executing a series of middleware hooks. -type Logic struct { - announceInterval time.Duration - minAnnounceInterval time.Duration - preHooks []Hook - postHooks []Hook - pingers []Pinger -} - // HandleAnnounce generates a response for an Announce. +// +// Returns the updated context, the generated AnnounceResponse and no error +// on success; nil and error on failure. func (l *Logic) HandleAnnounce(ctx context.Context, req *bittorrent.AnnounceRequest) (_ context.Context, resp *bittorrent.AnnounceResponse, err error) { logger.Debug().Object("request", req).Msg("new announce request") resp = &bittorrent.AnnounceResponse{ @@ -62,8 +60,8 @@ func (l *Logic) HandleAnnounce(ctx context.Context, req *bittorrent.AnnounceRequ return ctx, resp, nil } -// AfterAnnounce does something with the results of an Announce after it has -// been completed. +// AfterAnnounce does something with the results of an Announce after it +// has been completed. func (l *Logic) AfterAnnounce(ctx context.Context, req *bittorrent.AnnounceRequest, resp *bittorrent.AnnounceResponse) { var err error for _, h := range l.postHooks { @@ -78,6 +76,9 @@ func (l *Logic) AfterAnnounce(ctx context.Context, req *bittorrent.AnnounceReque } // HandleScrape generates a response for a Scrape. +// +// Returns the updated context, the generated AnnounceResponse and no error +// on success; nil and error on failure. func (l *Logic) HandleScrape(ctx context.Context, req *bittorrent.ScrapeRequest) (_ context.Context, resp *bittorrent.ScrapeResponse, err error) { logger.Debug().Object("request", req).Msg("new scrape request") resp = &bittorrent.ScrapeResponse{ @@ -88,13 +89,13 @@ func (l *Logic) HandleScrape(ctx context.Context, req *bittorrent.ScrapeRequest) return nil, nil, err } } + sort.Sort(&resp.Files) logger.Debug().Object("response", resp).Msg("generated scrape response") return ctx, resp, nil } -// AfterScrape does something with the results of a Scrape after it has been -// completed. +// AfterScrape does something with the results of a Scrape after it has been completed. func (l *Logic) AfterScrape(ctx context.Context, req *bittorrent.ScrapeRequest, resp *bittorrent.ScrapeResponse) { var err error for _, h := range l.postHooks { @@ -109,7 +110,7 @@ func (l *Logic) AfterScrape(ctx context.Context, req *bittorrent.ScrapeRequest, } } -// Ping performs check if all Hook-s are operational +// Ping executes checks if all Hook-s are operational func (l *Logic) Ping(ctx context.Context) (err error) { for _, p := range l.pingers { if err = p.Ping(ctx); err != nil { diff --git a/middleware/middleware.go b/middleware/middleware.go index bb8bf05..d25fb30 100644 --- a/middleware/middleware.go +++ b/middleware/middleware.go @@ -1,92 +1,65 @@ -// Package middleware implements the TrackerLogic interface by executing +// Package middleware implements the Logic interface by executing // a series of middleware hooks. package middleware import ( - "errors" + "fmt" "sync" "github.com/sot-tech/mochi/pkg/conf" + "github.com/sot-tech/mochi/pkg/log" "github.com/sot-tech/mochi/storage" ) var ( - driversM sync.RWMutex - drivers = make(map[string]Builder) - - // ErrBuilderDoesNotExist is the error returned by NewMiddleware when a - // middleware driver with that name does not exist. - ErrBuilderDoesNotExist = errors.New("middleware builder with that name does not exist") + logger = log.NewLogger("middleware") + buildersMU sync.RWMutex + builders = make(map[string]Builder) ) -// Builder is the interface used to initialize a new type of middleware. -// -// The `options` parameter is map of parameters that should be unmarshalled into -// the hook's custom configuration. -type Builder func(options conf.MapConfig, storage storage.PeerStorage) (Hook, error) +// Builder is the function used to initialize a new Hook +// with provided configuration. +type Builder func(conf.MapConfig, storage.PeerStorage) (Hook, error) // RegisterBuilder makes a Builder available by the provided name. // // If called twice with the same name, the name is blank, or if the provided // Builder is nil, this function panics. -func RegisterBuilder(name string, d Builder) { +func RegisterBuilder(name string, b Builder) { if name == "" { - panic("middleware: could not register a Builder with an empty name") + panic("middleware: could not register Builder with an empty name") } - if d == nil { + if b == nil { panic("middleware: could not register a nil Builder") } - driversM.Lock() - defer driversM.Unlock() + buildersMU.Lock() + defer buildersMU.Unlock() - if _, dup := drivers[name]; dup { + if _, dup := builders[name]; dup { panic("middleware: RegisterBuilder called twice for " + name) } - drivers[name] = d -} - -// NewHook attempts to initialize a new middleware instance from the -// list of registered Builders. -// -// If a driver does not exist, returns ErrBuilderDoesNotExist. -func NewHook(name string, options conf.MapConfig, storage storage.PeerStorage) (Hook, error) { - driversM.RLock() - defer driversM.RUnlock() - - var newHook Builder - newHook, ok := drivers[name] - if !ok { - return nil, ErrBuilderDoesNotExist - } - - return newHook(options, storage) -} - -// Config is the generic configuration format used for all registered Hooks. -type Config struct { - Name string - Options conf.MapConfig + builders[name] = b } // NewHooks is a utility function for initializing Hooks in bulk. -// each element of configs must contain pairs `name` - string and `options` - map[string]any -func NewHooks(configs []conf.MapConfig, storage storage.PeerStorage) (hooks []Hook, err error) { - for _, cfg := range configs { - var c Config - - if err = cfg.Unmarshal(&c); err != nil { +func NewHooks(configs []conf.NamedMapConfig, storage storage.PeerStorage) (hooks []Hook, err error) { + buildersMU.RLock() + defer buildersMU.RUnlock() + for _, c := range configs { + logger.Debug().Object("hook", c).Msg("starting hook") + newHook, ok := builders[c.Name] + if !ok { + err = fmt.Errorf("hook with name '%s' does not exists", c.Name) break } - var h Hook - h, err = NewHook(c.Name, c.Options, storage) - if err != nil { + if h, err = newHook(c.Config, storage); err != nil { break } - hooks = append(hooks, h) + logger.Info().Str("name", c.Name).Msg("hook started") } return diff --git a/middleware/torrentapproval/container/directory/directory.go b/middleware/torrentapproval/container/directory/directory.go index a4867ad..d002fa4 100644 --- a/middleware/torrentapproval/container/directory/directory.go +++ b/middleware/torrentapproval/container/directory/directory.go @@ -20,13 +20,10 @@ import ( "github.com/sot-tech/mochi/storage" ) -// Name of this container for registry -const Name = "directory" - -var logger = log.NewLogger("torrent approval directory") +var logger = log.NewLogger("middleware/torrent approval/directory") func init() { - container.Register(Name, build) + container.Register("directory", build) } // Config - implementation of directory container configuration. diff --git a/middleware/torrentapproval/container/list/list.go b/middleware/torrentapproval/container/list/list.go index 1ffb4ea..0440411 100644 --- a/middleware/torrentapproval/container/list/list.go +++ b/middleware/torrentapproval/container/list/list.go @@ -12,13 +12,10 @@ import ( "github.com/sot-tech/mochi/storage" ) -// Name of this container for registry. -const Name = "list" - -var logger = log.NewLogger("torrent approval list") +var logger = log.NewLogger("middleware/torrent approval/list") func init() { - container.Register(Name, build) + container.Register("list", build) } // Config - implementation of list container configuration. diff --git a/middleware/torrentapproval/torrentapproval.go b/middleware/torrentapproval/torrentapproval.go index 2bed5f5..fab9f8a 100644 --- a/middleware/torrentapproval/torrentapproval.go +++ b/middleware/torrentapproval/torrentapproval.go @@ -38,18 +38,18 @@ type baseConfig struct { Configuration conf.MapConfig } -func build(options conf.MapConfig, st storage.PeerStorage) (h middleware.Hook, err error) { +func build(config conf.MapConfig, st storage.PeerStorage) (h middleware.Hook, err error) { var cfg baseConfig - if err = options.Unmarshal(&cfg); err != nil { + if err = config.Unmarshal(&cfg); err != nil { return nil, fmt.Errorf("middleware %s: %w", Name, err) } if len(cfg.Source) == 0 { - return nil, fmt.Errorf("invalid options for middleware %s: source not provided", Name) + return nil, fmt.Errorf("invalid config for middleware %s: source not provided", Name) } if cfg.Configuration == nil { - return nil, fmt.Errorf("invalid options for middleware %s: options not provided", Name) + return nil, fmt.Errorf("invalid config for middleware %s: config not provided", Name) } var ds storage.DataStorage = st diff --git a/middleware/varinterval/varinterval.go b/middleware/varinterval/varinterval.go index c677c76..0587d3e 100644 --- a/middleware/varinterval/varinterval.go +++ b/middleware/varinterval/varinterval.go @@ -23,10 +23,10 @@ func init() { middleware.RegisterBuilder(Name, build) } -func build(options conf.MapConfig, _ storage.PeerStorage) (h middleware.Hook, err error) { +func build(config conf.MapConfig, _ storage.PeerStorage) (h middleware.Hook, err error) { var cfg Config - if err = options.Unmarshal(&cfg); err != nil { + if err = config.Unmarshal(&cfg); err != nil { err = fmt.Errorf("middleware %s: %w", Name, err) } else { if err := checkConfig(cfg); err == nil { diff --git a/pkg/bytepool/bufferpool.go b/pkg/bytepool/bufferpool.go new file mode 100644 index 0000000..3fdcb72 --- /dev/null +++ b/pkg/bytepool/bufferpool.go @@ -0,0 +1,27 @@ +package bytepool + +import ( + "bytes" + "sync" +) + +// ByteBufferPool is a cached pool of reusable byte buffers. +type ByteBufferPool struct { + sync.Pool +} + +// NewBufferPool allocates a new ByteBufferPool. +func NewBufferPool() *ByteBufferPool { + return &ByteBufferPool{sync.Pool{New: func() any { return new(bytes.Buffer) }}} +} + +// Get returns a bytes.Buffer from the pool. +func (bbp *ByteBufferPool) Get() *bytes.Buffer { + return bbp.Pool.Get().(*bytes.Buffer) +} + +// Put returns bytes.Buffer to the pool. +func (bbp *ByteBufferPool) Put(b *bytes.Buffer) { + b.Reset() + bbp.Pool.Put(b) +} diff --git a/frontend/udp/bytepool/bytepool.go b/pkg/bytepool/bytepool.go similarity index 86% rename from frontend/udp/bytepool/bytepool.go rename to pkg/bytepool/bytepool.go index 8482803..bb05bfa 100644 --- a/frontend/udp/bytepool/bytepool.go +++ b/pkg/bytepool/bytepool.go @@ -8,8 +8,8 @@ type BytePool struct { sync.Pool } -// New allocates a new BytePool with slices of equal length and capacity. -func New(length int) *BytePool { +// NewBytePool allocates a new BytePool with slices of equal length and capacity. +func NewBytePool(length int) *BytePool { var bp BytePool bp.Pool.New = func() any { // This avoids allocations for the slice metadata, see: diff --git a/pkg/conf/decoder.go b/pkg/conf/config.go similarity index 81% rename from pkg/conf/decoder.go rename to pkg/conf/config.go index 2f28be1..2b84325 100644 --- a/pkg/conf/decoder.go +++ b/pkg/conf/config.go @@ -51,3 +51,14 @@ func (m MapConfig) Unmarshal(into any) (err error) { } return } + +// NamedMapConfig encapsulates MapConfig with string Name +type NamedMapConfig struct { + Name string + Config MapConfig +} + +// MarshalZerologObject writes Name and Config into zerolog event +func (nm NamedMapConfig) MarshalZerologObject(e *zerolog.Event) { + e.Str("name", nm.Name).Dict("config", zerolog.Dict().EmbedObject(nm.Config)) +} diff --git a/storage/keydb/storage.go b/storage/keydb/storage.go index 3d241d3..dbb9acb 100644 --- a/storage/keydb/storage.go +++ b/storage/keydb/storage.go @@ -15,7 +15,6 @@ import ( "time" "github.com/go-redis/redis/v8" - "github.com/rs/zerolog" "github.com/sot-tech/mochi/bittorrent" "github.com/sot-tech/mochi/pkg/conf" @@ -25,14 +24,10 @@ import ( r "github.com/sot-tech/mochi/storage/redis" ) -// Name is name of this storage -const ( - Name = "keydb" - expireMemberCmd = "EXPIREMEMBER" -) +const expireMemberCmd = "EXPIREMEMBER" var ( - logger = log.NewLogger(Name) + logger = log.NewLogger("storage/keydb") // errNotKeyDB returned from initializer if connected does not support KeyDB // specific command (EXPIREMEMBER) errNotKeyDB = errors.New("provided instance seems not KeyDB") @@ -40,7 +35,7 @@ var ( func init() { // Register the storage driver. - storage.RegisterBuilder(Name, builder) + storage.RegisterDriver("keydb", builder) } func builder(icfg conf.MapConfig) (storage.PeerStorage, error) { @@ -98,16 +93,11 @@ type store struct { peerTTL uint } -// MarshalZerologObject writes configuration into zerolog event -func (s store) MarshalZerologObject(e *zerolog.Event) { - e.Str("type", Name).Object("config", s.Config) -} - -func (s store) setPeerTTL(infoHashKey, peerID string) error { +func (s *store) setPeerTTL(infoHashKey, peerID string) error { return s.Process(context.TODO(), redis.NewCmd(context.TODO(), expireMemberCmd, infoHashKey, peerID, s.peerTTL)) } -func (s store) addPeer(infoHashKey, peerID string) (err error) { +func (s *store) addPeer(infoHashKey, peerID string) (err error) { logger.Trace(). Str("infoHashKey", infoHashKey). Str("peerID", peerID). @@ -118,7 +108,7 @@ func (s store) addPeer(infoHashKey, peerID string) (err error) { return } -func (s store) delPeer(infoHashKey, peerID string) error { +func (s *store) delPeer(infoHashKey, peerID string) error { logger.Trace(). Str("infoHashKey", infoHashKey). Str("peerID", peerID). @@ -132,23 +122,23 @@ func (s store) delPeer(infoHashKey, peerID string) error { return err } -func (s store) PutSeeder(ih bittorrent.InfoHash, peer bittorrent.Peer) error { +func (s *store) PutSeeder(ih bittorrent.InfoHash, peer bittorrent.Peer) error { return s.addPeer(r.InfoHashKey(ih.RawString(), true, peer.Addr().Is6()), peer.RawString()) } -func (s store) DeleteSeeder(ih bittorrent.InfoHash, peer bittorrent.Peer) error { +func (s *store) DeleteSeeder(ih bittorrent.InfoHash, peer bittorrent.Peer) error { return s.delPeer(r.InfoHashKey(ih.RawString(), true, peer.Addr().Is6()), peer.RawString()) } -func (s store) PutLeecher(ih bittorrent.InfoHash, peer bittorrent.Peer) error { +func (s *store) PutLeecher(ih bittorrent.InfoHash, peer bittorrent.Peer) error { return s.addPeer(r.InfoHashKey(ih.RawString(), false, peer.Addr().Is6()), peer.RawString()) } -func (s store) DeleteLeecher(ih bittorrent.InfoHash, peer bittorrent.Peer) error { +func (s *store) DeleteLeecher(ih bittorrent.InfoHash, peer bittorrent.Peer) error { return s.delPeer(r.InfoHashKey(ih.RawString(), false, peer.Addr().Is6()), peer.RawString()) } -func (s store) GraduateLeecher(ih bittorrent.InfoHash, peer bittorrent.Peer) (err error) { +func (s *store) GraduateLeecher(ih bittorrent.InfoHash, peer bittorrent.Peer) (err error) { logger.Trace(). Stringer("infoHash", ih). Object("peer", peer). @@ -163,12 +153,15 @@ func (s store) GraduateLeecher(ih bittorrent.InfoHash, peer bittorrent.Peer) (er } else { err = s.addPeer(ihSeederKey, peerID) } + if err == nil { + err = s.HIncrBy(context.TODO(), r.CountDownloadsKey, infoHash, 1).Err() + } } return err } // AnnouncePeers is the same function as redis.AnnouncePeers -func (s store) AnnouncePeers(ih bittorrent.InfoHash, forSeeder bool, numWant int, v6 bool) ([]bittorrent.Peer, error) { +func (s *store) AnnouncePeers(ih bittorrent.InfoHash, forSeeder bool, numWant int, v6 bool) ([]bittorrent.Peer, error) { logger.Trace(). Stringer("infoHash", ih). Bool("forSeeder", forSeeder). @@ -182,25 +175,24 @@ func (s store) AnnouncePeers(ih bittorrent.InfoHash, forSeeder bool, numWant int } // ScrapeSwarm is the same function as redis.ScrapeSwarm except `SCard` call instead of `HLen` -func (s store) ScrapeSwarm(ih bittorrent.InfoHash) (leechers uint32, seeders uint32, snatched uint32) { +func (s *store) ScrapeSwarm(ih bittorrent.InfoHash) (uint32, uint32, uint32) { logger.Trace(). Stringer("infoHash", ih). Msg("scrape swarm") - leechers, seeders = s.CountPeers(ih, s.SCard) - return + return s.ScrapeIH(ih, s.SCard) } -func (store) GCAware() bool { +func (*store) GCAware() bool { return false } -func (store) ScheduleGC(_, _ time.Duration) {} +func (*store) ScheduleGC(_, _ time.Duration) {} -func (store) StatisticsAware() bool { +func (*store) StatisticsAware() bool { return false } -func (store) ScheduleStatisticsCollection(_ time.Duration) {} +func (*store) ScheduleStatisticsCollection(_ time.Duration) {} func (s *store) Stop() stop.Result { c := make(stop.Channel) diff --git a/storage/memory/storage.go b/storage/memory/storage.go index 16f5fad..b586e5f 100644 --- a/storage/memory/storage.go +++ b/storage/memory/storage.go @@ -9,8 +9,6 @@ import ( "sync" "time" - "github.com/rs/zerolog" - "github.com/sot-tech/mochi/bittorrent" "github.com/sot-tech/mochi/pkg/conf" "github.com/sot-tech/mochi/pkg/log" @@ -21,17 +19,13 @@ import ( ) // Default config constants. -const ( - // Name is the name by which this peer store is registered with Conf. - Name = "memory" - defaultShardCount = 1024 -) +const defaultShardCount = 1024 -var logger = log.NewLogger(Name) +var logger = log.NewLogger("storage/memory") func init() { // Register the storage driver. - storage.RegisterBuilder(Name, builder) + storage.RegisterDriver("memory", builder) } func builder(icfg conf.MapConfig) (storage.PeerStorage, error) { @@ -47,11 +41,6 @@ type Config struct { ShardCount int `cfg:"shard_count"` } -// MarshalZerologObject writes configuration into zerolog event -func (cfg Config) MarshalZerologObject(e *zerolog.Event) { - e.Int("shardCount", cfg.ShardCount) -} - // Validate sanity checks values set in a config and returns a new config with // default values replacing anything that is invalid. // @@ -110,11 +99,6 @@ type peerStore struct { wg sync.WaitGroup } -// MarshalZerologObject writes configuration into zerolog event -func (ps *peerStore) MarshalZerologObject(e *zerolog.Event) { - e.Str("type", Name).Object("config", ps.cfg) -} - var _ storage.PeerStorage = &peerStore{} func (ps *peerStore) ScheduleGC(gcInterval, peerLifeTime time.Duration) { @@ -406,7 +390,7 @@ func (ps *peerStore) AnnouncePeers(ih bittorrent.InfoHash, forSeeder bool, numWa return } -func (ps *peerStore) countPeers(ih bittorrent.InfoHash, v6 bool) (leechers uint32, seeders uint32) { +func (ps *peerStore) countPeers(ih bittorrent.InfoHash, v6 bool) (leechers, seeders uint32) { shard := ps.shards[ps.shardIndex(ih, v6)] shard.RLock() defer shard.RUnlock() diff --git a/storage/pg/storage.go b/storage/pg/storage.go index 164bbf8..7e8dd1b 100644 --- a/storage/pg/storage.go +++ b/storage/pg/storage.go @@ -15,7 +15,6 @@ import ( "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" - "github.com/rs/zerolog" "github.com/sot-tech/mochi/bittorrent" "github.com/sot-tech/mochi/pkg/conf" @@ -27,25 +26,33 @@ import ( ) const ( - // Name is the name by which this peer store is registered with Conf. - Name = "pg" - defaultPingQuery = "SELECT 0" errRequiredParameterNotSetMsg = "required parameter not provided: %s" errRequiredColumnsNotFoundMsg = "one or more required columns not found in result set: %v" errRollBackMsg = "error occurred while rolling back failed query: %v, failed query error: %v" + + pCtx = "context" + pKey = "key" + pValue = "value" + pInfoHash = "info_hash" + pPeerID = "peer_id" + pAddress = "address" + pPort = "port" + pV6 = "is_v6" + pSeeder = "is_seeder" + pCreated = "created" + pCount = "count" ) var ( - logger = log.NewLogger(Name) - + logger = log.NewLogger("storage/pg") errConnectionStringNotProvided = errors.New("database connection string not provided") ) func init() { // Register the storage builder. - storage.RegisterBuilder(Name, builder) + storage.RegisterDriver("pg", builder) } func builder(icfg conf.MapConfig) (storage.PeerStorage, error) { @@ -95,39 +102,23 @@ type dataQueryConf struct { DelQuery string `cfg:"del_query"` } +type downloadQueryConf struct { + GetQuery string `cfg:"get_query"` + IncrementQuery string `cfg:"inc_query"` +} + // Config holds the configuration of a redis PeerStorage. type Config struct { ConnectionString string `cfg:"connection_string"` PingQuery string `cfg:"ping_query"` Peer peerQueryConf Announce announceQueryConf + Downloads downloadQueryConf Data dataQueryConf GCQuery string `cfg:"gc_query"` InfoHashCountQuery string `cfg:"info_hash_count_query"` } -// MarshalZerologObject writes configuration fields into zerolog event -func (cfg Config) MarshalZerologObject(e *zerolog.Event) { - e.Str("connectionString", ""). - Str("pingQuery", cfg.PingQuery). - Str("peer.addQuery", cfg.Peer.AddQuery). - Str("peer.delQuery", cfg.Peer.DelQuery). - Str("peer.graduateQuery", cfg.Peer.GraduateQuery). - Str("peer.countQuery", cfg.Peer.CountQuery). - Str("peer.countSeedersColumn", cfg.Peer.CountSeedersColumn). - Str("peer.countLeechersColumn", cfg.Peer.CountLeechersColumn). - Str("peer.byInfoHashClause", cfg.Peer.ByInfoHashClause). - Str("announce.query", cfg.Announce.Query). - Str("announce.peerIDColumn", cfg.Announce.PeerIDColumn). - Str("announce.addressColumn", cfg.Announce.AddressColumn). - Str("announce.portColumn", cfg.Announce.PortColumn). - Str("data.addQuery", cfg.Data.AddQuery). - Str("data.getQuery", cfg.Data.GetQuery). - Str("data.delQuery", cfg.Data.DelQuery). - Str("gcQuery", cfg.GCQuery). - Str("infoHashCountQuery", cfg.InfoHashCountQuery) -} - // Validate sanity checks values set in a config and returns a new config with // default values replacing anything that is invalid. // @@ -228,18 +219,13 @@ type store struct { closed chan any } -func (s *store) Put(ctx string, values ...storage.Entry) (err error) { +func (s *store) txBatch(ctx context.Context, batch *pgx.Batch) (err error) { var tx pgx.Tx - if tx, err = s.Begin(context.TODO()); err == nil { - for _, v := range values { - if _, err = tx.Exec(context.TODO(), s.Data.AddQuery, ctx, []byte(v.Key), v.Value); err != nil { - break - } - } - if err == nil { - err = tx.Commit(context.TODO()) + if tx, err = s.Begin(ctx); err == nil { + if err = tx.SendBatch(context.TODO(), batch).Close(); err == nil { + err = tx.Commit(ctx) } else { - if txErr := tx.Rollback(context.TODO()); txErr != nil { + if txErr := tx.Rollback(ctx); txErr != nil { err = fmt.Errorf(errRollBackMsg, txErr, err) } } @@ -247,9 +233,25 @@ func (s *store) Put(ctx string, values ...storage.Entry) (err error) { return } +func (s *store) Put(ctx string, values ...storage.Entry) (err error) { + switch len(values) { + case 0: + // ignore + case 1: + _, err = s.Exec(context.TODO(), s.Data.AddQuery, pgx.NamedArgs{pCtx: ctx, pKey: []byte(values[0].Key), pValue: values[0].Value}) + default: + var batch pgx.Batch + for _, v := range values { + batch.Queue(s.Data.AddQuery, pgx.NamedArgs{pCtx: ctx, pKey: []byte(v.Key), pValue: v.Value}) + } + err = s.txBatch(context.TODO(), &batch) + } + return +} + func (s *store) Contains(ctx string, key string) (contains bool, err error) { var rows pgx.Rows - if rows, err = s.Query(context.TODO(), s.Data.GetQuery, ctx, []byte(key)); err == nil { + if rows, err = s.Query(context.TODO(), s.Data.GetQuery, pgx.NamedArgs{pCtx: ctx, pKey: []byte(key)}); err == nil { defer rows.Close() contains = rows.Next() err = rows.Err() @@ -258,28 +260,19 @@ func (s *store) Contains(ctx string, key string) (contains bool, err error) { } func (s *store) Load(ctx string, key string) (out []byte, err error) { - row := s.QueryRow(context.TODO(), s.Data.GetQuery, ctx, []byte(key)) - if err = row.Scan(&out); errors.Is(err, pgx.ErrNoRows) { + if err = s.QueryRow(context.TODO(), s.Data.GetQuery, pgx.NamedArgs{pCtx: ctx, pKey: []byte(key)}).Scan(&out); errors.Is(err, pgx.ErrNoRows) { err = nil } return } func (s *store) Delete(ctx string, keys ...string) (err error) { - var tx pgx.Tx - if tx, err = s.Begin(context.TODO()); err == nil { - for _, k := range keys { - if _, err = tx.Exec(context.TODO(), s.Data.DelQuery, ctx, []byte(k)); err != nil { - break - } - } - if err == nil { - err = tx.Commit(context.TODO()) - } else { - if txErr := tx.Rollback(context.TODO()); txErr != nil { - err = fmt.Errorf(errRollBackMsg, txErr, err) - } + if len(keys) > 0 { + baKeys := make([][]byte, len(keys)) + for i, k := range keys { + baKeys[i] = []byte(k) } + _, err = s.Exec(context.TODO(), s.Data.DelQuery, pgx.NamedArgs{pCtx: ctx, pKey: baKeys}) } return } @@ -304,7 +297,7 @@ func (s *store) ScheduleGC(gcInterval, peerLifeTime time.Duration) { return case <-t.C: start := time.Now() - _, err := s.Exec(context.Background(), s.GCQuery, time.Now().Add(-peerLifeTime)) + _, err := s.Exec(context.Background(), s.GCQuery, pgx.NamedArgs{pCreated: time.Now().Add(-peerLifeTime)}) duration := time.Since(start) if err != nil { logger.Error().Err(err).Msg("error occurred while GC") @@ -335,10 +328,9 @@ func (s *store) ScheduleStatisticsCollection(reportInterval time.Duration) { case <-t.C: if metrics.Enabled() { before := time.Now() - sc, lc := s.countPeers(bittorrent.NoneInfoHash) + sc, lc := s.countPeers(nil) var hc int - row := s.QueryRow(context.Background(), s.InfoHashCountQuery) - if err := row.Scan(&hc); err != nil && !errors.Is(err, pgx.ErrNoRows) { + if err := s.QueryRow(context.Background(), s.InfoHashCountQuery).Scan(&hc); err != nil && !errors.Is(err, pgx.ErrNoRows) { logger.Error().Err(err).Msg("error occurred while get info hash count") } @@ -352,27 +344,40 @@ func (s *store) ScheduleStatisticsCollection(reportInterval time.Duration) { }() } -func (s *store) putPeer(ih bittorrent.InfoHash, peer bittorrent.Peer, seeder bool) error { +func (s *store) putPeer(ih bittorrent.InfoHash, peer bittorrent.Peer, seeder bool) (err error) { logger.Trace(). Stringer("infoHash", ih). Object("peer", peer). Bool("seeder", seeder). Msg("put peer") - args := []any{[]byte(ih), peer.ID[:], net.IP(peer.Addr().AsSlice()), peer.Port(), seeder, peer.Addr().Is6()} - if s.GCAware() { - args = append(args, timecache.Now()) + args := pgx.NamedArgs{ + pInfoHash: []byte(ih), + pPeerID: peer.ID[:], + pAddress: net.IP(peer.Addr().AsSlice()), + pPort: peer.Port(), + pSeeder: seeder, + pV6: peer.Addr().Is6(), } - _, err := s.Exec(context.TODO(), s.Peer.AddQuery, args...) - return err + if s.GCAware() { + args[pCreated] = timecache.Now() + } + _, err = s.Exec(context.TODO(), s.Peer.AddQuery, args) + return } -func (s *store) delPeer(ih bittorrent.InfoHash, peer bittorrent.Peer, seeder bool) error { +func (s *store) delPeer(ih bittorrent.InfoHash, peer bittorrent.Peer, seeder bool) (err error) { logger.Trace(). Stringer("infoHash", ih). Object("peer", peer). Msg("del peer") - _, err := s.Exec(context.TODO(), s.Peer.DelQuery, []byte(ih), peer.ID[:], net.IP(peer.Addr().AsSlice()), peer.Port(), seeder) - return err + _, err = s.Exec(context.TODO(), s.Peer.DelQuery, pgx.NamedArgs{ + pInfoHash: []byte(ih), + pPeerID: peer.ID[:], + pAddress: net.IP(peer.Addr().AsSlice()), + pPort: peer.Port(), + pSeeder: seeder, + }) + return } func (s *store) PutSeeder(ih bittorrent.InfoHash, peer bittorrent.Peer) error { @@ -396,14 +401,26 @@ func (s *store) GraduateLeecher(ih bittorrent.InfoHash, peer bittorrent.Peer) er Stringer("infoHash", ih). Object("peer", peer). Msg("graduate leecher") - _, err := s.Exec(context.TODO(), s.Peer.GraduateQuery, []byte(ih), peer.ID[:], net.IP(peer.Addr().AsSlice()), peer.Port()) - return err + var batch pgx.Batch + ihb := []byte(ih) + batch.Queue(s.Peer.GraduateQuery, pgx.NamedArgs{ + pInfoHash: ihb, + pPeerID: peer.ID[:], + pAddress: net.IP(peer.Addr().AsSlice()), + pPort: peer.Port(), + }) + batch.Queue(s.Downloads.IncrementQuery, pgx.NamedArgs{pInfoHash: ihb}) + return s.txBatch(context.TODO(), &batch) } func (s *store) getPeers(ih bittorrent.InfoHash, seeders bool, maxCount int, isV6 bool) (peers []bittorrent.Peer, err error) { var rows pgx.Rows - // TODO: see https://github.com/jackc/pgx/issues/387#issuecomment-1107666716 - if rows, err = s.Query(context.TODO(), s.Announce.Query, []byte(ih), seeders, isV6, maxCount); err == nil { + if rows, err = s.Query(context.TODO(), s.Announce.Query, pgx.NamedArgs{ + pInfoHash: []byte(ih), + pSeeder: seeders, + pV6: isV6, + pCount: maxCount, + }); err == nil { defer rows.Close() idIndex, ipIndex, portIndex := -1, -1, -1 for i, field := range rows.FieldDescriptions() { @@ -418,7 +435,11 @@ func (s *store) getPeers(ih bittorrent.InfoHash, seeders bool, maxCount int, isV } } if idIndex < 0 || ipIndex < 0 || portIndex < 0 { - err = fmt.Errorf(errRequiredColumnsNotFoundMsg, []string{s.Announce.PeerIDColumn, s.Announce.AddressColumn, s.Announce.PortColumn}) + err = fmt.Errorf(errRequiredColumnsNotFoundMsg, []string{ + s.Announce.PeerIDColumn, + s.Announce.AddressColumn, + s.Announce.PortColumn, + }) return } var maxIndex int @@ -492,13 +513,13 @@ func (s *store) AnnouncePeers(ih bittorrent.InfoHash, forSeeder bool, numWant in return } -func (s *store) countPeers(ih bittorrent.InfoHash) (seeders int, leechers int) { +func (s *store) countPeers(ih []byte) (seeders uint32, leechers uint32) { var rows pgx.Rows var err error - if ih == bittorrent.NoneInfoHash { + if len(ih) == 0 { rows, err = s.Query(context.TODO(), s.Peer.CountQuery) } else { - rows, err = s.Query(context.TODO(), s.Peer.CountQuery+" "+s.Peer.ByInfoHashClause, []byte(ih)) + rows, err = s.Query(context.TODO(), s.Peer.CountQuery+" "+s.Peer.ByInfoHashClause, pgx.NamedArgs{pInfoHash: ih}) } if err == nil { defer rows.Close() @@ -530,7 +551,7 @@ func (s *store) countPeers(ih bittorrent.InfoHash) (seeders int, leechers int) { } } if err != nil { - logger.Error().Err(err).Stringer("infoHash", ih).Msg("unable to get peers count") + logger.Error().Err(err).Bytes("infoHash", ih).Msg("unable to get peers count") } return } @@ -539,8 +560,14 @@ func (s *store) ScrapeSwarm(ih bittorrent.InfoHash) (leechers uint32, seeders ui logger.Trace(). Stringer("infoHash", ih). Msg("scrape swarm") - sc, lc := s.countPeers(ih) - seeders, leechers = uint32(sc), uint32(lc) + ihb := []byte(ih) + seeders, leechers = s.countPeers(ihb) + if len(s.Downloads.GetQuery) > 0 { + if err := s.QueryRow(context.TODO(), s.Downloads.GetQuery, pgx.NamedArgs{pInfoHash: ihb}).Scan(&snatched); err != nil && !errors.Is(err, pgx.ErrNoRows) { + logger.Error().Stringer("infoHash", ih).Err(err).Msg("error occurred while get info downloads count") + } + } + return } @@ -565,7 +592,3 @@ func (s *store) Stop() stop.Result { }() return c.Result() } - -func (s *store) MarshalZerologObject(e *zerolog.Event) { - e.Str("type", Name).Object("config", s.Config) -} diff --git a/storage/pg/storage_test.go b/storage/pg/storage_test.go index bac4805..bab82de 100644 --- a/storage/pg/storage_test.go +++ b/storage/pg/storage_test.go @@ -26,6 +26,12 @@ CREATE TABLE mo_peers ( CREATE INDEX mo_peers_created_idx ON mo_peers(created); CREATE INDEX mo_peers_announce_idx ON mo_peers(info_hash, is_seeder, is_v6); +DROP TABLE IF EXISTS mo_downloads; +CREATE TABLE mo_downloads ( + info_hash bytea PRIMARY KEY NOT NULL, + downloads int NOT NULL DEFAULT 1 +); + DROP TABLE IF EXISTS mo_kv; CREATE TABLE mo_kv ( context varchar NOT NULL, @@ -40,26 +46,30 @@ var cfg = Config{ ConnectionString: "host=127.0.0.1 database=test user=postgres pool_max_conns=50", PingQuery: "SELECT 1", Peer: peerQueryConf{ - AddQuery: "INSERT INTO mo_peers VALUES($1, $2, $3, $4, $5, $6, $7) ON CONFLICT (info_hash, peer_id, address, port) DO UPDATE SET created = EXCLUDED.created, is_seeder = EXCLUDED.is_seeder", - DelQuery: "DELETE FROM mo_peers WHERE info_hash=$1 AND peer_id=$2 AND address=$3 AND port=$4 AND is_seeder=$5", - GraduateQuery: "UPDATE mo_peers SET is_seeder=TRUE WHERE info_hash=$1 AND peer_id=$2 AND address=$3 AND port=$4 AND NOT is_seeder", + AddQuery: "INSERT INTO mo_peers VALUES(@info_hash, @peer_id, @address, @port, @is_seeder, @is_v6, @created) ON CONFLICT (info_hash, peer_id, address, port) DO UPDATE SET created = EXCLUDED.created, is_seeder = EXCLUDED.is_seeder", + DelQuery: "DELETE FROM mo_peers WHERE info_hash=@info_hash AND peer_id=@peer_id AND address=@address AND port=@port AND is_seeder=@is_seeder", + GraduateQuery: "UPDATE mo_peers SET is_seeder=TRUE WHERE info_hash=@info_hash AND peer_id=peer_id AND address=@address AND port=@port AND NOT is_seeder", CountQuery: "SELECT COUNT(1) FILTER (WHERE is_seeder) AS seeders, COUNT(1) FILTER (WHERE NOT is_seeder) AS leechers FROM mo_peers", CountSeedersColumn: "seeders", CountLeechersColumn: "leechers", - ByInfoHashClause: "WHERE info_hash = $1", + ByInfoHashClause: "WHERE info_hash = @info_hash", }, Announce: announceQueryConf{ - Query: "SELECT peer_id, address, port FROM mo_peers WHERE info_hash=$1 AND is_seeder=$2 AND is_v6=$3 LIMIT $4", + Query: "SELECT peer_id, address, port FROM mo_peers WHERE info_hash=@info_hash AND is_seeder=@is_seeder AND is_v6=@is_v6 LIMIT @count", PeerIDColumn: "peer_id", AddressColumn: "address", PortColumn: "port", }, - Data: dataQueryConf{ - AddQuery: "INSERT INTO mo_kv VALUES($1, $2, $3) ON CONFLICT (context, name) DO NOTHING", - GetQuery: "SELECT value FROM mo_kv WHERE context=$1 AND name=$2", - DelQuery: "DELETE FROM mo_kv WHERE context=$1 AND name=$2", + Downloads: downloadQueryConf{ + GetQuery: "SELECT downloads FROM mo_downloads where info_hash=@info_hash", + IncrementQuery: "INSERT INTO mo_downloads VALUES(@info_hash) ON CONFLICT(info_hash) DO UPDATE SET downloads = mo_downloads.downloads + 1", }, - GCQuery: "DELETE FROM mo_peers WHERE created <= $1", + Data: dataQueryConf{ + AddQuery: "INSERT INTO mo_kv VALUES(@context, @key, @value) ON CONFLICT (context, name) DO NOTHING", + GetQuery: "SELECT value FROM mo_kv WHERE context=@context AND name=@key", + DelQuery: "DELETE FROM mo_kv WHERE context=@context AND name = ANY(@key)", + }, + GCQuery: "DELETE FROM mo_peers WHERE created <= @created", InfoHashCountQuery: "SELECT COUNT(DISTINCT info_hash) as info_hashes FROM mo_peers", } diff --git a/storage/redis/storage.go b/storage/redis/storage.go index 78d25a0..f7c26bf 100644 --- a/storage/redis/storage.go +++ b/storage/redis/storage.go @@ -1,6 +1,6 @@ // Package redis implements the storage interface. // BitTorrent tracker keeping peer data in redis with hash. -// There two categories of hash: +// There three categories of hash: // // - CHI_{L,S}{4,6}_ (hash type) // To save peers that hold the infohash, used for fast searching, @@ -10,6 +10,9 @@ // To save all the infohashes, used for garbage collection, // metrics aggregation and leecher graduation // +// - CHI_D (hash type) +// To record the number of torrent downloads. +// // Two keys are used to record the count of seeders and leechers. // // - CHI_C_S (key type) @@ -28,7 +31,6 @@ import ( "time" "github.com/go-redis/redis/v8" - "github.com/rs/zerolog" "github.com/sot-tech/mochi/bittorrent" "github.com/sot-tech/mochi/pkg/conf" @@ -40,8 +42,6 @@ import ( ) const ( - // Name is the name by which this peer store is registered with Conf. - Name = "redis" // Default config constants. defaultRedisAddress = "127.0.0.1:6379" defaultReadTimeout = time.Second * 15 @@ -63,17 +63,19 @@ const ( CountSeederKey = "CHI_C_S" // CountLeecherKey redis key for leecher count CountLeecherKey = "CHI_C_L" + // CountDownloadsKey redis key for snatches (downloads) count + CountDownloadsKey = "CHI_D" ) var ( - logger = log.NewLogger(Name) + logger = log.NewLogger("storage/redis") // errSentinelAndClusterChecked returned from initializer if both Config.Sentinel and Config.Cluster provided errSentinelAndClusterChecked = errors.New("unable to use both cluster and sentinel mode") ) func init() { // Register the storage builder. - storage.RegisterBuilder(Name, builder) + storage.RegisterDriver("redis", builder) } func builder(icfg conf.MapConfig) (storage.PeerStorage, error) { @@ -121,20 +123,6 @@ type Config struct { ConnectTimeout time.Duration `cfg:"connect_timeout"` } -// MarshalZerologObject writes configuration fields into zerolog event -func (cfg Config) MarshalZerologObject(e *zerolog.Event) { - e.Strs("addresses", cfg.Addresses). - Int("db", cfg.DB). - Int("poolSize", cfg.PoolSize). - Bool("sentinel", cfg.Sentinel). - Str("sentinelMaster", cfg.SentinelMaster). - Bool("cluster", cfg.Cluster). - Dur("readTimeout", cfg.ReadTimeout). - Dur("writeTimeout", cfg.WriteTimeout). - Dur("connectTimeout", cfg.ConnectTimeout). - Dur("peerLifetime", cfg.PeerLifetime) -} - // Validate sanity checks values set in a config and returns a new config with // default values replacing anything that is invalid. // @@ -238,13 +226,7 @@ func (cfg Config) Connect() (con Connection, err error) { _ = rs.Close() rs = nil } - cfg.Login, cfg.Password = "", "" - return Connection{rs, cfg}, err -} - -// MarshalZerologObject writes configuration into zerolog event -func (ps *store) MarshalZerologObject(e *zerolog.Event) { - e.Str("type", Name).Object("config", ps.Config) + return Connection{rs}, err } func (ps *store) ScheduleGC(gcInterval, peerLifeTime time.Duration) { @@ -301,7 +283,6 @@ func (ps *store) ScheduleStatisticsCollection(reportInterval time.Duration) { // Connection is wrapper for redis.UniversalClient type Connection struct { redis.UniversalClient - Config } type store struct { @@ -454,6 +435,9 @@ func (ps *store) GraduateLeecher(ih bittorrent.InfoHash, peer bittorrent.Peer) e if err == nil { err = tx.SAdd(context.TODO(), IHKey, ihSeederKey).Err() } + if err == nil { + err = tx.HIncrBy(context.TODO(), CountDownloadsKey, infoHash, 1).Err() + } return err }) } @@ -537,26 +521,28 @@ func (ps *Connection) countPeers(infoHashKey string, countFn getPeerCountFn) uin return uint32(count) } -// CountPeers calls provided countFn and returns seeders and leechers count for specified info hash -func (ps *Connection) CountPeers(ih bittorrent.InfoHash, countFn getPeerCountFn) (leechersCount, seedersCount uint32) { +// ScrapeIH calls provided countFn and returns seeders, leechers and downloads count for specified info hash +func (ps *Connection) ScrapeIH(ih bittorrent.InfoHash, countFn getPeerCountFn) (leechersCount, seedersCount, downloadsCount uint32) { infoHash := ih.RawString() leechersCount = ps.countPeers(InfoHashKey(infoHash, false, false), countFn) + ps.countPeers(InfoHashKey(infoHash, false, true), countFn) seedersCount = ps.countPeers(InfoHashKey(infoHash, true, false), countFn) + ps.countPeers(InfoHashKey(infoHash, true, true), countFn) + d, err := ps.HGet(context.TODO(), CountDownloadsKey, infoHash).Uint64() + if err = AsNil(err); err != nil { + logger.Error().Err(err).Str("infoHash", infoHash).Msg("downloads count calculation failure") + } + downloadsCount = uint32(d) return } -func (ps *store) ScrapeSwarm(ih bittorrent.InfoHash) (leechers uint32, seeders uint32, snatched uint32) { +func (ps *store) ScrapeSwarm(ih bittorrent.InfoHash) (uint32, uint32, uint32) { logger.Trace(). Stringer("infoHash", ih). Msg("scrape swarm") - - leechers, seeders = ps.CountPeers(ih, ps.HLen) - - return + return ps.ScrapeIH(ih, ps.HLen) } const argNumErrorMsg = "ERR wrong number of arguments" @@ -621,7 +607,7 @@ func (ps *Connection) Delete(ctx string, keys ...string) (err error) { } // Preservable - storage.DataStorage implementation -func (Connection) Preservable() bool { +func (*Connection) Preservable() bool { return true } diff --git a/storage/storage.go b/storage/storage.go index 3fca68b..01628cd 100644 --- a/storage/storage.go +++ b/storage/storage.go @@ -3,12 +3,10 @@ package storage import ( - "errors" + "fmt" "sync" "time" - "github.com/rs/zerolog" - "github.com/sot-tech/mochi/bittorrent" "github.com/sot-tech/mochi/pkg/conf" "github.com/sot-tech/mochi/pkg/log" @@ -25,9 +23,9 @@ const ( ) var ( - logger = log.NewLogger("storage configurator") - driversM sync.RWMutex - drivers = make(map[string]Builder) + logger = log.NewLogger("storage") + driversMU sync.RWMutex + drivers = make(map[string]Driver) ) // Config holds configuration for periodic execution tasks, which may or may not implement @@ -83,19 +81,14 @@ type Entry struct { Value []byte } -// Builder is the function used to initialize a new type of PeerStorage. -type Builder func(cfg conf.MapConfig) (PeerStorage, error) +// Driver is the function used to initialize a new PeerStorage +// with provided configuration. +type Driver func(conf.MapConfig) (PeerStorage, error) -var ( - // ErrResourceDoesNotExist is the error returned by all delete methods and the - // AnnouncePeers method of the PeerStorage interface if the requested resource - // does not exist. - ErrResourceDoesNotExist = bittorrent.ClientError("resource does not exist") - - // ErrDriverDoesNotExist is the error returned by NewStorage when a peer - // store driver with that name does not exist. - ErrDriverDoesNotExist = errors.New("peer store driver with that name does not exist") -) +// ErrResourceDoesNotExist is the error returned by all delete methods and the +// AnnouncePeers method of the PeerStorage interface if the requested resource +// does not exist. +var ErrResourceDoesNotExist = bittorrent.ClientError("resource does not exist") // DataStorage is the interface, used for implementing store for arbitrary data type DataStorage interface { @@ -216,86 +209,83 @@ type PeerStorage interface { // Stopper is an interface that expects a Stop method to stop the PeerStorage. // For more details see the documentation in the stop package. stop.Stopper - - // LogObjectMarshaler returns a loggable version of the data used to configure and - // operate a particular PeerStorage. - zerolog.LogObjectMarshaler } -// RegisterBuilder makes a Builder available by the provided name. +// RegisterDriver makes a Driver available by the provided name. // // If called twice with the same name, the name is blank, or if the provided // Driver is nil, this function panics. -func RegisterBuilder(name string, b Builder) { +func RegisterDriver(name string, d Driver) { if name == "" { - panic("storage: could not register a Builder with an empty name") + panic("storage: could not register a Driver with an empty name") } - if b == nil { - panic("storage: could not register a nil Builder") + if d == nil { + panic("storage: could not register a nil Driver") } - driversM.Lock() - defer driversM.Unlock() + driversMU.Lock() + defer driversMU.Unlock() if _, dup := drivers[name]; dup { - panic("storage: RegisterBuilder called twice for " + name) + panic("storage: RegisterDriver called twice for " + name) } - drivers[name] = b + drivers[name] = d } // NewStorage attempts to initialize a new PeerStorage instance from -// the list of registered Drivers. -// -// If a builder does not exist, returns ErrDriverDoesNotExist. -func NewStorage(name string, cfg conf.MapConfig) (ps PeerStorage, err error) { - driversM.RLock() - defer driversM.RUnlock() +// the list of registered drivers. +func NewStorage(cfg conf.NamedMapConfig) (ps PeerStorage, err error) { + driversMU.RLock() + defer driversMU.RUnlock() + logger.Debug().Object("cfg", cfg).Msg("staring storage") - var b Builder - b, ok := drivers[name] + var b Driver + b, ok := drivers[cfg.Name] if !ok { - return nil, ErrDriverDoesNotExist + return nil, fmt.Errorf("storage with name '%s' does not exists", cfg.Name) } c := new(Config) - if err = cfg.Unmarshal(c); err != nil { + if err = cfg.Config.Unmarshal(c); err != nil { return } - if ps, err = b(cfg); err != nil { + if ps, err = b(cfg.Config); err != nil { return } if gc := ps.GCAware(); gc { gcInterval, peerTTL := c.sanitizeGCConfig() logger.Info(). - Str("type", name). + Str("name", cfg.Name). Dur("gcInterval", gcInterval). Dur("peerTTL", peerTTL). Msg("scheduling GC") ps.ScheduleGC(gcInterval, peerTTL) } else { logger.Debug(). - Str("type", name). + Str("name", cfg.Name). Msg("storage does not support GC") } if st := ps.StatisticsAware(); st { if statInterval := c.sanitizeStatisticsConfig(); statInterval > 0 { logger.Info(). - Str("type", name). + Str("name", cfg.Name). Dur("statInterval", statInterval). Msg("scheduling statistics collection") ps.ScheduleStatisticsCollection(statInterval) } else { - logger.Info().Str("type", name).Msg("statistics collection disabled because of zero reporting interval") + logger.Info().Str("name", cfg.Name).Msg("statistics collection disabled because of zero reporting interval") } } else { logger.Debug(). - Str("type", name). + Str("name", cfg.Name). Msg("storage does not support statistics collection") } + logger.Info().Str("name", cfg.Name).Msg("storage started") + return }