Mirror/mochi
1
0
Fork 0
mirror of https://github.com/sot-tech/mochi.git synced 2026-06-12 15:53:32 -07:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #20 from sot-tech/pg_named

Browse Source 
Add named parameters in SQL queries, refactor config file
This commit is contained in:
SOT-TECH
2022-10-27 03:50:50 +03:00
committed by GitHub
parent 640c6c605b 352c09031b
commit c3fbd49ffa
39 changed files with 1298 additions and 1169 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/build.yaml
+2 -5
View File
@@ -82,13 +82,10 @@ jobs:
run: |
go install ./cmd/mochi
go install ./cmd/mochi-e2e
curl -LO https://github.com/jzelinskie/faq/releases/download/0.0.6/faq-linux-amd64
chmod +x faq-linux-amd64
./faq-linux-amd64 '.mochi.storage = {"config":{"gc_interval":"3m","peer_lifetime":"31m","prometheus_reporting_interval":"1s","connect_timeout":"15s","read_timeout":"15s","write_timeout":"15s"},"name":"redis"}' ./dist/example_config.yaml > ./dist/example_redis_config.yaml
cat ./dist/example_redis_config.yaml
cat ./dist/example_config_redis.yaml
- name: "Run end-to-end tests"
run: |
mochi --config=./dist/example_redis_config.yaml --logLevel debug --logPretty &
mochi --config=./dist/example_config_redis.yaml --logLevel debug --logPretty &
pid=$!
sleep 2
mochi-e2e
bittorrent/bittorrent.go → bittorrent/peer.go
-16
View File
@@ -117,22 +117,6 @@ func (i InfoHash) RawString() string {
return string(i)
}
// Scrape represents the state of a swarm that is returned in a scrape response.
type Scrape struct {
InfoHash InfoHash
Snatches uint32
Complete uint32
Incomplete uint32
}
// MarshalZerologObject writes fields into zerolog event
func (s Scrape) MarshalZerologObject(e *zerolog.Event) {
e.Stringer("infoHash", s.InfoHash).
Uint32("snatches", s.Snatches).
Uint32("complete", s.Complete).
Uint32("incomplete", s.Incomplete)
}
// Peer represents the connection details of a peer that is returned in an
// announce response.
type Peer struct {
bittorrent/request.go
+51 -19
View File
@@ -35,19 +35,19 @@ func (a RequestAddress) MarshalZerologObject(e *zerolog.Event) {
// connection information about peer
type RequestAddresses []RequestAddress
func (aa RequestAddresses) Len() int {
return len(aa)
func (aa *RequestAddresses) Len() int {
return len(*aa)
}
// Less returns true only if i-th RequestAddress is marked as
// RequestAddress.Provided and j-th is not (provided address has
// higher priority)
func (aa RequestAddresses) Less(i, j int) bool {
return aa[i].Provided && !aa[j].Provided
func (aa *RequestAddresses) Less(i, j int) bool {
return (*aa)[i].Provided && !(*aa)[j].Provided
}
func (aa RequestAddresses) Swap(i, j int) {
aa[i], aa[j] = aa[j], aa[i]
func (aa *RequestAddresses) Swap(i, j int) {
(*aa)[i], (*aa)[j] = (*aa)[j], (*aa)[i]
}
// Add checks if provided RequestAddress is valid and adds unmapped
@@ -82,25 +82,27 @@ func (aa *RequestAddresses) Sanitize(ignorePrivate bool) bool {
*aa = append(*aa, RequestAddress{a, p})
}
if len(*aa) > 1 {
sort.Sort(*aa)
sort.Sort(aa)
}
return len(uniqueAddresses) > 0
}
// GetFirst returns first address from array
// or empty netip.Addr if array is empty
func (aa RequestAddresses) GetFirst() netip.Addr {
func (aa *RequestAddresses) GetFirst() netip.Addr {
var a netip.Addr
if len(aa) > 0 {
a = aa[0].Addr
if len(*aa) > 0 {
a = (*aa)[0].Addr
}
return a
}
// MarshalZerologArray writes array elements to zerolog event
func (aa RequestAddresses) MarshalZerologArray(a *zerolog.Array) {
for _, addr := range aa {
a.Object(addr)
func (aa *RequestAddresses) MarshalZerologArray(a *zerolog.Array) {
if aa != nil {
for _, addr := range *aa {
a.Object(addr)
}
}
}
@@ -137,7 +139,7 @@ func (rp RequestPeer) Peers() (peers Peers) {
// MarshalZerologObject writes fields into zerolog event
func (rp RequestPeer) MarshalZerologObject(e *zerolog.Event) {
e.Stringer("id", rp.ID).
Array("addresses", rp.RequestAddresses).
Array("addresses", &rp.RequestAddresses).
Uint16("port", rp.Port)
}
@@ -216,18 +218,48 @@ type ScrapeRequest struct {
// MarshalZerologObject writes fields into zerolog event
func (r ScrapeRequest) MarshalZerologObject(e *zerolog.Event) {
e.Array("addresses", r.RequestAddresses).
e.Array("addresses", &r.RequestAddresses).
Array("infoHashes", r.InfoHashes).
Object("params", r.Params)
}
// Scrape represents the state of a swarm that is returned in a scrape response.
type Scrape struct {
InfoHash InfoHash
Snatches uint32
Complete uint32
Incomplete uint32
}
// MarshalZerologObject writes fields into zerolog event
func (s Scrape) MarshalZerologObject(e *zerolog.Event) {
e.Stringer("infoHash", s.InfoHash).
Uint32("snatches", s.Snatches).
Uint32("complete", s.Complete).
Uint32("incomplete", s.Incomplete)
}
// Scrapes wrapper of array of Scrape-s
type Scrapes []Scrape
func (s *Scrapes) Len() int {
return len(*s)
}
func (s *Scrapes) Less(i, j int) bool {
return (*s)[i].InfoHash < (*s)[j].InfoHash
}
func (s *Scrapes) Swap(i, j int) {
(*s)[i], (*s)[j] = (*s)[j], (*s)[i]
}
// MarshalZerologArray writes array elements to zerolog event
func (s Scrapes) MarshalZerologArray(a *zerolog.Array) {
for _, scrape := range s {
a.Object(scrape)
func (s *Scrapes) MarshalZerologArray(a *zerolog.Array) {
if s != nil {
for _, scrape := range *s {
a.Object(scrape)
}
}
}
@@ -241,5 +273,5 @@ type ScrapeResponse struct {
// MarshalZerologObject writes fields into zerolog event
func (sr ScrapeResponse) MarshalZerologObject(e *zerolog.Event) {
e.Array("scrapes", sr.Files)
e.Array("scrapes", &sr.Files)
}
bittorrent/sanitize.go
+1 -1
View File
@@ -5,7 +5,7 @@ import (
)
var (
logger = log.NewLogger("request sanitizer")
logger = log.NewLogger("bittorrent/sanitize")
// ErrInvalidIP indicates an invalid IP for an Announce.
ErrInvalidIP = ClientError("invalid IP")
cmd/mochi/config.go
+15 -20
View File
@@ -9,7 +9,11 @@ import (
"github.com/sot-tech/mochi/pkg/conf"
// Imports to register middleware drivers.
// Imports to register frontends
_ "github.com/sot-tech/mochi/frontend/http"
_ "github.com/sot-tech/mochi/frontend/udp"
// Imports to register middleware hooks.
_ "github.com/sot-tech/mochi/middleware/clientapproval"
_ "github.com/sot-tech/mochi/middleware/jwt"
_ "github.com/sot-tech/mochi/middleware/torrentapproval"
@@ -29,29 +33,20 @@ type Config struct {
// We can make Conf extensible enough that you can program a new response
// generator at the cost of making it possible for users to create config that
// won't compose a functional tracker.
AnnounceInterval time.Duration `yaml:"announce_interval"`
MinAnnounceInterval time.Duration `yaml:"min_announce_interval"`
MetricsAddr string `yaml:"metrics_addr"`
HTTPConfig conf.MapConfig `yaml:"http"`
UDPConfig conf.MapConfig `yaml:"udp"`
Storage struct {
Name string `yaml:"name"`
Config conf.MapConfig `yaml:"config"`
} `yaml:"storage"`
PreHooks []conf.MapConfig `yaml:"prehooks"`
PostHooks []conf.MapConfig `yaml:"posthooks"`
AnnounceInterval time.Duration `yaml:"announce_interval"`
MinAnnounceInterval time.Duration `yaml:"min_announce_interval"`
MetricsAddr string `yaml:"metrics_addr"`
Frontends []conf.NamedMapConfig `yaml:"frontends"`
Storage conf.NamedMapConfig `yaml:"storage"`
PreHooks []conf.NamedMapConfig `yaml:"prehooks"`
PostHooks []conf.NamedMapConfig `yaml:"posthooks"`
}
// ConfigFile represents a namespaced YAML configation file.
type ConfigFile struct {
Conf Config `yaml:"mochi"`
}
// ParseConfigFile returns a new ConfigFile given the path to a YAML
// ParseConfigFile returns a new Config given the path to a YAML
// configuration file.
//
// It supports relative and absolute paths and environment variables.
func ParseConfigFile(path string) (*ConfigFile, error) {
func ParseConfigFile(path string) (*Config, error) {
if path == "" {
return nil, errors.New("no config path specified")
}
@@ -59,7 +54,7 @@ func ParseConfigFile(path string) (*ConfigFile, error) {
f, err := os.Open(os.ExpandEnv(path))
if err == nil {
defer f.Close()
cfgFile := new(ConfigFile)
cfgFile := new(Config)
err = yaml.NewDecoder(f).Decode(cfgFile)
return cfgFile, err
}
cmd/mochi/server.go
+16 -34
View File
@@ -4,8 +4,7 @@ import (
"errors"
"fmt"
"github.com/sot-tech/mochi/frontend/http"
"github.com/sot-tech/mochi/frontend/udp"
"github.com/sot-tech/mochi/frontend"
"github.com/sot-tech/mochi/middleware"
"github.com/sot-tech/mochi/pkg/log"
"github.com/sot-tech/mochi/pkg/metrics"
@@ -24,11 +23,10 @@ type Server struct {
// It is optional to provide an instance of the peer store to avoid the
// creation of a new one.
func (r *Server) Run(configFilePath string) error {
configFile, err := ParseConfigFile(configFilePath)
cfg, err := ParseConfigFile(configFilePath)
if err != nil {
return fmt.Errorf("failed to read config: %w", err)
}
cfg := configFile.Conf
r.sg = stop.NewGroup()
@@ -39,51 +37,35 @@ func (r *Server) Run(configFilePath string) error {
log.Info().Msg("metrics disabled because of empty address")
}
log.Info().Str("name", cfg.Storage.Name).Msg("starting storage")
r.storage, err = storage.NewStorage(cfg.Storage.Name, cfg.Storage.Config)
r.storage, err = storage.NewStorage(cfg.Storage)
if err != nil {
return fmt.Errorf("failed to create storage: %w", err)
}
log.Info().Object("config", r.storage).Msg("started storage")
preHooks, err := middleware.NewHooks(cfg.PreHooks, r.storage)
if err != nil {
return fmt.Errorf("failed to validate hook config: %w", err)
return fmt.Errorf("failed to configure pre-hooks: %w", err)
}
postHooks, err := middleware.NewHooks(cfg.PostHooks, r.storage)
if err != nil {
return fmt.Errorf("failed to validate hook config: %w", err)
return fmt.Errorf("failed to configure post-hooks: %w", err)
}
r.logic = middleware.NewLogic(cfg.AnnounceInterval, cfg.MinAnnounceInterval, r.storage, preHooks, postHooks)
var started bool
if len(cfg.HTTPConfig) > 0 {
log.Info().Object("config", cfg.HTTPConfig).Msg("starting HTTP frontend")
httpFE, err := http.NewFrontend(r.logic, cfg.HTTPConfig)
if err == nil {
r.sg.Add(httpFE)
started = true
if len(cfg.Frontends) > 0 {
var fs []frontend.Frontend
r.logic = middleware.NewLogic(cfg.AnnounceInterval, cfg.MinAnnounceInterval, r.storage, preHooks, postHooks)
if fs, err = frontend.NewFrontends(cfg.Frontends, r.logic); err == nil {
for _, f := range fs {
r.sg.Add(f)
}
} else {
return err
err = fmt.Errorf("failed to configure frontends: %w", err)
}
} else {
err = errors.New("no frontends configured")
}
if len(cfg.UDPConfig) > 0 {
log.Info().Object("config", cfg.UDPConfig).Msg("starting UDP frontend")
udpFE, err := udp.NewFrontend(r.logic, cfg.UDPConfig)
if err == nil {
r.sg.Add(udpFE)
started = true
} else {
return err
}
}
if !started {
return errors.New("no frontends configured")
}
return nil
return err
}
// Dispose shuts down an instance of Server.
dist/example_config.yaml
Vendored
+175 -277
View File
@@ -1,303 +1,201 @@
# @formatter:off
mochi:
# The interval communicated with BitTorrent clients informing them how
# frequently they should announce in between client events.
announce_interval: 30m
# The interval communicated with BitTorrent clients informing them how
# frequently they should announce in between client events.
announce_interval: 30m
# The interval communicated with BitTorrent clients informing them of the
# minimal duration between announces.
min_announce_interval: 15m
# The interval communicated with BitTorrent clients informing them of the
# minimal duration between announces.
min_announce_interval: 15m
# The network interface that will bind to an HTTP endpoint that can be
# scraped by programs collecting metrics.
#
# /metrics serves metrics in the Prometheus format
# /debug/pprof/{cmdline,profile,symbol,trace} serves profiles in the pprof format
metrics_addr: "0.0.0.0:6880"
# The network interface that will bind to an HTTP endpoint that can be
# scraped by programs collecting metrics.
#
# /metrics serves metrics in the Prometheus format
# /debug/pprof/{cmdline,profile,symbol,trace} serves profiles in the pprof format
metrics_addr: "0.0.0.0:6880"
frontends:
# This block defines configuration for the tracker's HTTP interface.
# If you do not wish to run this, delete this section.
http:
# The network interface that will bind to an HTTP server for serving
# BitTorrent traffic. Remove this to disable the non-TLS listener.
addr: "0.0.0.0:6969"
# The network interface that will bind to an HTTPS server for serving
# BitTorrent traffic. If set, tls_cert_path and tls_key_path are required.
https_addr: ""
# The path to the required files to listen via HTTPS.
tls_cert_path: ""
tls_key_path: ""
# Enable SO_REUSEPORT to allow starting multiple mochi instances with the same HTTP(S) port.
reuse_port: true
# The timeout durations for HTTP requests.
read_timeout: 5s
write_timeout: 5s
# When true, persistent connections will be allowed. Generally this is not
# useful for a public tracker, but helps performance in some cases (use of
# a reverse proxy, or when there are few clients issuing many requests).
enable_keepalive: false
idle_timeout: 30s
# Whether to time requests.
# Disabling this should increase performance/decrease load.
enable_request_timing: false
# An array of routes to listen on for announce requests. This is an option
# to support trackers that do not listen for /announce or need to listen
# on multiple routes.
#
# This supports named parameters and catch-all parameters as described at
# https://github.com/julienschmidt/httprouter#named-parameters
announce_routes:
- "/announce"
# - "/announce.php"
# An array of routes to listen on for scrape requests. This is an option
# to support trackers that do not listen for /scrape or need to listen
# on multiple routes.
#
# This supports named parameters and catch-all parameters as described at
# https://github.com/julienschmidt/httprouter#named-parameters
scrape_routes:
- "/scrape"
# - "/scrape.php"
# An array of routes to listen ping requests.
# Used just to ensure if server is operational. Returns nothing,
# just HTTP 200 without body. Listens both GET and HEAD HTTP methods.
# HEAD method just checks http server, GET checks all hooks,
# which support ping
ping_routes:
- "/ping"
# When not enabled, tracker will use only address from which client connected to tracker.
# When enabled, the IP address that clients advertise as their IP address will
# be appended as announce candidate.
allow_ip_spoofing: false
# When enabled, IPs from private, local and loopback subnets will be ignored
filter_private_ips: false
# The HTTP Header containing the IP address of the client.
# This is only necessary if using a reverse proxy.
real_ip_header: "x-real-ip"
# The maximum number of peers returned for an individual request.
max_numwant: 100
# The default number of peers returned for an individual request.
default_numwant: 50
# The maximum number of infohashes that can be scraped in one request.
max_scrape_infohashes: 50
# This block defines configuration for the tracker's UDP interface.
# If you do not wish to run this, delete this section.
udp:
# The network interface that will bind to a UDP server for serving
# BitTorrent traffic.
addr: "0.0.0.0:6969"
# Enable SO_REUSEPORT to allow starting multiple mochi instances with the same UDP port.
reuse_port: true
# The leeway for a timestamp on a connection ID.
max_clock_skew: 10s
# The key used to encrypt connection IDs.
private_key: "paste a random string here that will be used to hmac connection IDs"
# Whether to time requests.
# Disabling this should increase performance/decrease load.
enable_request_timing: false
# When not enabled, tracker will use only address from which client connected to tracker.
# When enabled, the IP address that clients advertise as their IP address will
# be appended as announce candidate.
allow_ip_spoofing: false
# When enabled, IPs from private, local and loopback subnets will be ignored
filter_private_ips: false
# The maximum number of peers returned for an individual request.
max_numwant: 100
# The default number of peers returned for an individual request.
default_numwant: 50
# The maximum number of infohashes that can be scraped in one request.
max_scrape_infohashes: 50
# This block defines configuration used for the storage of peer data.
storage:
name: memory
- name: http
config:
# The frequency which stale peers are removed.
# This balances between
# - collecting garbage more often, potentially using more CPU time, but potentially using less memory (lower value)
# - collecting garbage less frequently, saving CPU time, but keeping old peers long, thus using more memory (higher value).
gc_interval: 3m
# The network interface that will bind to an HTTP server for serving
# BitTorrent traffic. Remove this to disable the non-TLS listener.
addr: "0.0.0.0:6969"
# The amount of time until a peer is considered stale.
# To avoid churn, keep this slightly larger than `announce_interval`
peer_lifetime: 31m
# Mark this frontend as HTTPS server for serving
# BitTorrent traffic. If set, tls_cert_path and tls_key_path are required.
tls: false
# The number of partitions data will be divided into in order to provide a
# higher degree of parallelism.
shard_count: 1024
# The path to the required files to listen via HTTPS.
tls_cert_path: ""
tls_key_path: ""
# The interval at which metrics about the number of infohashes and peers
# are collected and posted to Prometheus.
prometheus_reporting_interval: 1s
# Enable SO_REUSEPORT to allow starting multiple mochi instances with the same HTTP(S) port.
reuse_port: true
# This block defines configuration used for redis storage.
#storage:
#name: redis
#config:
# The frequency which stale peers are removed.
# This balances between
# - collecting garbage more often, potentially using more CPU time, but potentially using less memory (lower value)
# - collecting garbage less frequently, saving CPU time, but keeping old peers long, thus using more memory (higher value).
#gc_interval: 3m
# The timeout durations for HTTP requests.
read_timeout: 5s
write_timeout: 5s
# The interval at which metrics about the number of infohashes and peers
# are collected and posted to Prometheus.
#prometheus_reporting_interval: 1s
# When true, persistent connections will be allowed. Generally this is not
# useful for a public tracker, but helps performance in some cases (use of
# a reverse proxy, or when there are few clients issuing many requests).
enable_keepalive: false
idle_timeout: 30s
# The amount of time until a peer is considered stale.
# To avoid churn, keep this slightly larger than `announce_interval`
#peer_lifetime: 31m
# Whether to time requests.
# Disabling this should increase performance/decrease load.
enable_request_timing: false
# The addresses of redis storage.
# If neither sentinel not cluster switched,
# only first address used
#addresses: ["127.0.0.1:6379"]
# An array of routes to listen on for announce requests. This is an option
# to support trackers that do not listen for /announce or need to listen
# on multiple routes.
#
# This supports named parameters and catch-all parameters as described at
# https://github.com/julienschmidt/httprouter#named-parameters
announce_routes:
- "/announce"
# - "/announce.php"
# Database to be selected after connecting to the server.
#db: 0
# An array of routes to listen on for scrape requests. This is an option
# to support trackers that do not listen for /scrape or need to listen
# on multiple routes.
#
# This supports named parameters and catch-all parameters as described at
# https://github.com/julienschmidt/httprouter#named-parameters
scrape_routes:
- "/scrape"
# - "/scrape.php"
# Maximum number of socket connections, default is 10 per CPU
#pool_size: 10
# An array of routes to listen ping requests.
# Used just to ensure if server is operational. Returns nothing,
# just HTTP 200 without body. Listens both GET and HEAD HTTP methods.
# HEAD method just checks http server, GET checks all hooks,
# which support ping
ping_routes:
- "/ping"
# Use the specified login/username to authenticate the current connection
#login: ""
# When not enabled, tracker will use only address from which client connected to tracker.
# When enabled, the IP address that clients advertise as their IP address will
# be appended as announce candidate.
allow_ip_spoofing: false
# Optional password
#password: ""
# When enabled, IPs from private, local and loopback subnets will be ignored
filter_private_ips: false
# Connect to sentinel nodes
#sentinel: false
# The HTTP Header containing the IP address of the client.
# This is only necessary if using a reverse proxy.
real_ip_header: "x-real-ip"
# The master name
#sentinel_master: ""
# The maximum number of peers returned for an individual request.
max_numwant: 100
# Connect to the redis cluster
#cluster: false
# The default number of peers returned for an individual request.
default_numwant: 50
# The timeout for reading a command reply from redis.
#read_timeout: 15s
# The maximum number of infohashes that can be scraped in one request.
max_scrape_infohashes: 50
# The timeout for writing a command to redis.
#write_timeout: 15s
# This block defines configuration for the tracker's UDP interface.
# If you do not wish to run this, delete this section.
- name: udp
config:
# The network interface that will bind to a UDP server for serving
# BitTorrent traffic.
addr: "0.0.0.0:6969"
# Dial timeout for establishing new connections.
#connect_timeout: 15s
# Enable SO_REUSEPORT to allow starting multiple mochi instances with the same UDP port.
reuse_port: true
# This block defines configuration used for PostgreSQL storage.
# example `mo_peers` table structure:
# - info_hash bytea
# - peer_id bytea
# - address inet or bytea
# - port int4
# - is_seeder bool
# - is_v6 bool
# - created timestamp
#storage:
#name: pg
#config:
# connection string to pg storage. may be URL (postgres://...) or DSN (host=... port=...)
#connection_string: host=127.0.0.1 database=test user=postgres pool_max_conns=50
# query and parameters for announce operation
#announce:
#query: SELECT peer_id, address, port FROM mo_peers WHERE info_hash=$1 AND is_seeder=$2 AND is_v6=$3 LIMIT $4
#peer_id_column: peer_id
#address_column: address
#port_column: port
#peer:
# expected parameters: 1 - info hash (bytea), 2 - peer id (bytea), 3 - ip address (bytea/inet)
# 4 - port (int), 5 - is seeder (bool), 6 - is IPv6 (bool), 7 - create date and time (timestamp)
#add_query: INSERT INTO mo_peers VALUES($1, $2, $3, $4, $5, $6, $7) ON CONFLICT (info_hash, peer_id, address, port) DO UPDATE SET created = EXCLUDED.created, is_seeder = EXCLUDED.is_seeder
#del_query: DELETE FROM mo_peers WHERE info_hash=$1 AND peer_id=$2 AND address=$3 AND port=$4 AND is_seeder=$5
#graduate_query: UPDATE mo_peers SET is_seeder=TRUE WHERE info_hash=$1 AND peer_id=$2 AND address=$3 AND port=$4 AND NOT is_seeder
#count_query: SELECT COUNT(1) FILTER (WHERE is_seeder) AS seeders, COUNT(1) FILTER (WHERE NOT is_seeder) AS leechers FROM mo_peers
# predicate part of `count_query` for get count of peers by info hash
#by_info_hash_clause: WHERE info_hash = $1
#count_seeders_column: seeders
#count_leechers_column: leechers
# queries for KV-store
#data:
# expected parameters: 1 - context (varchar), 2 - name (bytea), 3 - value (bytea)
#add_query: INSERT INTO mo_kv VALUES($1, $2, $3) ON CONFLICT (context, name) DO NOTHING
#del_query: DELETE FROM mo_kv WHERE context=$1 AND name=$2
#get_query: SELECT value FROM mo_kv WHERE context=$1 AND name=$2
# query for check if database is alive
#ping_query: SELECT 1
# query for garbage collection, expected parameter is timestamp
#gc_query: DELETE FROM mo_peers WHERE created <= $1
# The amount of time until a peer is considered stale.
# To avoid churn, keep this slightly larger than `announce_interval`
#peer_lifetime: 31m
# The frequency which stale peers are removed.
#gc_interval: 3m
# query for info hash statistics
#info_hash_count_query: SELECT COUNT(DISTINCT info_hash) as info_hashes FROM mo_peers
# The interval at which metrics about the number of info hashes and peers
# are collected and posted to Prometheus.
#prometheus_reporting_interval: 1s
# The leeway for a timestamp on a connection ID.
max_clock_skew: 10s
# This block defines configuration used for middleware executed before a
# response has been returned to a BitTorrent client.
prehooks:
# - name: jwt
# options:
# header: "authorization"
# issuer: "https://issuer.com"
# audience: "https://some.issuer.com"
# jwk_set_url: "https://issuer.com/keys"
# jwk_set_update_interval: 5m
# handle_announce: true
# handle_scrape: false
#
# - name: client approval
# options:
# whitelist:
# - "OP1011"
# blacklist:
# - "OP1012"
#
# - name: interval variation
# options:
# modify_response_probability: 0.2
# max_increase_delta: 60
# modify_min_interval: true
#
# This block defines configuration used for torrent approval, it requires to be given
# hashes for whitelist or for blacklist. Hashes are hexadecimal-encoaded.
# - name: torrent approval
# options:
# initial_source: list
# Save data provided by source in storage above
# preserve: false
# configuration:
# hash_list:
# - "a1b2c3d4e5a1b2c3d4e5a1b2c3d4e5a1b2c3d4e5"
# true - whitelist mode, false - blacklist
# invert: false
# Name of storage context where store hash list
# storage_ctx: APPROVED_HASH
posthooks:
# The key used to encrypt connection IDs.
private_key: "paste a random string here that will be used to hmac connection IDs"
# Whether to time requests.
# Disabling this should increase performance/decrease load.
enable_request_timing: false
# When not enabled, tracker will use only address from which client connected to tracker.
# When enabled, the IP address that clients advertise as their IP address will
# be appended as announce candidate.
allow_ip_spoofing: false
# When enabled, IPs from private, local and loopback subnets will be ignored
filter_private_ips: false
# The maximum number of peers returned for an individual request.
max_numwant: 100
# The default number of peers returned for an individual request.
default_numwant: 50
# The maximum number of infohashes that can be scraped in one request.
max_scrape_infohashes: 50
# This block defines configuration used for the storage of peer data.
storage:
name: memory
config:
# The frequency which stale peers are removed.
# This balances between
# - collecting garbage more often, potentially using more CPU time, but potentially using less memory (lower value)
# - collecting garbage less frequently, saving CPU time, but keeping old peers long, thus using more memory (higher value).
gc_interval: 3m
# The amount of time until a peer is considered stale.
# To avoid churn, keep this slightly larger than `announce_interval`
peer_lifetime: 31m
# The number of partitions data will be divided into in order to provide a
# higher degree of parallelism.
shard_count: 1024
# The interval at which metrics about the number of infohashes and peers
# are collected and posted to Prometheus.
prometheus_reporting_interval: 1s
# This block defines configuration used for middleware executed before a
# response has been returned to a BitTorrent client.
posthooks: []
prehooks:
# - name: jwt
# config:
# header: "authorization"
# issuer: "https://issuer.com"
# audience: "https://some.issuer.com"
# jwk_set_url: "https://issuer.com/keys"
# jwk_set_update_interval: 5m
# handle_announce: true
# handle_scrape: false
#
# - name: client approval
# config:
# whitelist:
# - "OP1011"
# blacklist:
# - "OP1012"
#
# - name: interval variation
# config:
# modify_response_probability: 0.2
# max_increase_delta: 60
# modify_min_interval: true
#
# This block defines configuration used for torrent approval, it requires to be given
# hashes for whitelist or for blacklist. Hashes are hexadecimal-encoaded.
# - name: torrent approval
# config:
# initial_source: list
# Save data provided by source in storage above
# preserve: false
# configuration:
# hash_list:
# - "a1b2c3d4e5a1b2c3d4e5a1b2c3d4e5a1b2c3d4e5"
# true - whitelist mode, false - blacklist
# invert: false
# Name of storage context where store hash list
# storage_ctx: APPROVED_HASH
dist/example_config_pg.yaml
Vendored
+116
View File
@@ -0,0 +1,116 @@
# @formatter:off
# Note: see `example_config.yaml` for `frontends` and `*hooks` config description
announce_interval: 30m
min_announce_interval: 15m
metrics_addr: ""
frontends:
- name: http
config:
addr: "0.0.0.0:6969"
tls: false
tls_cert_path: ""
tls_key_path: ""
reuse_port: true
read_timeout: 5s
write_timeout: 5s
enable_keepalive: false
idle_timeout: 30s
enable_request_timing: false
announce_routes:
- "/announce"
scrape_routes:
- "/scrape"
ping_routes:
- "/ping"
allow_ip_spoofing: false
filter_private_ips: false
real_ip_header: "x-real-ip"
max_numwant: 100
default_numwant: 50
max_scrape_infohashes: 50
- name: udp
config:
addr: "0.0.0.0:6969"
reuse_port: true
max_clock_skew: 10s
private_key: "paste a random string here that will be used to hmac connection IDs"
enable_request_timing: false
allow_ip_spoofing: false
filter_private_ips: false
max_numwant: 100
default_numwant: 50
max_scrape_infohashes: 50
# This block defines configuration used for PostgreSQL storage.
# example peers table structure:
# - info_hash bytea
# - peer_id bytea
# - address inet or bytea
# - port int4
# - is_seeder bool
# - is_v6 bool
# - created timestamp
# example downloads table structure:
# - info_hash bytea
# - downloads int
storage:
name: pg
config:
# connection string to pg storage. may be URL (postgres://...) or DSN (host=... port=...)
connection_string: host=127.0.0.1 database=test user=postgres pool_max_conns=50
# query and parameters for announce operation
announce:
query: SELECT peer_id, address, port FROM mo_peers WHERE info_hash=@info_hash AND is_seeder=@is_seeder AND is_v6=@is_v6 LIMIT @count
peer_id_column: peer_id
address_column: address
port_column: port
# queries to get/increment 'snatched' (downloaded) count
downloads:
get_query: SELECT downloads FROM mo_downloads where info_hash=@info_hash
inc_query: INSERT INTO mo_downloads VALUES(@info_hash) ON CONFLICT(info_hash) DO UPDATE SET downloads = mo_downloads.downloads + 1
# queries and parameters for add/delete/count peers operations
peer:
add_query: INSERT INTO mo_peers VALUES(@info_hash, @peer_id, @address, @port, @is_seeder, @is_v6, @created) ON CONFLICT (info_hash, peer_id, address, port) DO UPDATE SET created = EXCLUDED.created, is_seeder = EXCLUDED.is_seeder
del_query: DELETE FROM mo_peers WHERE info_hash=@info_hash AND peer_id=@peer_id AND address=@address AND port=@port AND is_seeder=@is_seeder
graduate_query: UPDATE mo_peers SET is_seeder=TRUE WHERE info_hash=@info_hash AND peer_id=peer_id AND address=@address AND port=@port AND NOT is_seeder
count_query: SELECT COUNT(1) FILTER (WHERE is_seeder) AS seeders, COUNT(1) FILTER (WHERE NOT is_seeder) AS leechers FROM mo_peers
# predicate part of `count_query` to get count of peers by info hash
by_info_hash_clause: WHERE info_hash = @info_hash
count_seeders_column: seeders
count_leechers_column: leechers
# queries for KV-store
data:
add_query: INSERT INTO mo_kv VALUES(@context, @key, @value) ON CONFLICT (context, name) DO NOTHING
# Note: in del_query @key parameter is array, NOT single value
del_query: DELETE FROM mo_kv WHERE context=@context AND name = ANY(@key)
get_query: SELECT value FROM mo_kv WHERE context=@context AND name=@key
# query for check if database is alive
ping_query: SELECT 1
# query for garbage collection, expected parameter is timestamp
gc_query: DELETE FROM mo_peers WHERE created <= @created
# The amount of time until a peer is considered stale.
# To avoid churn, keep this slightly larger than `announce_interval`
peer_lifetime: 31m
# The frequency which stale peers are removed.
gc_interval: 3m
# query for info hash statistics
info_hash_count_query: SELECT COUNT(DISTINCT info_hash) as info_hashes FROM mo_peers
# The interval at which metrics about the number of info hashes and peers
# are collected and posted to Prometheus.
prometheus_reporting_interval: 1s
posthooks: []
prehooks: []
dist/example_config_redis.yaml
Vendored
+103
View File
@@ -0,0 +1,103 @@
# @formatter:off
# Note: see `example_config.yaml` for `frontends` and `*hooks` config description
announce_interval: 30m
min_announce_interval: 15m
metrics_addr: ""
frontends:
- name: http
config:
addr: "0.0.0.0:6969"
tls: false
tls_cert_path: ""
tls_key_path: ""
reuse_port: true
read_timeout: 5s
write_timeout: 5s
enable_keepalive: false
idle_timeout: 30s
enable_request_timing: false
announce_routes:
- "/announce"
scrape_routes:
- "/scrape"
ping_routes:
- "/ping"
allow_ip_spoofing: false
filter_private_ips: false
real_ip_header: "x-real-ip"
max_numwant: 100
default_numwant: 50
max_scrape_infohashes: 50
- name: udp
config:
addr: "0.0.0.0:6969"
reuse_port: true
max_clock_skew: 10s
private_key: "paste a random string here that will be used to hmac connection IDs"
enable_request_timing: false
allow_ip_spoofing: false
filter_private_ips: false
max_numwant: 100
default_numwant: 50
max_scrape_infohashes: 50
# This block defines configuration used for redis storage.
storage:
# If used keydb fork, set `keydb` name
name: redis
config:
# The frequency which stale peers are removed.
# This balances between
# - collecting garbage more often, potentially using more CPU time, but potentially using less memory (lower value)
# - collecting garbage less frequently, saving CPU time, but keeping old peers long, thus using more memory (higher value).
gc_interval: 3m
# The interval at which metrics about the number of infohashes and peers
# are collected and posted to Prometheus.
prometheus_reporting_interval: 1s
# The amount of time until a peer is considered stale.
# To avoid churn, keep this slightly larger than `announce_interval`
peer_lifetime: 31m
# The addresses of redis storage.
# If neither sentinel not cluster switched,
# only first address used
addresses: ["127.0.0.1:6379"]
# Database to be selected after connecting to the server.
db: 0
# Maximum number of socket connections, default is 10 per CPU
pool_size: 10
# Use the specified login/username to authenticate the current connection
login: ""
# Optional password
password: ""
# Connect to sentinel nodes
sentinel: false
# The master name
sentinel_master: ""
# Connect to the redis cluster
cluster: false
# The timeout for reading a command reply from redis.
read_timeout: 15s
# The timeout for writing a command to redis.
write_timeout: 15s
# Dial timeout for establishing new connections.
connect_timeout: 15s
posthooks: []
prehooks: []
docs/storage/postgres.md
+39 -26
View File
@@ -34,17 +34,20 @@ you provide in configuration.
Implementation expects next data types:
* Table for peers:
* info hash - byte array (`bytea`)
* peer ID - byte array (`bytea`)
* peer address - `inet` or byte array (`bytea`)
* peer port - integer or derivative type (`int4`, `integer`)
* is seeder - boolean (`bool`)
* is IPv6 - boolean (`bool`)
* peer creation date and time - `timestamp`
* info hash - byte array (`bytea`)
* peer ID - byte array (`bytea`)
* peer address - `inet` or byte array (`bytea`)
* peer port - integer or derivative type (`int4`, `integer`)
* is seeder - boolean (`bool`)
* is IPv6 - boolean (`bool`)
* peer creation date and time - `timestamp`
* Table of download counts (optional)
* info hash - byte array (`bytea`)
* count - or derivative type (`int4`, `integer`)
* Table for arbitrary data (KV store):
* context - string (`varchar`, `character varying`)
* name - byte array (`bytea`)*
* value - byte array (`bytea`)
* context - string (`varchar`, `character varying`)
* name - byte array (`bytea`)*
* value - byte array (`bytea`)
(*) in KV table `name` present as byte array because of possibility
to place hash as _raw_ string, which is not supported by PostgreSQL.
@@ -67,6 +70,11 @@ CREATE TABLE mo_peers
CREATE INDEX mo_peers_created_idx ON mo_peers (created);
CREATE INDEX mo_peers_announce_idx ON mo_peers (info_hash, is_seeder, is_v6);
CREATE TABLE mo_downloads (
info_hash bytea PRIMARY KEY NOT NULL,
downloads int NOT NULL DEFAULT 1
);
CREATE TABLE mo_kv
(
context varchar NOT NULL,
@@ -98,49 +106,54 @@ storage:
peer:
# Query to add peer info.
# Expected arguments:
# 1 - info hash (bytea),
# 2 - peer id (bytea),
# 3 - ip address (bytea/inet)
# 1 - info_hash (bytea),
# 2 - peer_id (bytea),
# 3 - address (bytea/inet)
# 4 - port (int4),
# 5 - is seeder (bool),
# 6 - is IPv6 (bool),
# 7 - create date and time (timestamp)
# 5 - is_seeder (bool),
# 6 - is_v6 (bool),
# 7 - created (timestamp)
# Query MUST handle situations when tuple
# `info hash - peer ID - address - port` is already
# exists in table
add_query: INSERT INTO mo_peers VALUES($1, $2, $3, $4, $5, $6, $7) ON CONFLICT (info_hash, peer_id, address, port) DO UPDATE SET created = EXCLUDED.created, is_seeder = EXCLUDED.is_seeder
add_query: INSERT INTO mo_peers VALUES(@info_hash, @peer_id, @address, @port, @is_seeder, @is_v6, @created) ON CONFLICT (info_hash, peer_id, address, port) DO UPDATE SET created = EXCLUDED.created, is_seeder = EXCLUDED.is_seeder
# Query to delete peer info.
# Query SHOULD take into account value of `is seeder` flag
del_query: DELETE FROM mo_peers WHERE info_hash=$1 AND peer_id=$2 AND address=$3 AND port=$4 AND is_seeder=$5
# Query SHOULD take into account value of `is_seeder` flag
del_query: DELETE FROM mo_peers WHERE info_hash=@info_hash AND peer_id=@peer_id AND address=@address AND port=@port AND is_seeder=@is_seeder
# Query to update leecher to seeder
graduate_query: UPDATE mo_peers SET is_seeder=TRUE WHERE info_hash=$1 AND peer_id=$2 AND address=$3 AND port=$4 AND NOT is_seeder
graduate_query: UPDATE mo_peers SET is_seeder=TRUE WHERE info_hash=@info_hash AND peer_id=peer_id AND address=@address AND port=@port AND NOT is_seeder
# Query to get count of peers.
# Used both for statistics and for scrape (with clause suffix, see next).
# Only first returned row value used.
count_query: SELECT COUNT(1) FILTER (WHERE is_seeder) AS seeders, COUNT(1) FILTER (WHERE NOT is_seeder) AS leechers FROM mo_peers
# Predicate part of `count_query` for get count of peers by info hash
by_info_hash_clause: WHERE info_hash = $1
by_info_hash_clause: WHERE info_hash = @info_hash
# Column name of seeders count in `count_query` (case-insensitive).
count_seeders_column: seeders
# Column name of leechers count in `count_query` (case-insensitive).
count_leechers_column: leechers
# Queries to get/increment 'snatched' (downloaded) count
downloads:
get_query: SELECT downloads FROM mo_downloads where info_hash=@info_hash
inc_query: INSERT INTO mo_downloads VALUES(@info_hash) ON CONFLICT(info_hash) DO UPDATE SET downloads = mo_downloads.downloads + 1
# Queries for KV-store
data:
# Query to add data.
# Expected arguments:
# 1 - context (varchar),
# 2 - name (bytea),
# 2 - key (bytea),
# 3 - value (bytea)
add_query: INSERT INTO mo_kv VALUES($1, $2, $3) ON CONFLICT (context, name) DO NOTHING
add_query: INSERT INTO mo_kv VALUES(@context, @key, @value) ON CONFLICT (context, name) DO NOTHING
# Query to delete data.
del_query: DELETE FROM mo_kv WHERE context=$1 AND name=$2
# Note: @key parameter is array, NOT single value
del_query: DELETE FROM mo_kv WHERE context=@context AND name = ANY(@key)
# Query to get data.
# Only first returned row and column value used.
get_query: SELECT value FROM mo_kv WHERE context=$1 AND name=$2
get_query: SELECT value FROM mo_kv WHERE context=@context AND name=@key
# Query for check if database is alive (can be omitted)
ping_query: SELECT 1
# Query to delete stale peers (peers, which timestamp older than provided argument)
gc_query: DELETE FROM mo_peers WHERE created <= $1
gc_query: DELETE FROM mo_peers WHERE created <= @created
# The frequency which stale peers are removed.
gc_interval: 3m
# Query to get all info hash count (used for statistics).
docs/storage/redis.md
+2
View File
@@ -69,6 +69,8 @@ Here is an example:
- <peer 2 key>: <modification time in unix nanos>
- CHI_L4_<HASH2>
- <peer 3 key>: <modification time in unix nanos>
- CHI_D (hash type)
- <HASH1>: <number of downloads>
...
```
frontend/frontend.go
+60 -23
View File
@@ -3,34 +3,71 @@
package frontend
import (
"context"
"fmt"
"sync"
"github.com/sot-tech/mochi/bittorrent"
"github.com/sot-tech/mochi/middleware"
"github.com/sot-tech/mochi/pkg/conf"
"github.com/sot-tech/mochi/pkg/log"
"github.com/sot-tech/mochi/pkg/stop"
)
// TrackerLogic is the interface used by a frontend in order to: (1) generate a
// response from a parsed request, and (2) asynchronously observe anything
// after the response has been delivered to the client.
type TrackerLogic interface {
// HandleAnnounce generates a response for an Announce.
//
// Returns the updated context, the generated AnnounceResponse and no error
// on success; nil and error on failure.
HandleAnnounce(context.Context, *bittorrent.AnnounceRequest) (context.Context, *bittorrent.AnnounceResponse, error)
var (
logger = log.NewLogger("frontend")
buildersMU sync.RWMutex
builders = make(map[string]Builder)
)
// AfterAnnounce does something with the results of an Announce after it
// has been completed.
AfterAnnounce(context.Context, *bittorrent.AnnounceRequest, *bittorrent.AnnounceResponse)
// Builder is the function used to initialize a new Frontend
// with provided configuration.
type Builder func(conf.MapConfig, *middleware.Logic) (Frontend, error)
// HandleScrape generates a response for a Scrape.
//
// Returns the updated context, the generated AnnounceResponse and no error
// on success; nil and error on failure.
HandleScrape(context.Context, *bittorrent.ScrapeRequest) (context.Context, *bittorrent.ScrapeResponse, error)
// RegisterBuilder makes a Builder available by the provided name.
//
// If called twice with the same name, the name is blank, or if the provided
// Builder is nil, this function panics.
func RegisterBuilder(name string, b Builder) {
if name == "" {
panic("frontend: could not register Builder with an empty name")
}
if b == nil {
panic("frontend: could not register a nil Builder")
}
// AfterScrape does something with the results of a Scrape after it has been completed.
AfterScrape(context.Context, *bittorrent.ScrapeRequest, *bittorrent.ScrapeResponse)
buildersMU.Lock()
defer buildersMU.Unlock()
// Ping executes checks if all hooks are operational
Ping(context.Context) error
if _, dup := builders[name]; dup {
panic("frontend: RegisterBuilder called twice for " + name)
}
builders[name] = b
}
// Frontend dummy interface for bittorrent frontends
type Frontend interface {
stop.Stopper
}
// NewFrontends is a utility function for initializing Frontend-s in bulk.
// Returns nil hook and error if frontend with name provided in config
// does not exists.
func NewFrontends(configs []conf.NamedMapConfig, logic *middleware.Logic) (fs []Frontend, err error) {
buildersMU.RLock()
defer buildersMU.RUnlock()
for _, c := range configs {
logger.Debug().Object("frontend", c).Msg("starting frontend")
newFrontend, ok := builders[c.Name]
if !ok {
err = fmt.Errorf("hook with name '%s' does not exists", c.Name)
break
}
var f Frontend
if f, err = newFrontend(c.Config, logic); err != nil {
break
}
fs = append(fs, f)
logger.Info().Str("name", c.Name).Msg("frontend started")
}
return
}
frontend/http/frontend.go
+96 -203
View File
@@ -6,266 +6,159 @@ import (
"context"
"crypto/tls"
"errors"
"net"
"net/http"
"net/netip"
"sync"
"time"
"github.com/julienschmidt/httprouter"
"github.com/libp2p/go-reuseport"
"github.com/sot-tech/mochi/bittorrent"
"github.com/sot-tech/mochi/frontend"
"github.com/sot-tech/mochi/middleware"
"github.com/sot-tech/mochi/pkg/conf"
"github.com/sot-tech/mochi/pkg/log"
"github.com/sot-tech/mochi/pkg/metrics"
"github.com/sot-tech/mochi/pkg/stop"
)
var logger = log.NewLogger("http frontend")
var (
logger = log.NewLogger("frontend/http")
errTLSNotProvided = errors.New("tls certificate/key not provided")
errRoutesNotProvided = errors.New("routes not provided")
)
// Config represents all of the configurable options for an HTTP BitTorrent
// Frontend.
func init() {
frontend.RegisterBuilder("http", NewFrontend)
}
// Config represents all configurable options for an HTTP BitTorrent Frontend
type Config struct {
Addr string
HTTPSAddr string `cfg:"https_addr"`
ReadTimeout time.Duration `cfg:"read_timeout"`
WriteTimeout time.Duration `cfg:"write_timeout"`
IdleTimeout time.Duration `cfg:"idle_timeout"`
EnableKeepAlive bool `cfg:"enable_keepalive"`
TLSCertPath string `cfg:"tls_cert_path"`
TLSKeyPath string `cfg:"tls_key_path"`
ReusePort bool `cfg:"reuse_port"`
AnnounceRoutes []string `cfg:"announce_routes"`
ScrapeRoutes []string `cfg:"scrape_routes"`
PingRoutes []string `cfg:"ping_routes"`
EnableRequestTiming bool `cfg:"enable_request_timing"`
frontend.ListenOptions
IdleTimeout time.Duration `cfg:"idle_timeout"`
EnableKeepAlive bool `cfg:"enable_keepalive"`
UseTLS bool `cfg:"tls"`
TLSCertPath string `cfg:"tls_cert_path"`
TLSKeyPath string `cfg:"tls_key_path"`
AnnounceRoutes []string `cfg:"announce_routes"`
ScrapeRoutes []string `cfg:"scrape_routes"`
PingRoutes []string `cfg:"ping_routes"`
ParseOptions
}
// Default config constants.
const (
defaultReadTimeout = 2 * time.Second
defaultWriteTimeout = 2 * time.Second
defaultIdleTimeout = 30 * time.Second
)
const defaultIdleTimeout = 30 * time.Second
// Validate sanity checks values set in a config and returns a new config with
// default values replacing anything that is invalid.
//
// This function warns to the logger when a value is changed.
func (cfg Config) Validate() Config {
validcfg := cfg
if cfg.ReadTimeout <= 0 {
validcfg.ReadTimeout = defaultReadTimeout
logger.Warn().
Str("name", "http.ReadTimeout").
Dur("provided", cfg.ReadTimeout).
Dur("default", validcfg.ReadTimeout).
Msg("falling back to default configuration")
func (cfg Config) Validate() (validCfg Config, err error) {
validCfg = cfg
if validCfg.ListenOptions, err = cfg.ListenOptions.Validate(); err != nil {
return
}
if cfg.WriteTimeout <= 0 {
validcfg.WriteTimeout = defaultWriteTimeout
logger.Warn().
Str("name", "http.WriteTimeout").
Dur("provided", cfg.WriteTimeout).
Dur("default", validcfg.WriteTimeout).
Msg("falling back to default configuration")
if cfg.UseTLS && (len(cfg.TLSCertPath) == 0 || len(cfg.TLSKeyPath) == 0) {
err = errTLSNotProvided
return
}
if cfg.IdleTimeout <= 0 {
validcfg.IdleTimeout = defaultIdleTimeout
validCfg.IdleTimeout = defaultIdleTimeout
if cfg.EnableKeepAlive {
// If keepalive is disabled, this configuration isn't used anyway.
logger.Warn().
Str("name", "http.IdleTimeout").
Str("name", "IdleTimeout").
Dur("provided", cfg.IdleTimeout).
Dur("default", validcfg.IdleTimeout).
Dur("default", validCfg.IdleTimeout).
Msg("falling back to default configuration")
}
}
validcfg.ParseOptions.ParseOptions = cfg.ParseOptions.ParseOptions.Validate()
return validcfg
validCfg.ParseOptions.ParseOptions = cfg.ParseOptions.ParseOptions.Validate()
return
}
// Frontend represents the state of an HTTP BitTorrent Frontend.
type Frontend struct {
srv *http.Server
srvMu sync.Mutex
tlsSrv *http.Server
tlsSrvMu sync.Mutex
tlsCfg *tls.Config
logic frontend.TrackerLogic
Config
type httpFE struct {
srv *http.Server
logic *middleware.Logic
collectTimings bool
ParseOptions
}
// NewFrontend creates a new instance of an HTTP Frontend that asynchronously
// serves requests.
func NewFrontend(logic frontend.TrackerLogic, c conf.MapConfig) (*Frontend, error) {
var provided Config
if err := c.Unmarshal(&provided); err != nil {
return nil, err
// NewFrontend builds and starts http bittorrent frontend from provided configuration
func NewFrontend(c conf.MapConfig, logic *middleware.Logic) (_ frontend.Frontend, err error) {
var cfg Config
if err = c.Unmarshal(&cfg); err != nil {
return
}
cfg := provided.Validate()
f := &Frontend{
logic: logic,
Config: cfg,
srvMu: sync.Mutex{},
tlsSrvMu: sync.Mutex{},
if cfg, err = cfg.Validate(); err != nil {
return
}
if cfg.Addr == "" && cfg.HTTPSAddr == "" {
return nil, errors.New("must specify addr or https_addr or both")
}
if len(cfg.AnnounceRoutes) < 1 || len(cfg.ScrapeRoutes) < 1 {
return nil, errors.New("must specify routes")
err = errRoutesNotProvided
return
}
f := &httpFE{
logic: logic,
srv: &http.Server{
ReadTimeout: cfg.ReadTimeout,
ReadHeaderTimeout: cfg.ReadTimeout,
WriteTimeout: cfg.WriteTimeout,
IdleTimeout: cfg.IdleTimeout,
},
collectTimings: cfg.EnableRequestTiming,
ParseOptions: cfg.ParseOptions,
}
// If TLS is enabled, create a key pair.
if cfg.TLSCertPath != "" && cfg.TLSKeyPath != "" {
var err error
f.tlsCfg = &tls.Config{
Certificates: make([]tls.Certificate, 1),
if cfg.UseTLS {
var cert tls.Certificate
if cert, err = tls.LoadX509KeyPair(cfg.TLSCertPath, cfg.TLSKeyPath); err != nil {
return
}
f.srv.TLSConfig = &tls.Config{
Certificates: []tls.Certificate{cert},
MinVersion: tls.VersionTLS12,
}
f.tlsCfg.Certificates[0], err = tls.LoadX509KeyPair(cfg.TLSCertPath, cfg.TLSKeyPath)
if err != nil {
return nil, err
}
}
if (cfg.HTTPSAddr == "") != (f.tlsCfg == nil) {
return nil, errors.New("must specify both https_addr, tls_cert_path and tls_key_path")
}
router := httprouter.New()
for _, route := range f.AnnounceRoutes {
for _, route := range cfg.AnnounceRoutes {
router.GET(route, f.announceRoute)
}
for _, route := range f.ScrapeRoutes {
for _, route := range cfg.ScrapeRoutes {
router.GET(route, f.scrapeRoute)
}
for _, route := range cfg.PingRoutes {
router.GET(route, f.ping)
router.HEAD(route, f.ping)
}
f.srv.Handler = router
if len(f.PingRoutes) > 0 {
for _, route := range f.PingRoutes {
router.GET(route, f.ping)
router.HEAD(route, f.ping)
f.srv.SetKeepAlivesEnabled(cfg.EnableKeepAlive)
go func() {
ln, err := cfg.ListenTCP()
if err == nil {
if f.srv.TLSConfig == nil {
err = f.srv.Serve(ln)
} else {
err = f.srv.ServeTLS(ln, "", "")
}
}
}
if cfg.Addr != "" {
go func() {
if err := f.serveHTTP(router, false); err != nil {
logger.Fatal().Err(err).Str("proto", "http").Msg("failed while serving")
}
}()
}
if cfg.HTTPSAddr != "" {
go func() {
if err := f.serveHTTP(router, true); err != nil {
logger.Fatal().Err(err).Str("proto", "https").Msg("failed while serving")
}
}()
}
if !errors.Is(err, http.ErrServerClosed) {
logger.Fatal().Err(err).Msg("server failed")
}
}()
return f, nil
}
// Stop provides a thread-safe way to shut down a currently running Frontend.
func (f *Frontend) Stop() stop.Result {
stopGroup := stop.NewGroup()
f.srvMu.Lock()
func (f *httpFE) Stop() stop.Result {
c := make(stop.Channel)
if f.srv != nil {
stopGroup.AddFunc(f.makeStopFunc(f.srv))
}
f.srvMu.Unlock()
f.tlsSrvMu.Lock()
if f.tlsSrv != nil {
stopGroup.AddFunc(f.makeStopFunc(f.tlsSrv))
}
f.tlsSrvMu.Unlock()
return stopGroup.Stop()
}
func (f *Frontend) makeStopFunc(stopSrv *http.Server) stop.Func {
return func() stop.Result {
c := make(stop.Channel)
go func() {
c.Done(stopSrv.Shutdown(context.Background()))
c.Done(f.srv.Shutdown(context.Background()))
}()
return c.Result()
}
}
// serveHTTP blocks while listening and serving non-TLS HTTP BitTorrent
// requests until Stop() is called or an error is returned.
func (f *Frontend) serveHTTP(handler http.Handler, tls bool) error {
srv := &http.Server{
Handler: handler,
ReadTimeout: f.ReadTimeout,
ReadHeaderTimeout: f.ReadTimeout,
WriteTimeout: f.WriteTimeout,
IdleTimeout: f.IdleTimeout,
}
srv.SetKeepAlivesEnabled(f.EnableKeepAlive)
var err error
if tls {
if f.ReusePort {
var ln net.Listener
if ln, err = reuseport.Listen("tcp", f.HTTPSAddr); err == nil {
defer ln.Close()
srv.TLSConfig = f.tlsCfg
f.tlsSrvMu.Lock()
f.tlsSrv = srv
f.tlsSrvMu.Unlock()
err = srv.ServeTLS(ln, "", "")
}
} else {
srv.Addr = f.HTTPSAddr
srv.TLSConfig = f.tlsCfg
f.tlsSrvMu.Lock()
f.tlsSrv = srv
f.tlsSrvMu.Unlock()
err = srv.ListenAndServeTLS("", "")
}
} else {
if f.ReusePort {
var ln net.Listener
if ln, err = reuseport.Listen("tcp", f.Addr); err == nil {
defer ln.Close()
f.srvMu.Lock()
f.srv = srv
f.srvMu.Unlock()
err = srv.Serve(ln)
}
} else {
srv.Addr = f.Addr
f.srvMu.Lock()
f.srv = srv
f.srvMu.Unlock()
err = srv.ListenAndServe()
}
}
// Start the HTTP server.
if errors.Is(err, http.ErrServerClosed) {
err = nil
}
return err
return c.Result()
}
func injectRouteParamsToContext(ctx context.Context, ps httprouter.Params) context.Context {
@@ -277,12 +170,12 @@ func injectRouteParamsToContext(ctx context.Context, ps httprouter.Params) conte
}
// announceRoute parses and responds to an Announce.
func (f *Frontend) announceRoute(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
func (f *httpFE) announceRoute(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
var err error
var start time.Time
var addr netip.Addr
var req *bittorrent.AnnounceRequest
if f.EnableRequestTiming && metrics.Enabled() {
if f.collectTimings && metrics.Enabled() {
start = time.Now()
defer func() {
recordResponseDuration("announce", addr, err, time.Since(start))
@@ -314,11 +207,11 @@ func (f *Frontend) announceRoute(w http.ResponseWriter, r *http.Request, ps http
}
// scrapeRoute parses and responds to a Scrape.
func (f *Frontend) scrapeRoute(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
func (f *httpFE) scrapeRoute(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
var err error
var start time.Time
var addr netip.Addr
if f.EnableRequestTiming && metrics.Enabled() {
if f.collectTimings && metrics.Enabled() {
start = time.Now()
defer func() {
recordResponseDuration("scrape", addr, err, time.Since(start))
@@ -349,7 +242,7 @@ func (f *Frontend) scrapeRoute(w http.ResponseWriter, r *http.Request, ps httpro
go f.logic.AfterScrape(ctx, req, resp)
}
func (f *Frontend) ping(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
func (f *httpFE) ping(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
var err error
if r.Method == http.MethodGet {
err = f.logic.Ping(context.TODO())
frontend/http/writer.go
+75 -59
View File
@@ -1,16 +1,19 @@
package http
import (
"bytes"
"errors"
"net"
"net/http"
"strconv"
"time"
"github.com/anacrolix/torrent/bencode"
"github.com/sot-tech/mochi/bittorrent"
"github.com/sot-tech/mochi/pkg/bytepool"
)
var respBufferPool = bytepool.NewBufferPool()
// WriteError communicates an error to a BitTorrent client over HTTP.
func WriteError(w http.ResponseWriter, err error) {
message := "internal server error"
@@ -20,93 +23,106 @@ func WriteError(w http.ResponseWriter, err error) {
} else {
logger.Error().Err(err).Msg("http: internal error")
}
if err = bencode.NewEncoder(w).Encode(map[string]any{
"failure reason": message,
}); err != nil {
logger.Error().Err(err).Msg("unable to encode message")
}
_, _ = w.Write([]byte("d14:failure reason" + strconv.Itoa(len(message)) + ":" + message + "e"))
}
// WriteAnnounceResponse communicates the results of an Announce to a
// BitTorrent client over HTTP.
func WriteAnnounceResponse(w http.ResponseWriter, resp *bittorrent.AnnounceResponse) error {
bb := respBufferPool.Get()
defer respBufferPool.Put(bb)
if resp.Interval > 0 {
resp.Interval /= time.Second
}
if resp.Interval > 0 {
resp.MinInterval /= time.Second
}
bdict := map[string]any{
"complete": resp.Complete,
"incomplete": resp.Incomplete,
"interval": resp.Interval,
"min interval": resp.MinInterval,
}
bb.WriteString("d8:completei")
bb.WriteString(strconv.FormatUint(uint64(resp.Complete), 10))
bb.WriteString("e10:incompletei")
bb.WriteString(strconv.FormatUint(uint64(resp.Incomplete), 10))
bb.WriteString("e8:intervali")
bb.WriteString(strconv.FormatUint(uint64(resp.Interval), 10))
bb.WriteString("e12:min intervali")
bb.WriteString(strconv.FormatUint(uint64(resp.MinInterval), 10))
bb.WriteByte('e')
// Add the peers to the dictionary in the compact format.
if resp.Compact {
// Add the IPv4 peers to the dictionary.
compactAddresses := make([]byte, 0, (net.IPv4len+2)*len(resp.IPv4Peers))
for _, peer := range resp.IPv4Peers {
compactAddresses = append(compactAddresses, compactAddress(peer)...)
}
if len(compactAddresses) > 0 {
bdict["peers"] = compactAddresses
}
compactAddresses(bb, resp.IPv4Peers, false)
// Add the IPv6 peers to the dictionary.
compactAddresses = make([]byte, 0, (net.IPv6len+2)*len(resp.IPv6Peers)) // IP + port
for _, peer := range resp.IPv6Peers {
compactAddresses = append(compactAddresses, compactAddress(peer)...)
}
if len(compactAddresses) > 0 {
bdict["peers6"] = compactAddresses
}
compactAddresses(bb, resp.IPv6Peers, true)
} else {
// Add the peers to the dictionary.
peers := make([]map[string]any, 0, len(resp.IPv4Peers)+len(resp.IPv6Peers)) // IP + port
bb.WriteString("5:peersl")
for _, peer := range resp.IPv4Peers {
peers = append(peers, dict(peer))
dictAddress(bb, peer)
}
for _, peer := range resp.IPv6Peers {
peers = append(peers, dict(peer))
dictAddress(bb, peer)
}
bdict["peers"] = peers
bb.WriteByte('e')
}
bb.WriteByte('e')
return bencode.NewEncoder(w).Encode(bdict)
_, err := bb.WriteTo(w)
return err
}
func compactAddresses(bb *bytes.Buffer, peers bittorrent.Peers, v6 bool) {
l := len(peers)
if l > 0 {
key, al := "5:peers", net.IPv4len
if v6 {
key, al = "6:peers6", net.IPv6len
}
bb.WriteString(key)
bb.WriteString(strconv.Itoa((al + 2) * l))
bb.WriteByte(':')
for _, peer := range peers {
bb.Write(peer.Addr().AsSlice())
port := peer.Port()
bb.WriteByte(byte(port >> 8))
bb.WriteByte(byte(port))
}
}
}
func dictAddress(bb *bytes.Buffer, peer bittorrent.Peer) {
bb.WriteString("d2:ip")
addr := peer.Addr().String()
bb.WriteString(strconv.Itoa(len(addr)))
bb.WriteByte(':')
bb.WriteString(addr)
bb.WriteString("7:peer id20:")
bb.Write(peer.ID[:])
bb.WriteString("4:porti")
bb.WriteString(strconv.FormatUint(uint64(peer.Port()), 10))
bb.WriteString("ee")
}
// WriteScrapeResponse communicates the results of a Scrape to a BitTorrent
// client over HTTP.
func WriteScrapeResponse(w http.ResponseWriter, resp *bittorrent.ScrapeResponse) error {
filesDict := make(map[string]any, len(resp.Files))
bb := respBufferPool.Get()
defer respBufferPool.Put(bb)
bb.WriteString("d5:filesd")
for _, scrape := range resp.Files {
filesDict[string(scrape.InfoHash[:])] = map[string]any{
"complete": scrape.Complete,
"incomplete": scrape.Incomplete,
}
}
return bencode.NewEncoder(w).Encode(map[string]any{
"files": filesDict,
})
}
func compactAddress(peer bittorrent.Peer) (buf []byte) {
buf = append(buf, peer.Addr().AsSlice()...)
port := peer.Port()
buf = append(buf, byte(port>>8), byte(port))
return
}
func dict(peer bittorrent.Peer) map[string]any {
return map[string]any{
"peer id": peer.ID.RawString(),
"ip": peer.Addr(),
"port": peer.Port(),
bb.WriteString(strconv.Itoa(len(scrape.InfoHash)))
bb.WriteByte(':')
bb.Write([]byte(scrape.InfoHash))
bb.WriteString("d8:completei")
bb.WriteString(strconv.FormatUint(uint64(scrape.Complete), 10))
bb.WriteString("e10:downloadedi")
bb.WriteString(strconv.FormatUint(uint64(scrape.Snatches), 10))
bb.WriteString("e10:incompletei")
bb.WriteString(strconv.FormatUint(uint64(scrape.Incomplete), 10))
bb.WriteString("ee")
}
bb.WriteString("ee")
_, err := bb.WriteTo(w)
return err
}
frontend/options.go
+96 -2
View File
@@ -1,8 +1,102 @@
package frontend
import "github.com/sot-tech/mochi/pkg/log"
import (
"errors"
"net"
"time"
var logger = log.NewLogger("frontend configurator")
"github.com/libp2p/go-reuseport"
)
const (
defaultReadTimeout = 2 * time.Second
defaultWriteTimeout = 2 * time.Second
)
var (
// ErrAddressNotProvided returned if listen address not provided in configuration
ErrAddressNotProvided = errors.New("address not provided")
errUnexpectedListenerType = errors.New("unexpected listener type")
)
// ListenOptions is the base configuration which may be used in net listeners
type ListenOptions struct {
Addr string
ReusePort bool `cfg:"reuse_port"`
ReadTimeout time.Duration `cfg:"read_timeout"`
WriteTimeout time.Duration `cfg:"write_timeout"`
EnableRequestTiming bool `cfg:"enable_request_timing"`
}
// Validate checks if listen address provided and sets default
// timeout options if needed
func (lo ListenOptions) Validate() (validOptions ListenOptions, err error) {
validOptions = lo
if len(lo.Addr) == 0 {
err = ErrAddressNotProvided
} else {
if lo.ReadTimeout <= 0 {
validOptions.ReadTimeout = defaultReadTimeout
logger.Warn().
Str("name", "ReadTimeout").
Dur("provided", lo.ReadTimeout).
Dur("default", validOptions.ReadTimeout).
Msg("falling back to default configuration")
}
if lo.WriteTimeout <= 0 {
validOptions.WriteTimeout = defaultWriteTimeout
logger.Warn().
Str("name", "WriteTimeout").
Dur("provided", lo.WriteTimeout).
Dur("default", validOptions.WriteTimeout).
Msg("falling back to default configuration")
}
}
return
}
// ListenTCP listens at the given TCP Addr
// with SO_REUSEPORT and SO_REUSEADDR options enabled if
// ReusePort set to true
func (lo ListenOptions) ListenTCP() (conn *net.TCPListener, err error) {
if lo.ReusePort {
var ln net.Listener
if ln, err = reuseport.Listen("tcp", lo.Addr); err == nil {
var ok bool
if conn, ok = ln.(*net.TCPListener); !ok {
err = errUnexpectedListenerType
}
}
} else {
var addr *net.TCPAddr
if addr, err = net.ResolveTCPAddr("tcp", lo.Addr); err == nil {
conn, err = net.ListenTCP("tcp", addr)
}
}
return
}
// ListenUDP listens at the given UDP Addr
// with SO_REUSEPORT and SO_REUSEADDR options enabled if
// ReusePort set to true
func (lo ListenOptions) ListenUDP() (conn *net.UDPConn, err error) {
if lo.ReusePort {
var ln net.PacketConn
if ln, err = reuseport.ListenPacket("udp", lo.Addr); err == nil {
var ok bool
if conn, ok = ln.(*net.UDPConn); !ok {
err = errUnexpectedListenerType
}
}
} else {
var addr *net.UDPAddr
if addr, err = net.ResolveUDPAddr("udp", lo.Addr); err == nil {
conn, err = net.ListenUDP("udp", addr)
}
}
return
}
// ParseOptions is the configuration used to parse an Announce Request.
//
frontend/udp/frontend.go
+70 -81
View File
@@ -13,11 +13,10 @@ import (
"sync"
"time"
"github.com/libp2p/go-reuseport"
"github.com/sot-tech/mochi/bittorrent"
"github.com/sot-tech/mochi/frontend"
"github.com/sot-tech/mochi/frontend/udp/bytepool"
"github.com/sot-tech/mochi/middleware"
"github.com/sot-tech/mochi/pkg/bytepool"
"github.com/sot-tech/mochi/pkg/conf"
"github.com/sot-tech/mochi/pkg/log"
"github.com/sot-tech/mochi/pkg/metrics"
@@ -26,28 +25,32 @@ import (
)
var (
logger = log.NewLogger("udp frontend")
logger = log.NewLogger("frontend/udp")
allowedGeneratedPrivateKeyRunes = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890")
errUnexpectedConnType = errors.New("unexpected connection type (not UDPConn)")
)
// Config represents all of the configurable options for a UDP BitTorrent
func init() {
frontend.RegisterBuilder("udp", NewFrontend)
}
// Config represents all the configurable options for a UDP BitTorrent
// Tracker.
type Config struct {
Addr string
ReusePort bool `cfg:"reuse_port"`
PrivateKey string `cfg:"private_key"`
MaxClockSkew time.Duration `cfg:"max_clock_skew"`
EnableRequestTiming bool `cfg:"enable_request_timing"`
frontend.ListenOptions
PrivateKey string `cfg:"private_key"`
MaxClockSkew time.Duration `cfg:"max_clock_skew"`
frontend.ParseOptions
}
// Validate sanity checks values set in a config and returns a new config with
// default values replacing anything that is invalid.
//
// This function warns to the logger when a value is changed.
func (cfg Config) Validate() Config {
validcfg := cfg
func (cfg Config) Validate() (validCfg Config, err error) {
if len(cfg.Addr) == 0 {
err = frontend.ErrAddressNotProvided
return
}
validCfg = cfg
// Generate a private key if one isn't provided by the user.
if cfg.PrivateKey == "" {
@@ -55,45 +58,49 @@ func (cfg Config) Validate() Config {
for i := range pkeyRunes {
pkeyRunes[i] = allowedGeneratedPrivateKeyRunes[rand.Intn(len(allowedGeneratedPrivateKeyRunes))]
}
validcfg.PrivateKey = string(pkeyRunes)
validCfg.PrivateKey = string(pkeyRunes)
logger.Warn().
Str("name", "UDP.PrivateKey").
Str("name", "PrivateKey").
Str("provided", "").
Str("key", validcfg.PrivateKey).
Str("key", validCfg.PrivateKey).
Msg("falling back to default configuration")
}
validcfg.ParseOptions = cfg.ParseOptions.Validate()
validCfg.ParseOptions = cfg.ParseOptions.Validate()
return validcfg
return
}
// Frontend holds the state of a UDP BitTorrent Frontend.
type Frontend struct {
socket *net.UDPConn
closing chan struct{}
wg sync.WaitGroup
genPool *sync.Pool
logic frontend.TrackerLogic
Config
// udpFE holds the state of a UDP BitTorrent Frontend.
type udpFE struct {
socket *net.UDPConn
closing chan any
wg sync.WaitGroup
genPool *sync.Pool
logic *middleware.Logic
maxClockSkew time.Duration
collectTimings bool
frontend.ParseOptions
}
// NewFrontend creates a new instance of an UDP Frontend that asynchronously
// serves requests.
func NewFrontend(logic frontend.TrackerLogic, c conf.MapConfig) (*Frontend, error) {
var provided Config
if err := c.Unmarshal(&provided); err != nil {
// NewFrontend builds and starts udp bittorrent frontend from provided configuration
func NewFrontend(c conf.MapConfig, logic *middleware.Logic) (frontend.Frontend, error) {
var err error
var cfg Config
if err = c.Unmarshal(&cfg); err != nil {
return nil, err
}
if cfg, err = cfg.Validate(); err != nil {
return nil, err
}
cfg := provided.Validate()
f := &Frontend{
closing: make(chan struct{}),
logic: logic,
Config: cfg,
f := &udpFE{
closing: make(chan any),
logic: logic,
maxClockSkew: cfg.MaxClockSkew,
collectTimings: cfg.EnableRequestTiming,
ParseOptions: cfg.ParseOptions,
genPool: &sync.Pool{
New: func() any {
return NewConnectionIDGenerator(cfg.PrivateKey)
@@ -101,22 +108,20 @@ func NewFrontend(logic frontend.TrackerLogic, c conf.MapConfig) (*Frontend, erro
},
}
if err := f.listen(); err != nil {
return nil, err
if f.socket, err = cfg.ListenUDP(); err == nil {
f.wg.Add(1)
go func() {
if err := f.serve(); err != nil {
logger.Fatal().Err(err).Msg("server failed")
}
}()
}
f.wg.Add(1)
go func() {
if err := f.serve(); err != nil {
logger.Fatal().Err(err).Str("proto", "udp").Msg("failed while serving")
}
}()
return f, nil
return f, err
}
// Stop provides a thread-safe way to shut down a currently running Frontend.
func (t *Frontend) Stop() stop.Result {
func (t *udpFE) Stop() stop.Result {
select {
case <-t.closing:
return stop.AlreadyStopped
@@ -126,42 +131,26 @@ func (t *Frontend) Stop() stop.Result {
c := make(stop.Channel)
go func() {
close(t.closing)
_ = t.socket.SetReadDeadline(time.Now())
t.wg.Wait()
c.Done(t.socket.Close())
var err error
if t.socket != nil {
_ = t.socket.SetReadDeadline(time.Now())
t.wg.Wait()
err = t.socket.Close()
}
c.Done(err)
}()
return c.Result()
}
// listen resolves the address and binds the server socket.
func (t *Frontend) listen() (err error) {
if t.ReusePort {
var ln net.PacketConn
if ln, err = reuseport.ListenPacket("udp", t.Addr); err == nil {
var isOk bool
if t.socket, isOk = ln.(*net.UDPConn); !isOk {
err = errUnexpectedConnType
}
}
} else {
var udpAddr *net.UDPAddr
udpAddr, err = net.ResolveUDPAddr("udp", t.Addr)
if err == nil {
t.socket, err = net.ListenUDP("udp", udpAddr)
}
}
return err
}
// serve blocks while listening and serving UDP BitTorrent requests
// until Stop() is called or an error is returned.
func (t *Frontend) serve() error {
pool := bytepool.New(2048)
func (t *udpFE) serve() error {
pool := bytepool.NewBytePool(2048)
defer t.wg.Done()
for {
// Check to see if we need to shutdown.
// Check to see if we need shutdown.
select {
case <-t.closing:
log.Debug().Msg("serve received shutdown signal")
@@ -196,14 +185,14 @@ func (t *Frontend) serve() error {
// Handle the request.
addr := addrPort.Addr().Unmap()
var start time.Time
if t.EnableRequestTiming && metrics.Enabled() {
if t.collectTimings && metrics.Enabled() {
start = time.Now()
}
action, err := t.handleRequest(
Request{(*buffer)[:n], addr},
ResponseWriter{t.socket, addrPort},
)
if t.EnableRequestTiming && metrics.Enabled() {
if t.collectTimings && metrics.Enabled() {
recordResponseDuration(action, addr, err, time.Since(start))
}
}()
@@ -229,7 +218,7 @@ func (w ResponseWriter) Write(b []byte) (int, error) {
}
// handleRequest parses and responds to a UDP Request.
func (t *Frontend) handleRequest(r Request, w ResponseWriter) (actionName string, err error) {
func (t *udpFE) handleRequest(r Request, w ResponseWriter) (actionName string, err error) {
if len(r.Packet) < 16 {
// Malformed, no client packets are less than 16 bytes.
// We explicitly return nothing in case this is a DoS attempt.
@@ -248,7 +237,7 @@ func (t *Frontend) handleRequest(r Request, w ResponseWriter) (actionName string
// If this isn't requesting a new connection ID and the connection ID is
// invalid, then fail.
if actionID != connectActionID && !gen.Validate(connID, r.IP, timecache.Now(), t.MaxClockSkew) {
if actionID != connectActionID && !gen.Validate(connID, r.IP, timecache.Now(), t.maxClockSkew) {
err = errBadConnectionID
WriteError(w, txID, err)
return
frontend/udp/frontend_test.go
+5 -2
View File
@@ -17,12 +17,15 @@ func init() {
}
func TestStartStopRaceIssue437(t *testing.T) {
ps, err := storage.NewStorage("memory", conf.MapConfig{})
ps, err := storage.NewStorage(conf.NamedMapConfig{
Name: "memory",
Config: conf.MapConfig{},
})
if err != nil {
t.Fatal(err)
}
lgc := middleware.NewLogic(0, 0, ps, nil, nil)
fe, err := udp.NewFrontend(lgc, conf.MapConfig{"addr": "127.0.0.1:0"})
fe, err := udp.NewFrontend(conf.MapConfig{"addr": "127.0.0.1:0"}, lgc)
if err != nil {
t.Fatal(err)
}
frontend/udp/parser.go
+5 -21
View File
@@ -1,15 +1,14 @@
package udp
import (
"bytes"
"encoding/binary"
"fmt"
"net"
"net/netip"
"sync"
"github.com/sot-tech/mochi/bittorrent"
"github.com/sot-tech/mochi/frontend"
"github.com/sot-tech/mochi/pkg/bytepool"
)
const (
@@ -48,6 +47,8 @@ var (
errUnknownOptionType = bittorrent.ClientError("unknown option type")
errInvalidInfoHash = bittorrent.ClientError("invalid info hash")
errInvalidPeerID = bittorrent.ClientError("invalid info hash")
reqRespBufferPool = bytepool.NewBufferPool()
)
// ParseAnnounce parses an AnnounceRequest from a UDP request.
@@ -111,23 +112,6 @@ func ParseAnnounce(r Request, v6Action bool, opts frontend.ParseOptions) (*bitto
return request, err
}
type buffer struct {
bytes.Buffer
}
var bufferFree = sync.Pool{
New: func() any { return new(buffer) },
}
func newBuffer() *buffer {
return bufferFree.Get().(*buffer)
}
func (b *buffer) free() {
b.Reset()
bufferFree.Put(b)
}
// handleOptionalParameters parses the optional parameters as described in BEP
// 41 and updates an announce with the values parsed.
func handleOptionalParameters(packet []byte) (bittorrent.Params, error) {
@@ -135,8 +119,8 @@ func handleOptionalParameters(packet []byte) (bittorrent.Params, error) {
return bittorrent.ParseURLData("")
}
buf := newBuffer()
defer buf.free()
buf := reqRespBufferPool.Get()
defer reqRespBufferPool.Put(buf)
for i := 0; i < len(packet); {
option := packet[i]
frontend/udp/writer.go
+8 -8
View File
@@ -18,8 +18,8 @@ func WriteError(w io.Writer, txID []byte, err error) {
err = fmt.Errorf("internal error occurred: %w", err)
}
buf := newBuffer()
defer buf.free()
buf := reqRespBufferPool.Get()
defer reqRespBufferPool.Put(buf)
writeHeader(buf, txID, errorActionID)
_, _ = buf.WriteString(err.Error())
_, _ = buf.WriteRune('\000')
@@ -32,8 +32,8 @@ func WriteError(w io.Writer, txID []byte, err error) {
// If v6Action is set, the action will be 4, according to
// https://web.archive.org/web/20170503181830/http://opentracker.blog.h3q.com/2007/12/28/the-ipv6-situation/
func WriteAnnounce(w io.Writer, txID []byte, resp *bittorrent.AnnounceResponse, v6Action, v6Peers bool) {
buf := newBuffer()
defer buf.free()
buf := reqRespBufferPool.Get()
defer reqRespBufferPool.Put(buf)
if v6Action {
writeHeader(buf, txID, announceV6ActionID)
@@ -59,8 +59,8 @@ func WriteAnnounce(w io.Writer, txID []byte, resp *bittorrent.AnnounceResponse,
// WriteScrape encodes a scrape response according to BEP 15.
func WriteScrape(w io.Writer, txID []byte, resp *bittorrent.ScrapeResponse) {
buf := newBuffer()
defer buf.free()
buf := reqRespBufferPool.Get()
defer reqRespBufferPool.Put(buf)
writeHeader(buf, txID, scrapeActionID)
@@ -75,8 +75,8 @@ func WriteScrape(w io.Writer, txID []byte, resp *bittorrent.ScrapeResponse) {
// WriteConnectionID encodes a new connection response according to BEP 15.
func WriteConnectionID(w io.Writer, txID, connID []byte) {
buf := newBuffer()
defer buf.free()
buf := reqRespBufferPool.Get()
defer reqRespBufferPool.Put(buf)
writeHeader(buf, txID, connectActionID)
_, _ = buf.Write(connID)
go.mod
+13 -13
View File
@@ -3,48 +3,48 @@ module github.com/sot-tech/mochi
go 1.18
require (
code.cloudfoundry.org/go-diodes v0.0.0-20220927211948-2985a72b297c
github.com/MicahParks/keyfunc v1.4.0
code.cloudfoundry.org/go-diodes v0.0.0-20221017175818-728392b37655
github.com/MicahParks/keyfunc v1.5.1
github.com/anacrolix/torrent v1.47.0
github.com/go-redis/redis/v8 v8.11.5
github.com/golang-jwt/jwt/v4 v4.4.2
github.com/jackc/pgx/v5 v5.0.1
github.com/jackc/pgx/v5 v5.0.4
github.com/julienschmidt/httprouter v1.3.0
github.com/libp2p/go-reuseport v0.2.0
github.com/minio/sha256-simd v1.0.0
github.com/mitchellh/mapstructure v1.5.0
github.com/prometheus/client_golang v1.13.0
github.com/rs/zerolog v1.28.0
github.com/stretchr/testify v1.8.0
github.com/stretchr/testify v1.8.1
gopkg.in/yaml.v3 v3.0.1
)
require (
github.com/anacrolix/dht/v2 v2.19.0 // indirect
github.com/anacrolix/dht/v2 v2.19.1 // indirect
github.com/anacrolix/log v0.13.2-0.20220711050817-613cb738ef30 // indirect
github.com/anacrolix/missinggo v1.3.0 // indirect
github.com/anacrolix/missinggo/v2 v2.7.0 // indirect
github.com/anacrolix/missinggo/v2 v2.7.1 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 // indirect
github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/fsnotify/fsnotify v1.5.4 // indirect
github.com/fsnotify/fsnotify v1.6.0 // indirect
github.com/golang/protobuf v1.5.2 // indirect
github.com/huandu/xstrings v1.3.2 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect
github.com/jackc/puddle/v2 v2.0.0 // indirect
github.com/klauspost/cpuid/v2 v2.1.1 // indirect
github.com/klauspost/cpuid/v2 v2.1.2 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.16 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.2 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_model v0.2.0 // indirect
github.com/prometheus/client_model v0.3.0 // indirect
github.com/prometheus/common v0.37.0 // indirect
github.com/prometheus/procfs v0.8.0 // indirect
golang.org/x/crypto v0.0.0-20220926161630-eccd6366d1be // indirect
golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec // indirect
golang.org/x/text v0.3.7 // indirect
golang.org/x/crypto v0.1.0 // indirect
golang.org/x/sys v0.1.0 // indirect
golang.org/x/text v0.4.0 // indirect
google.golang.org/protobuf v1.28.1 // indirect
)
go.sum
+30 -26
View File
@@ -30,15 +30,15 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo
cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
code.cloudfoundry.org/go-diodes v0.0.0-20220927211948-2985a72b297c h1:6Gb9bHc7g8thQ4AWxwdA6qxhtDNcqNw2T1+Z+w37kOw=
code.cloudfoundry.org/go-diodes v0.0.0-20220927211948-2985a72b297c/go.mod h1:v30sK4Ipg4Ng0pRG22iLfeMRfYUHQNee9f4NkE5t1yc=
code.cloudfoundry.org/go-diodes v0.0.0-20221017175818-728392b37655 h1:3IlYr3dIyXkemR1GoOITnBWnWBHG27wnC6mpV9OLD7c=
code.cloudfoundry.org/go-diodes v0.0.0-20221017175818-728392b37655/go.mod h1:T+qZrTfw8xwdJgVRD5pSvVyx4CeqsP30YQooBYFADRc=
crawshaw.io/iox v0.0.0-20181124134642-c51c3df30797/go.mod h1:sXBiorCo8c46JlQV3oXPKINnZ8mcqnye1EkVkqsectk=
crawshaw.io/sqlite v0.3.2/go.mod h1:igAO5JulrQ1DbdZdtVq48mnZUBAPOeFzer7VhDWNtW4=
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/MicahParks/keyfunc v1.4.0 h1:/qTnDiQJSM0pkUvKRVb82jvvm0SAOffAqKSIrgUqwdI=
github.com/MicahParks/keyfunc v1.4.0/go.mod h1:IdnCilugA0O/99dW+/MkvlyrsX8+L8+x95xuVNtM5jw=
github.com/MicahParks/keyfunc v1.5.1 h1:RlyyYgKQI/adkIw1yXYtPvTAOb7hBhSX42aH23d8N0Q=
github.com/MicahParks/keyfunc v1.5.1/go.mod h1:IdnCilugA0O/99dW+/MkvlyrsX8+L8+x95xuVNtM5jw=
github.com/RoaringBitmap/roaring v0.4.7/go.mod h1:8khRDP4HmeXns4xIj9oGrKSz7XTQiJx2zgh7AcNke4w=
github.com/RoaringBitmap/roaring v0.4.17/go.mod h1:D3qVegWTmfCaX4Bl5CrBE9hfrSrrXIr8KVNvRsDi1NI=
github.com/RoaringBitmap/roaring v0.4.23/go.mod h1:D0gp8kJQgE1A4LQ5wFLggQEyvDi06Mq5mKs52e1TwOo=
@@ -49,8 +49,8 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
github.com/anacrolix/dht/v2 v2.19.0 h1:A9oMHWRGbLmCyx1JlYzg79bDrur8V60+0ts8ZwEVYt4=
github.com/anacrolix/dht/v2 v2.19.0/go.mod h1:0h83KnnAQ2AUYhpQ/CkoZP45K41pjDAlPR9zGHgFjQE=
github.com/anacrolix/dht/v2 v2.19.1 h1:V/UUGBASGYqYkSnmHJwX8uQmzkyhbgwE6jqcHKnNTD8=
github.com/anacrolix/dht/v2 v2.19.1/go.mod h1:3TU93c1s/oA8I/VH4m3CNP/BeKsiOGmo6HwfZBMTKUs=
github.com/anacrolix/envpprof v0.0.0-20180404065416-323002cec2fa/go.mod h1:KgHhUaQMc8cC0+cEflSgCFNFbKwi5h54gqtVn8yhP7c=
github.com/anacrolix/envpprof v1.0.0/go.mod h1:KgHhUaQMc8cC0+cEflSgCFNFbKwi5h54gqtVn8yhP7c=
github.com/anacrolix/envpprof v1.1.0/go.mod h1:My7T5oSqVfEn4MD4Meczkw/f5lSIndGAKu/0SM/rkf4=
@@ -67,8 +67,8 @@ github.com/anacrolix/missinggo v1.3.0/go.mod h1:bqHm8cE8xr+15uVfMG3BFui/TxyB6//H
github.com/anacrolix/missinggo/perf v1.0.0/go.mod h1:ljAFWkBuzkO12MQclXzZrosP5urunoLS0Cbvb4V0uMQ=
github.com/anacrolix/missinggo/v2 v2.2.0/go.mod h1:o0jgJoYOyaoYQ4E2ZMISVa9c88BbUBVQQW4QeRkNCGY=
github.com/anacrolix/missinggo/v2 v2.5.1/go.mod h1:WEjqh2rmKECd0t1VhQkLGTdIWXO6f6NLjp5GlMZ+6FA=
github.com/anacrolix/missinggo/v2 v2.7.0 h1:4fzOAAn/VCvfWGviLmh64MPMttrlYew81JdPO7nSHvI=
github.com/anacrolix/missinggo/v2 v2.7.0/go.mod h1:2IZIvmRTizALNYFYXsPR7ofXPzJgyBpKZ4kMqMEICkI=
github.com/anacrolix/missinggo/v2 v2.7.1 h1:Y+wL0JC6D2icpwhDpcrRM4THQB/uFcPNYUtZMbYvQgI=
github.com/anacrolix/missinggo/v2 v2.7.1/go.mod h1:2IZIvmRTizALNYFYXsPR7ofXPzJgyBpKZ4kMqMEICkI=
github.com/anacrolix/stm v0.2.0/go.mod h1:zoVQRvSiGjGoTmbM0vSLIiaKjWtNPeTvXUSdJQA4hsg=
github.com/anacrolix/tagflag v0.0.0-20180109131632-2146c8d41bf0/go.mod h1:1m2U/K6ZT+JZG0+bdMK6qauP49QT4wE5pmhJXOKKCHw=
github.com/anacrolix/tagflag v1.0.0/go.mod h1:1m2U/K6ZT+JZG0+bdMK6qauP49QT4wE5pmhJXOKKCHw=
@@ -113,8 +113,8 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI=
github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU=
github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE=
github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE=
github.com/glycerine/go-unsnap-stream v0.0.0-20190901134440-81cf024a9e0a/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE=
@@ -215,8 +215,8 @@ github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsI
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg=
github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E=
github.com/jackc/pgx/v5 v5.0.1 h1:JZu9othr7l8so2JMDAGeDUMXqERAuZpovyfl4H50tdg=
github.com/jackc/pgx/v5 v5.0.1/go.mod h1:JBbvW3Hdw77jKl9uJrEDATUZIFM2VFPzRq4RWIhkF4o=
github.com/jackc/pgx/v5 v5.0.4 h1:r5O6y84qHX/z/HZV40JBdx2obsHz7/uRj5b+CcYEdeY=
github.com/jackc/pgx/v5 v5.0.4/go.mod h1:U0ynklHtgg43fue9Ly30w3OCSTDPlXjig9ghrNGaguQ=
github.com/jackc/puddle/v2 v2.0.0 h1:Kwk/AlLigcnZsDssc3Zun1dk1tAtQNPaBBxBHWn0Mjc=
github.com/jackc/puddle/v2 v2.0.0/go.mod h1:itE7ZJY8xnoo0JqJEpSMprN0f+NQkMCuEV/N9j8h0oc=
github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
@@ -234,8 +234,8 @@ github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4d
github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.1.1 h1:t0wUqjowdm8ezddV5k0tLWVklVuvLJpoHeb4WBdydm0=
github.com/klauspost/cpuid/v2 v2.1.1/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY=
github.com/klauspost/cpuid/v2 v2.1.2 h1:XhdX4fqAJUA0yj+kUwMavO0hHrSPAecYdYf1ZmxHvak=
github.com/klauspost/cpuid/v2 v2.1.2/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
@@ -255,8 +255,8 @@ github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27k
github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ=
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/matttproud/golang_protobuf_extensions v1.0.2 h1:hAHbPm5IJGijwng3PWk09JkG9WeqChjprR5s9bBZ+OM=
github.com/matttproud/golang_protobuf_extensions v1.0.2/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g=
github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM=
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
@@ -275,7 +275,7 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W
github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/onsi/gomega v1.20.2 h1:8uQq0zMgLEfa0vRrrBgaJF2gyW9Da9BmfGV+OyUzfkY=
github.com/onsi/gomega v1.22.1 h1:pY8O4lBfsHKZHM/6nrxkhVPUznOlIu3quZcKP/M20KI=
github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw=
github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
@@ -298,8 +298,9 @@ github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:
github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M=
github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4=
github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w=
github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4=
@@ -337,6 +338,7 @@ github.com/smartystreets/goconvey v0.0.0-20190306220146-200a235640ff/go.mod h1:K
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
@@ -344,8 +346,9 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
github.com/tinylib/msgp v1.1.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
@@ -367,8 +370,8 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20220926161630-eccd6366d1be h1:fmw3UbQh+nxngCAHrDCCztao/kbYFnWjoqop8dHx05A=
golang.org/x/crypto v0.0.0-20220926161630-eccd6366d1be/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU=
golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -432,7 +435,7 @@ golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81R
golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b h1:PxfKdU9lEEDYjdIzOtC4qFWgkU2rGHdKlKowJSMN9h0=
golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -492,11 +495,11 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec h1:BkDtF2Ih9xZ7le9ndzTA7KJow28VbQW3odyk/8drmuI=
golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U=
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -505,8 +508,9 @@ golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg=
golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
middleware/clientapproval/clientapproval.go
+2 -2
View File
@@ -39,10 +39,10 @@ type hook struct {
unapproved map[ClientID]struct{}
}
func build(options conf.MapConfig, _ storage.PeerStorage) (middleware.Hook, error) {
func build(config conf.MapConfig, _ storage.PeerStorage) (middleware.Hook, error) {
var cfg Config
if err := options.Unmarshal(&cfg); err != nil {
if err := config.Unmarshal(&cfg); err != nil {
return nil, fmt.Errorf("middleware %s: %w", Name, err)
}
middleware/hooks.go
+1 -1
View File
@@ -19,7 +19,7 @@ type Hook interface {
}
// Pinger is an optional interface that may be implemented by a pre Hook
// to check if it is operational. Used in frontend.TrackerLogic.
// to check if it is operational. Used in frontend.Logic.
//
// It may be useful in cases when Hook performs foreign requests to
// some external resources (i.e. storage) and `ping` request should
middleware/jwt/jwt.go
+8 -12
View File
@@ -23,19 +23,17 @@ import (
"github.com/sot-tech/mochi/storage"
)
// Name is the name by which this middleware is registered with Conf.
const (
Name = "jwt"
authorizationHeader = "authorization"
bearerAuthPrefix = "bearer "
)
func init() {
middleware.RegisterBuilder(Name, build)
middleware.RegisterBuilder("jwt", build)
}
var (
logger = log.NewLogger(Name)
logger = log.NewLogger("middleware/jwt")
// ErrMissingJWT is returned when a JWT is missing from a request.
ErrMissingJWT = bittorrent.ClientError("unapproved request: missing jwt")
@@ -70,13 +68,11 @@ type hook struct {
jwks *keyfunc.JWKS
}
func build(options conf.MapConfig, _ storage.PeerStorage) (h middleware.Hook, err error) {
func build(config conf.MapConfig, _ storage.PeerStorage) (h middleware.Hook, err error) {
var cfg Config
logger.Debug().Object("options", options).Msg("creating new JWT middleware")
if err = options.Unmarshal(&cfg); err != nil {
return nil, fmt.Errorf("middleware %s: %w", Name, err)
if err = config.Unmarshal(&cfg); err != nil {
return nil, fmt.Errorf("unable to deserialise configuration: %w", err)
}
if len(cfg.JWKSetURL) > 0 && len(cfg.Issuer) > 0 && len(cfg.Audience) > 0 {
@@ -206,7 +202,7 @@ func (h *hook) HandleScrape(ctx context.Context, req *bittorrent.ScrapeRequest,
if errs := h.validateBaseJWT(jwtParam, claims); len(errs) > 0 {
logger.Info().
Errs("errors", errs).
Array("source", req.RequestAddresses).
Array("source", &req.RequestAddresses).
Msg("JWT validation failed")
err = ErrInvalidJWT
} else {
@@ -217,7 +213,7 @@ func (h *hook) HandleScrape(ctx context.Context, req *bittorrent.ScrapeRequest,
} else {
logger.Info().
Err(err).
Array("addresses", req.RequestAddresses).
Array("addresses", &req.RequestAddresses).
Msg("'infohashes' claim parse failed")
}
}
@@ -241,7 +237,7 @@ func (h *hook) HandleScrape(ctx context.Context, req *bittorrent.ScrapeRequest,
logger.Info().
Array("claimInfoHashes", claimIHs).
Array("requestInfoHashes", req.InfoHashes).
Array("addresses", req.RequestAddresses).
Array("addresses", &req.RequestAddresses).
Msg("unequal 'infohashes' claim when validating JWT")
err = ErrInvalidJWT
}
middleware/logic.go
+23 -22
View File
@@ -2,21 +2,26 @@ package middleware
import (
"context"
"sort"
"time"
"github.com/sot-tech/mochi/bittorrent"
"github.com/sot-tech/mochi/frontend"
"github.com/sot-tech/mochi/pkg/log"
"github.com/sot-tech/mochi/pkg/stop"
"github.com/sot-tech/mochi/storage"
)
var (
logger = log.NewLogger("middleware")
_ frontend.TrackerLogic = &Logic{}
)
// Logic used by a frontend in order to: (1) generate a
// response from a parsed request, and (2) asynchronously observe anything
// after the response has been delivered to the client.
type Logic struct {
announceInterval time.Duration
minAnnounceInterval time.Duration
preHooks []Hook
postHooks []Hook
pingers []Pinger
}
// NewLogic creates a new instance of a TrackerLogic that executes the provided
// NewLogic creates a new instance of a Logic that executes the provided
// middleware hooks.
func NewLogic(annInterval, minAnnInterval time.Duration, peerStore storage.PeerStorage, preHooks, postHooks []Hook) *Logic {
l := &Logic{
@@ -34,17 +39,10 @@ func NewLogic(annInterval, minAnnInterval time.Duration, peerStore storage.PeerS
return l
}
// Logic is an implementation of the TrackerLogic that functions by
// executing a series of middleware hooks.
type Logic struct {
announceInterval time.Duration
minAnnounceInterval time.Duration
preHooks []Hook
postHooks []Hook
pingers []Pinger
}
// HandleAnnounce generates a response for an Announce.
//
// Returns the updated context, the generated AnnounceResponse and no error
// on success; nil and error on failure.
func (l *Logic) HandleAnnounce(ctx context.Context, req *bittorrent.AnnounceRequest) (_ context.Context, resp *bittorrent.AnnounceResponse, err error) {
logger.Debug().Object("request", req).Msg("new announce request")
resp = &bittorrent.AnnounceResponse{
@@ -62,8 +60,8 @@ func (l *Logic) HandleAnnounce(ctx context.Context, req *bittorrent.AnnounceRequ
return ctx, resp, nil
}
// AfterAnnounce does something with the results of an Announce after it has
// been completed.
// AfterAnnounce does something with the results of an Announce after it
// has been completed.
func (l *Logic) AfterAnnounce(ctx context.Context, req *bittorrent.AnnounceRequest, resp *bittorrent.AnnounceResponse) {
var err error
for _, h := range l.postHooks {
@@ -78,6 +76,9 @@ func (l *Logic) AfterAnnounce(ctx context.Context, req *bittorrent.AnnounceReque
}
// HandleScrape generates a response for a Scrape.
//
// Returns the updated context, the generated AnnounceResponse and no error
// on success; nil and error on failure.
func (l *Logic) HandleScrape(ctx context.Context, req *bittorrent.ScrapeRequest) (_ context.Context, resp *bittorrent.ScrapeResponse, err error) {
logger.Debug().Object("request", req).Msg("new scrape request")
resp = &bittorrent.ScrapeResponse{
@@ -88,13 +89,13 @@ func (l *Logic) HandleScrape(ctx context.Context, req *bittorrent.ScrapeRequest)
return nil, nil, err
}
}
sort.Sort(&resp.Files)
logger.Debug().Object("response", resp).Msg("generated scrape response")
return ctx, resp, nil
}
// AfterScrape does something with the results of a Scrape after it has been
// completed.
// AfterScrape does something with the results of a Scrape after it has been completed.
func (l *Logic) AfterScrape(ctx context.Context, req *bittorrent.ScrapeRequest, resp *bittorrent.ScrapeResponse) {
var err error
for _, h := range l.postHooks {
@@ -109,7 +110,7 @@ func (l *Logic) AfterScrape(ctx context.Context, req *bittorrent.ScrapeRequest,
}
}
// Ping performs check if all Hook-s are operational
// Ping executes checks if all Hook-s are operational
func (l *Logic) Ping(ctx context.Context) (err error) {
for _, p := range l.pingers {
if err = p.Ping(ctx); err != nil {
middleware/middleware.go
+26 -53
View File
@@ -1,92 +1,65 @@
// Package middleware implements the TrackerLogic interface by executing
// Package middleware implements the Logic interface by executing
// a series of middleware hooks.
package middleware
import (
"errors"
"fmt"
"sync"
"github.com/sot-tech/mochi/pkg/conf"
"github.com/sot-tech/mochi/pkg/log"
"github.com/sot-tech/mochi/storage"
)
var (
driversM sync.RWMutex
drivers = make(map[string]Builder)
// ErrBuilderDoesNotExist is the error returned by NewMiddleware when a
// middleware driver with that name does not exist.
ErrBuilderDoesNotExist = errors.New("middleware builder with that name does not exist")
logger = log.NewLogger("middleware")
buildersMU sync.RWMutex
builders = make(map[string]Builder)
)
// Builder is the interface used to initialize a new type of middleware.
//
// The `options` parameter is map of parameters that should be unmarshalled into
// the hook's custom configuration.
type Builder func(options conf.MapConfig, storage storage.PeerStorage) (Hook, error)
// Builder is the function used to initialize a new Hook
// with provided configuration.
type Builder func(conf.MapConfig, storage.PeerStorage) (Hook, error)
// RegisterBuilder makes a Builder available by the provided name.
//
// If called twice with the same name, the name is blank, or if the provided
// Builder is nil, this function panics.
func RegisterBuilder(name string, d Builder) {
func RegisterBuilder(name string, b Builder) {
if name == "" {
panic("middleware: could not register a Builder with an empty name")
panic("middleware: could not register Builder with an empty name")
}
if d == nil {
if b == nil {
panic("middleware: could not register a nil Builder")
}
driversM.Lock()
defer driversM.Unlock()
buildersMU.Lock()
defer buildersMU.Unlock()
if _, dup := drivers[name]; dup {
if _, dup := builders[name]; dup {
panic("middleware: RegisterBuilder called twice for " + name)
}
drivers[name] = d
}
// NewHook attempts to initialize a new middleware instance from the
// list of registered Builders.
//
// If a driver does not exist, returns ErrBuilderDoesNotExist.
func NewHook(name string, options conf.MapConfig, storage storage.PeerStorage) (Hook, error) {
driversM.RLock()
defer driversM.RUnlock()
var newHook Builder
newHook, ok := drivers[name]
if !ok {
return nil, ErrBuilderDoesNotExist
}
return newHook(options, storage)
}
// Config is the generic configuration format used for all registered Hooks.
type Config struct {
Name string
Options conf.MapConfig
builders[name] = b
}
// NewHooks is a utility function for initializing Hooks in bulk.
// each element of configs must contain pairs `name` - string and `options` - map[string]any
func NewHooks(configs []conf.MapConfig, storage storage.PeerStorage) (hooks []Hook, err error) {
for _, cfg := range configs {
var c Config
if err = cfg.Unmarshal(&c); err != nil {
func NewHooks(configs []conf.NamedMapConfig, storage storage.PeerStorage) (hooks []Hook, err error) {
buildersMU.RLock()
defer buildersMU.RUnlock()
for _, c := range configs {
logger.Debug().Object("hook", c).Msg("starting hook")
newHook, ok := builders[c.Name]
if !ok {
err = fmt.Errorf("hook with name '%s' does not exists", c.Name)
break
}
var h Hook
h, err = NewHook(c.Name, c.Options, storage)
if err != nil {
if h, err = newHook(c.Config, storage); err != nil {
break
}
hooks = append(hooks, h)
logger.Info().Str("name", c.Name).Msg("hook started")
}
return
middleware/torrentapproval/container/directory/directory.go
+2 -5
View File
@@ -20,13 +20,10 @@ import (
"github.com/sot-tech/mochi/storage"
)
// Name of this container for registry
const Name = "directory"
var logger = log.NewLogger("torrent approval directory")
var logger = log.NewLogger("middleware/torrent approval/directory")
func init() {
container.Register(Name, build)
container.Register("directory", build)
}
// Config - implementation of directory container configuration.
middleware/torrentapproval/container/list/list.go
+2 -5
View File
@@ -12,13 +12,10 @@ import (
"github.com/sot-tech/mochi/storage"
)
// Name of this container for registry.
const Name = "list"
var logger = log.NewLogger("torrent approval list")
var logger = log.NewLogger("middleware/torrent approval/list")
func init() {
container.Register(Name, build)
container.Register("list", build)
}
// Config - implementation of list container configuration.
middleware/torrentapproval/torrentapproval.go
+4 -4
View File
@@ -38,18 +38,18 @@ type baseConfig struct {
Configuration conf.MapConfig
}
func build(options conf.MapConfig, st storage.PeerStorage) (h middleware.Hook, err error) {
func build(config conf.MapConfig, st storage.PeerStorage) (h middleware.Hook, err error) {
var cfg baseConfig
if err = options.Unmarshal(&cfg); err != nil {
if err = config.Unmarshal(&cfg); err != nil {
return nil, fmt.Errorf("middleware %s: %w", Name, err)
}
if len(cfg.Source) == 0 {
return nil, fmt.Errorf("invalid options for middleware %s: source not provided", Name)
return nil, fmt.Errorf("invalid config for middleware %s: source not provided", Name)
}
if cfg.Configuration == nil {
return nil, fmt.Errorf("invalid options for middleware %s: options not provided", Name)
return nil, fmt.Errorf("invalid config for middleware %s: config not provided", Name)
}
var ds storage.DataStorage = st
middleware/varinterval/varinterval.go
+2 -2
View File
@@ -23,10 +23,10 @@ func init() {
middleware.RegisterBuilder(Name, build)
}
func build(options conf.MapConfig, _ storage.PeerStorage) (h middleware.Hook, err error) {
func build(config conf.MapConfig, _ storage.PeerStorage) (h middleware.Hook, err error) {
var cfg Config
if err = options.Unmarshal(&cfg); err != nil {
if err = config.Unmarshal(&cfg); err != nil {
err = fmt.Errorf("middleware %s: %w", Name, err)
} else {
if err := checkConfig(cfg); err == nil {
pkg/bytepool/bufferpool.go
+27
View File
@@ -0,0 +1,27 @@
package bytepool
import (
"bytes"
"sync"
)
// ByteBufferPool is a cached pool of reusable byte buffers.
type ByteBufferPool struct {
sync.Pool
}
// NewBufferPool allocates a new ByteBufferPool.
func NewBufferPool() *ByteBufferPool {
return &ByteBufferPool{sync.Pool{New: func() any { return new(bytes.Buffer) }}}
}
// Get returns a bytes.Buffer from the pool.
func (bbp *ByteBufferPool) Get() *bytes.Buffer {
return bbp.Pool.Get().(*bytes.Buffer)
}
// Put returns bytes.Buffer to the pool.
func (bbp *ByteBufferPool) Put(b *bytes.Buffer) {
b.Reset()
bbp.Pool.Put(b)
}
frontend/udp/bytepool/bytepool.go → pkg/bytepool/bytepool.go
+2 -2
View File
@@ -8,8 +8,8 @@ type BytePool struct {
sync.Pool
}
// New allocates a new BytePool with slices of equal length and capacity.
func New(length int) *BytePool {
// NewBytePool allocates a new BytePool with slices of equal length and capacity.
func NewBytePool(length int) *BytePool {
var bp BytePool
bp.Pool.New = func() any {
// This avoids allocations for the slice metadata, see:
pkg/conf/decoder.go → pkg/conf/config.go
+11
View File
@@ -51,3 +51,14 @@ func (m MapConfig) Unmarshal(into any) (err error) {
}
return
}
// NamedMapConfig encapsulates MapConfig with string Name
type NamedMapConfig struct {
Name string
Config MapConfig
}
// MarshalZerologObject writes Name and Config into zerolog event
func (nm NamedMapConfig) MarshalZerologObject(e *zerolog.Event) {
e.Str("name", nm.Name).Dict("config", zerolog.Dict().EmbedObject(nm.Config))
}
storage/keydb/storage.go
+21 -29
View File
@@ -15,7 +15,6 @@ import (
"time"
"github.com/go-redis/redis/v8"
"github.com/rs/zerolog"
"github.com/sot-tech/mochi/bittorrent"
"github.com/sot-tech/mochi/pkg/conf"
@@ -25,14 +24,10 @@ import (
r "github.com/sot-tech/mochi/storage/redis"
)
// Name is name of this storage
const (
Name = "keydb"
expireMemberCmd = "EXPIREMEMBER"
)
const expireMemberCmd = "EXPIREMEMBER"
var (
logger = log.NewLogger(Name)
logger = log.NewLogger("storage/keydb")
// errNotKeyDB returned from initializer if connected does not support KeyDB
// specific command (EXPIREMEMBER)
errNotKeyDB = errors.New("provided instance seems not KeyDB")
@@ -40,7 +35,7 @@ var (
func init() {
// Register the storage driver.
storage.RegisterBuilder(Name, builder)
storage.RegisterDriver("keydb", builder)
}
func builder(icfg conf.MapConfig) (storage.PeerStorage, error) {
@@ -98,16 +93,11 @@ type store struct {
peerTTL uint
}
// MarshalZerologObject writes configuration into zerolog event
func (s store) MarshalZerologObject(e *zerolog.Event) {
e.Str("type", Name).Object("config", s.Config)
}
func (s store) setPeerTTL(infoHashKey, peerID string) error {
func (s *store) setPeerTTL(infoHashKey, peerID string) error {
return s.Process(context.TODO(), redis.NewCmd(context.TODO(), expireMemberCmd, infoHashKey, peerID, s.peerTTL))
}
func (s store) addPeer(infoHashKey, peerID string) (err error) {
func (s *store) addPeer(infoHashKey, peerID string) (err error) {
logger.Trace().
Str("infoHashKey", infoHashKey).
Str("peerID", peerID).
@@ -118,7 +108,7 @@ func (s store) addPeer(infoHashKey, peerID string) (err error) {
return
}
func (s store) delPeer(infoHashKey, peerID string) error {
func (s *store) delPeer(infoHashKey, peerID string) error {
logger.Trace().
Str("infoHashKey", infoHashKey).
Str("peerID", peerID).
@@ -132,23 +122,23 @@ func (s store) delPeer(infoHashKey, peerID string) error {
return err
}
func (s store) PutSeeder(ih bittorrent.InfoHash, peer bittorrent.Peer) error {
func (s *store) PutSeeder(ih bittorrent.InfoHash, peer bittorrent.Peer) error {
return s.addPeer(r.InfoHashKey(ih.RawString(), true, peer.Addr().Is6()), peer.RawString())
}
func (s store) DeleteSeeder(ih bittorrent.InfoHash, peer bittorrent.Peer) error {
func (s *store) DeleteSeeder(ih bittorrent.InfoHash, peer bittorrent.Peer) error {
return s.delPeer(r.InfoHashKey(ih.RawString(), true, peer.Addr().Is6()), peer.RawString())
}
func (s store) PutLeecher(ih bittorrent.InfoHash, peer bittorrent.Peer) error {
func (s *store) PutLeecher(ih bittorrent.InfoHash, peer bittorrent.Peer) error {
return s.addPeer(r.InfoHashKey(ih.RawString(), false, peer.Addr().Is6()), peer.RawString())
}
func (s store) DeleteLeecher(ih bittorrent.InfoHash, peer bittorrent.Peer) error {
func (s *store) DeleteLeecher(ih bittorrent.InfoHash, peer bittorrent.Peer) error {
return s.delPeer(r.InfoHashKey(ih.RawString(), false, peer.Addr().Is6()), peer.RawString())
}
func (s store) GraduateLeecher(ih bittorrent.InfoHash, peer bittorrent.Peer) (err error) {
func (s *store) GraduateLeecher(ih bittorrent.InfoHash, peer bittorrent.Peer) (err error) {
logger.Trace().
Stringer("infoHash", ih).
Object("peer", peer).
@@ -163,12 +153,15 @@ func (s store) GraduateLeecher(ih bittorrent.InfoHash, peer bittorrent.Peer) (er
} else {
err = s.addPeer(ihSeederKey, peerID)
}
if err == nil {
err = s.HIncrBy(context.TODO(), r.CountDownloadsKey, infoHash, 1).Err()
}
}
return err
}
// AnnouncePeers is the same function as redis.AnnouncePeers
func (s store) AnnouncePeers(ih bittorrent.InfoHash, forSeeder bool, numWant int, v6 bool) ([]bittorrent.Peer, error) {
func (s *store) AnnouncePeers(ih bittorrent.InfoHash, forSeeder bool, numWant int, v6 bool) ([]bittorrent.Peer, error) {
logger.Trace().
Stringer("infoHash", ih).
Bool("forSeeder", forSeeder).
@@ -182,25 +175,24 @@ func (s store) AnnouncePeers(ih bittorrent.InfoHash, forSeeder bool, numWant int
}
// ScrapeSwarm is the same function as redis.ScrapeSwarm except `SCard` call instead of `HLen`
func (s store) ScrapeSwarm(ih bittorrent.InfoHash) (leechers uint32, seeders uint32, snatched uint32) {
func (s *store) ScrapeSwarm(ih bittorrent.InfoHash) (uint32, uint32, uint32) {
logger.Trace().
Stringer("infoHash", ih).
Msg("scrape swarm")
leechers, seeders = s.CountPeers(ih, s.SCard)
return
return s.ScrapeIH(ih, s.SCard)
}
func (store) GCAware() bool {
func (*store) GCAware() bool {
return false
}
func (store) ScheduleGC(_, _ time.Duration) {}
func (*store) ScheduleGC(_, _ time.Duration) {}
func (store) StatisticsAware() bool {
func (*store) StatisticsAware() bool {
return false
}
func (store) ScheduleStatisticsCollection(_ time.Duration) {}
func (*store) ScheduleStatisticsCollection(_ time.Duration) {}
func (s *store) Stop() stop.Result {
c := make(stop.Channel)
storage/memory/storage.go
+4 -20
View File
@@ -9,8 +9,6 @@ import (
"sync"
"time"
"github.com/rs/zerolog"
"github.com/sot-tech/mochi/bittorrent"
"github.com/sot-tech/mochi/pkg/conf"
"github.com/sot-tech/mochi/pkg/log"
@@ -21,17 +19,13 @@ import (
)
// Default config constants.
const (
// Name is the name by which this peer store is registered with Conf.
Name = "memory"
defaultShardCount = 1024
)
const defaultShardCount = 1024
var logger = log.NewLogger(Name)
var logger = log.NewLogger("storage/memory")
func init() {
// Register the storage driver.
storage.RegisterBuilder(Name, builder)
storage.RegisterDriver("memory", builder)
}
func builder(icfg conf.MapConfig) (storage.PeerStorage, error) {
@@ -47,11 +41,6 @@ type Config struct {
ShardCount int `cfg:"shard_count"`
}
// MarshalZerologObject writes configuration into zerolog event
func (cfg Config) MarshalZerologObject(e *zerolog.Event) {
e.Int("shardCount", cfg.ShardCount)
}
// Validate sanity checks values set in a config and returns a new config with
// default values replacing anything that is invalid.
//
@@ -110,11 +99,6 @@ type peerStore struct {
wg sync.WaitGroup
}
// MarshalZerologObject writes configuration into zerolog event
func (ps *peerStore) MarshalZerologObject(e *zerolog.Event) {
e.Str("type", Name).Object("config", ps.cfg)
}
var _ storage.PeerStorage = &peerStore{}
func (ps *peerStore) ScheduleGC(gcInterval, peerLifeTime time.Duration) {
@@ -406,7 +390,7 @@ func (ps *peerStore) AnnouncePeers(ih bittorrent.InfoHash, forSeeder bool, numWa
return
}
func (ps *peerStore) countPeers(ih bittorrent.InfoHash, v6 bool) (leechers uint32, seeders uint32) {
func (ps *peerStore) countPeers(ih bittorrent.InfoHash, v6 bool) (leechers, seeders uint32) {
shard := ps.shards[ps.shardIndex(ih, v6)]
shard.RLock()
defer shard.RUnlock()
storage/pg/storage.go
+106 -83
View File
@@ -15,7 +15,6 @@ import (
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/rs/zerolog"
"github.com/sot-tech/mochi/bittorrent"
"github.com/sot-tech/mochi/pkg/conf"
@@ -27,25 +26,33 @@ import (
)
const (
// Name is the name by which this peer store is registered with Conf.
Name = "pg"
defaultPingQuery = "SELECT 0"
errRequiredParameterNotSetMsg = "required parameter not provided: %s"
errRequiredColumnsNotFoundMsg = "one or more required columns not found in result set: %v"
errRollBackMsg = "error occurred while rolling back failed query: %v, failed query error: %v"
pCtx = "context"
pKey = "key"
pValue = "value"
pInfoHash = "info_hash"
pPeerID = "peer_id"
pAddress = "address"
pPort = "port"
pV6 = "is_v6"
pSeeder = "is_seeder"
pCreated = "created"
pCount = "count"
)
var (
logger = log.NewLogger(Name)
logger = log.NewLogger("storage/pg")
errConnectionStringNotProvided = errors.New("database connection string not provided")
)
func init() {
// Register the storage builder.
storage.RegisterBuilder(Name, builder)
storage.RegisterDriver("pg", builder)
}
func builder(icfg conf.MapConfig) (storage.PeerStorage, error) {
@@ -95,39 +102,23 @@ type dataQueryConf struct {
DelQuery string `cfg:"del_query"`
}
type downloadQueryConf struct {
GetQuery string `cfg:"get_query"`
IncrementQuery string `cfg:"inc_query"`
}
// Config holds the configuration of a redis PeerStorage.
type Config struct {
ConnectionString string `cfg:"connection_string"`
PingQuery string `cfg:"ping_query"`
Peer peerQueryConf
Announce announceQueryConf
Downloads downloadQueryConf
Data dataQueryConf
GCQuery string `cfg:"gc_query"`
InfoHashCountQuery string `cfg:"info_hash_count_query"`
}
// MarshalZerologObject writes configuration fields into zerolog event
func (cfg Config) MarshalZerologObject(e *zerolog.Event) {
e.Str("connectionString", "<hidden>").
Str("pingQuery", cfg.PingQuery).
Str("peer.addQuery", cfg.Peer.AddQuery).
Str("peer.delQuery", cfg.Peer.DelQuery).
Str("peer.graduateQuery", cfg.Peer.GraduateQuery).
Str("peer.countQuery", cfg.Peer.CountQuery).
Str("peer.countSeedersColumn", cfg.Peer.CountSeedersColumn).
Str("peer.countLeechersColumn", cfg.Peer.CountLeechersColumn).
Str("peer.byInfoHashClause", cfg.Peer.ByInfoHashClause).
Str("announce.query", cfg.Announce.Query).
Str("announce.peerIDColumn", cfg.Announce.PeerIDColumn).
Str("announce.addressColumn", cfg.Announce.AddressColumn).
Str("announce.portColumn", cfg.Announce.PortColumn).
Str("data.addQuery", cfg.Data.AddQuery).
Str("data.getQuery", cfg.Data.GetQuery).
Str("data.delQuery", cfg.Data.DelQuery).
Str("gcQuery", cfg.GCQuery).
Str("infoHashCountQuery", cfg.InfoHashCountQuery)
}
// Validate sanity checks values set in a config and returns a new config with
// default values replacing anything that is invalid.
//
@@ -228,18 +219,13 @@ type store struct {
closed chan any
}
func (s *store) Put(ctx string, values ...storage.Entry) (err error) {
func (s *store) txBatch(ctx context.Context, batch *pgx.Batch) (err error) {
var tx pgx.Tx
if tx, err = s.Begin(context.TODO()); err == nil {
for _, v := range values {
if _, err = tx.Exec(context.TODO(), s.Data.AddQuery, ctx, []byte(v.Key), v.Value); err != nil {
break
}
}
if err == nil {
err = tx.Commit(context.TODO())
if tx, err = s.Begin(ctx); err == nil {
if err = tx.SendBatch(context.TODO(), batch).Close(); err == nil {
err = tx.Commit(ctx)
} else {
if txErr := tx.Rollback(context.TODO()); txErr != nil {
if txErr := tx.Rollback(ctx); txErr != nil {
err = fmt.Errorf(errRollBackMsg, txErr, err)
}
}
@@ -247,9 +233,25 @@ func (s *store) Put(ctx string, values ...storage.Entry) (err error) {
return
}
func (s *store) Put(ctx string, values ...storage.Entry) (err error) {
switch len(values) {
case 0:
// ignore
case 1:
_, err = s.Exec(context.TODO(), s.Data.AddQuery, pgx.NamedArgs{pCtx: ctx, pKey: []byte(values[0].Key), pValue: values[0].Value})
default:
var batch pgx.Batch
for _, v := range values {
batch.Queue(s.Data.AddQuery, pgx.NamedArgs{pCtx: ctx, pKey: []byte(v.Key), pValue: v.Value})
}
err = s.txBatch(context.TODO(), &batch)
}
return
}
func (s *store) Contains(ctx string, key string) (contains bool, err error) {
var rows pgx.Rows
if rows, err = s.Query(context.TODO(), s.Data.GetQuery, ctx, []byte(key)); err == nil {
if rows, err = s.Query(context.TODO(), s.Data.GetQuery, pgx.NamedArgs{pCtx: ctx, pKey: []byte(key)}); err == nil {
defer rows.Close()
contains = rows.Next()
err = rows.Err()
@@ -258,28 +260,19 @@ func (s *store) Contains(ctx string, key string) (contains bool, err error) {
}
func (s *store) Load(ctx string, key string) (out []byte, err error) {
row := s.QueryRow(context.TODO(), s.Data.GetQuery, ctx, []byte(key))
if err = row.Scan(&out); errors.Is(err, pgx.ErrNoRows) {
if err = s.QueryRow(context.TODO(), s.Data.GetQuery, pgx.NamedArgs{pCtx: ctx, pKey: []byte(key)}).Scan(&out); errors.Is(err, pgx.ErrNoRows) {
err = nil
}
return
}
func (s *store) Delete(ctx string, keys ...string) (err error) {
var tx pgx.Tx
if tx, err = s.Begin(context.TODO()); err == nil {
for _, k := range keys {
if _, err = tx.Exec(context.TODO(), s.Data.DelQuery, ctx, []byte(k)); err != nil {
break
}
}
if err == nil {
err = tx.Commit(context.TODO())
} else {
if txErr := tx.Rollback(context.TODO()); txErr != nil {
err = fmt.Errorf(errRollBackMsg, txErr, err)
}
if len(keys) > 0 {
baKeys := make([][]byte, len(keys))
for i, k := range keys {
baKeys[i] = []byte(k)
}
_, err = s.Exec(context.TODO(), s.Data.DelQuery, pgx.NamedArgs{pCtx: ctx, pKey: baKeys})
}
return
}
@@ -304,7 +297,7 @@ func (s *store) ScheduleGC(gcInterval, peerLifeTime time.Duration) {
return
case <-t.C:
start := time.Now()
_, err := s.Exec(context.Background(), s.GCQuery, time.Now().Add(-peerLifeTime))
_, err := s.Exec(context.Background(), s.GCQuery, pgx.NamedArgs{pCreated: time.Now().Add(-peerLifeTime)})
duration := time.Since(start)
if err != nil {
logger.Error().Err(err).Msg("error occurred while GC")
@@ -335,10 +328,9 @@ func (s *store) ScheduleStatisticsCollection(reportInterval time.Duration) {
case <-t.C:
if metrics.Enabled() {
before := time.Now()
sc, lc := s.countPeers(bittorrent.NoneInfoHash)
sc, lc := s.countPeers(nil)
var hc int
row := s.QueryRow(context.Background(), s.InfoHashCountQuery)
if err := row.Scan(&hc); err != nil && !errors.Is(err, pgx.ErrNoRows) {
if err := s.QueryRow(context.Background(), s.InfoHashCountQuery).Scan(&hc); err != nil && !errors.Is(err, pgx.ErrNoRows) {
logger.Error().Err(err).Msg("error occurred while get info hash count")
}
@@ -352,27 +344,40 @@ func (s *store) ScheduleStatisticsCollection(reportInterval time.Duration) {
}()
}
func (s *store) putPeer(ih bittorrent.InfoHash, peer bittorrent.Peer, seeder bool) error {
func (s *store) putPeer(ih bittorrent.InfoHash, peer bittorrent.Peer, seeder bool) (err error) {
logger.Trace().
Stringer("infoHash", ih).
Object("peer", peer).
Bool("seeder", seeder).
Msg("put peer")
args := []any{[]byte(ih), peer.ID[:], net.IP(peer.Addr().AsSlice()), peer.Port(), seeder, peer.Addr().Is6()}
if s.GCAware() {
args = append(args, timecache.Now())
args := pgx.NamedArgs{
pInfoHash: []byte(ih),
pPeerID: peer.ID[:],
pAddress: net.IP(peer.Addr().AsSlice()),
pPort: peer.Port(),
pSeeder: seeder,
pV6: peer.Addr().Is6(),
}
_, err := s.Exec(context.TODO(), s.Peer.AddQuery, args...)
return err
if s.GCAware() {
args[pCreated] = timecache.Now()
}
_, err = s.Exec(context.TODO(), s.Peer.AddQuery, args)
return
}
func (s *store) delPeer(ih bittorrent.InfoHash, peer bittorrent.Peer, seeder bool) error {
func (s *store) delPeer(ih bittorrent.InfoHash, peer bittorrent.Peer, seeder bool) (err error) {
logger.Trace().
Stringer("infoHash", ih).
Object("peer", peer).
Msg("del peer")
_, err := s.Exec(context.TODO(), s.Peer.DelQuery, []byte(ih), peer.ID[:], net.IP(peer.Addr().AsSlice()), peer.Port(), seeder)
return err
_, err = s.Exec(context.TODO(), s.Peer.DelQuery, pgx.NamedArgs{
pInfoHash: []byte(ih),
pPeerID: peer.ID[:],
pAddress: net.IP(peer.Addr().AsSlice()),
pPort: peer.Port(),
pSeeder: seeder,
})
return
}
func (s *store) PutSeeder(ih bittorrent.InfoHash, peer bittorrent.Peer) error {
@@ -396,14 +401,26 @@ func (s *store) GraduateLeecher(ih bittorrent.InfoHash, peer bittorrent.Peer) er
Stringer("infoHash", ih).
Object("peer", peer).
Msg("graduate leecher")
_, err := s.Exec(context.TODO(), s.Peer.GraduateQuery, []byte(ih), peer.ID[:], net.IP(peer.Addr().AsSlice()), peer.Port())
return err
var batch pgx.Batch
ihb := []byte(ih)
batch.Queue(s.Peer.GraduateQuery, pgx.NamedArgs{
pInfoHash: ihb,
pPeerID: peer.ID[:],
pAddress: net.IP(peer.Addr().AsSlice()),
pPort: peer.Port(),
})
batch.Queue(s.Downloads.IncrementQuery, pgx.NamedArgs{pInfoHash: ihb})
return s.txBatch(context.TODO(), &batch)
}
func (s *store) getPeers(ih bittorrent.InfoHash, seeders bool, maxCount int, isV6 bool) (peers []bittorrent.Peer, err error) {
var rows pgx.Rows
// TODO: see https://github.com/jackc/pgx/issues/387#issuecomment-1107666716
if rows, err = s.Query(context.TODO(), s.Announce.Query, []byte(ih), seeders, isV6, maxCount); err == nil {
if rows, err = s.Query(context.TODO(), s.Announce.Query, pgx.NamedArgs{
pInfoHash: []byte(ih),
pSeeder: seeders,
pV6: isV6,
pCount: maxCount,
}); err == nil {
defer rows.Close()
idIndex, ipIndex, portIndex := -1, -1, -1
for i, field := range rows.FieldDescriptions() {
@@ -418,7 +435,11 @@ func (s *store) getPeers(ih bittorrent.InfoHash, seeders bool, maxCount int, isV
}
}
if idIndex < 0 || ipIndex < 0 || portIndex < 0 {
err = fmt.Errorf(errRequiredColumnsNotFoundMsg, []string{s.Announce.PeerIDColumn, s.Announce.AddressColumn, s.Announce.PortColumn})
err = fmt.Errorf(errRequiredColumnsNotFoundMsg, []string{
s.Announce.PeerIDColumn,
s.Announce.AddressColumn,
s.Announce.PortColumn,
})
return
}
var maxIndex int
@@ -492,13 +513,13 @@ func (s *store) AnnouncePeers(ih bittorrent.InfoHash, forSeeder bool, numWant in
return
}
func (s *store) countPeers(ih bittorrent.InfoHash) (seeders int, leechers int) {
func (s *store) countPeers(ih []byte) (seeders uint32, leechers uint32) {
var rows pgx.Rows
var err error
if ih == bittorrent.NoneInfoHash {
if len(ih) == 0 {
rows, err = s.Query(context.TODO(), s.Peer.CountQuery)
} else {
rows, err = s.Query(context.TODO(), s.Peer.CountQuery+" "+s.Peer.ByInfoHashClause, []byte(ih))
rows, err = s.Query(context.TODO(), s.Peer.CountQuery+" "+s.Peer.ByInfoHashClause, pgx.NamedArgs{pInfoHash: ih})
}
if err == nil {
defer rows.Close()
@@ -530,7 +551,7 @@ func (s *store) countPeers(ih bittorrent.InfoHash) (seeders int, leechers int) {
}
}
if err != nil {
logger.Error().Err(err).Stringer("infoHash", ih).Msg("unable to get peers count")
logger.Error().Err(err).Bytes("infoHash", ih).Msg("unable to get peers count")
}
return
}
@@ -539,8 +560,14 @@ func (s *store) ScrapeSwarm(ih bittorrent.InfoHash) (leechers uint32, seeders ui
logger.Trace().
Stringer("infoHash", ih).
Msg("scrape swarm")
sc, lc := s.countPeers(ih)
seeders, leechers = uint32(sc), uint32(lc)
ihb := []byte(ih)
seeders, leechers = s.countPeers(ihb)
if len(s.Downloads.GetQuery) > 0 {
if err := s.QueryRow(context.TODO(), s.Downloads.GetQuery, pgx.NamedArgs{pInfoHash: ihb}).Scan(&snatched); err != nil && !errors.Is(err, pgx.ErrNoRows) {
logger.Error().Stringer("infoHash", ih).Err(err).Msg("error occurred while get info downloads count")
}
}
return
}
@@ -565,7 +592,3 @@ func (s *store) Stop() stop.Result {
}()
return c.Result()
}
func (s *store) MarshalZerologObject(e *zerolog.Event) {
e.Str("type", Name).Object("config", s.Config)
}
storage/pg/storage_test.go
+20 -10
View File
@@ -26,6 +26,12 @@ CREATE TABLE mo_peers (
CREATE INDEX mo_peers_created_idx ON mo_peers(created);
CREATE INDEX mo_peers_announce_idx ON mo_peers(info_hash, is_seeder, is_v6);
DROP TABLE IF EXISTS mo_downloads;
CREATE TABLE mo_downloads (
info_hash bytea PRIMARY KEY NOT NULL,
downloads int NOT NULL DEFAULT 1
);
DROP TABLE IF EXISTS mo_kv;
CREATE TABLE mo_kv (
context varchar NOT NULL,
@@ -40,26 +46,30 @@ var cfg = Config{
ConnectionString: "host=127.0.0.1 database=test user=postgres pool_max_conns=50",
PingQuery: "SELECT 1",
Peer: peerQueryConf{
AddQuery: "INSERT INTO mo_peers VALUES($1, $2, $3, $4, $5, $6, $7) ON CONFLICT (info_hash, peer_id, address, port) DO UPDATE SET created = EXCLUDED.created, is_seeder = EXCLUDED.is_seeder",
DelQuery: "DELETE FROM mo_peers WHERE info_hash=$1 AND peer_id=$2 AND address=$3 AND port=$4 AND is_seeder=$5",
GraduateQuery: "UPDATE mo_peers SET is_seeder=TRUE WHERE info_hash=$1 AND peer_id=$2 AND address=$3 AND port=$4 AND NOT is_seeder",
AddQuery: "INSERT INTO mo_peers VALUES(@info_hash, @peer_id, @address, @port, @is_seeder, @is_v6, @created) ON CONFLICT (info_hash, peer_id, address, port) DO UPDATE SET created = EXCLUDED.created, is_seeder = EXCLUDED.is_seeder",
DelQuery: "DELETE FROM mo_peers WHERE info_hash=@info_hash AND peer_id=@peer_id AND address=@address AND port=@port AND is_seeder=@is_seeder",
GraduateQuery: "UPDATE mo_peers SET is_seeder=TRUE WHERE info_hash=@info_hash AND peer_id=peer_id AND address=@address AND port=@port AND NOT is_seeder",
CountQuery: "SELECT COUNT(1) FILTER (WHERE is_seeder) AS seeders, COUNT(1) FILTER (WHERE NOT is_seeder) AS leechers FROM mo_peers",
CountSeedersColumn: "seeders",
CountLeechersColumn: "leechers",
ByInfoHashClause: "WHERE info_hash = $1",
ByInfoHashClause: "WHERE info_hash = @info_hash",
},
Announce: announceQueryConf{
Query: "SELECT peer_id, address, port FROM mo_peers WHERE info_hash=$1 AND is_seeder=$2 AND is_v6=$3 LIMIT $4",
Query: "SELECT peer_id, address, port FROM mo_peers WHERE info_hash=@info_hash AND is_seeder=@is_seeder AND is_v6=@is_v6 LIMIT @count",
PeerIDColumn: "peer_id",
AddressColumn: "address",
PortColumn: "port",
},
Data: dataQueryConf{
AddQuery: "INSERT INTO mo_kv VALUES($1, $2, $3) ON CONFLICT (context, name) DO NOTHING",
GetQuery: "SELECT value FROM mo_kv WHERE context=$1 AND name=$2",
DelQuery: "DELETE FROM mo_kv WHERE context=$1 AND name=$2",
Downloads: downloadQueryConf{
GetQuery: "SELECT downloads FROM mo_downloads where info_hash=@info_hash",
IncrementQuery: "INSERT INTO mo_downloads VALUES(@info_hash) ON CONFLICT(info_hash) DO UPDATE SET downloads = mo_downloads.downloads + 1",
},
GCQuery: "DELETE FROM mo_peers WHERE created <= $1",
Data: dataQueryConf{
AddQuery: "INSERT INTO mo_kv VALUES(@context, @key, @value) ON CONFLICT (context, name) DO NOTHING",
GetQuery: "SELECT value FROM mo_kv WHERE context=@context AND name=@key",
DelQuery: "DELETE FROM mo_kv WHERE context=@context AND name = ANY(@key)",
},
GCQuery: "DELETE FROM mo_peers WHERE created <= @created",
InfoHashCountQuery: "SELECT COUNT(DISTINCT info_hash) as info_hashes FROM mo_peers",
}
storage/redis/storage.go
+22 -36
View File
@@ -1,6 +1,6 @@
// Package redis implements the storage interface.
// BitTorrent tracker keeping peer data in redis with hash.
// There two categories of hash:
// There three categories of hash:
//
// - CHI_{L,S}{4,6}_<HASH> (hash type)
// To save peers that hold the infohash, used for fast searching,
@@ -10,6 +10,9 @@
// To save all the infohashes, used for garbage collection,
// metrics aggregation and leecher graduation
//
// - CHI_D (hash type)
// To record the number of torrent downloads.
//
// Two keys are used to record the count of seeders and leechers.
//
// - CHI_C_S (key type)
@@ -28,7 +31,6 @@ import (
"time"
"github.com/go-redis/redis/v8"
"github.com/rs/zerolog"
"github.com/sot-tech/mochi/bittorrent"
"github.com/sot-tech/mochi/pkg/conf"
@@ -40,8 +42,6 @@ import (
)
const (
// Name is the name by which this peer store is registered with Conf.
Name = "redis"
// Default config constants.
defaultRedisAddress = "127.0.0.1:6379"
defaultReadTimeout = time.Second * 15
@@ -63,17 +63,19 @@ const (
CountSeederKey = "CHI_C_S"
// CountLeecherKey redis key for leecher count
CountLeecherKey = "CHI_C_L"
// CountDownloadsKey redis key for snatches (downloads) count
CountDownloadsKey = "CHI_D"
)
var (
logger = log.NewLogger(Name)
logger = log.NewLogger("storage/redis")
// errSentinelAndClusterChecked returned from initializer if both Config.Sentinel and Config.Cluster provided
errSentinelAndClusterChecked = errors.New("unable to use both cluster and sentinel mode")
)
func init() {
// Register the storage builder.
storage.RegisterBuilder(Name, builder)
storage.RegisterDriver("redis", builder)
}
func builder(icfg conf.MapConfig) (storage.PeerStorage, error) {
@@ -121,20 +123,6 @@ type Config struct {
ConnectTimeout time.Duration `cfg:"connect_timeout"`
}
// MarshalZerologObject writes configuration fields into zerolog event
func (cfg Config) MarshalZerologObject(e *zerolog.Event) {
e.Strs("addresses", cfg.Addresses).
Int("db", cfg.DB).
Int("poolSize", cfg.PoolSize).
Bool("sentinel", cfg.Sentinel).
Str("sentinelMaster", cfg.SentinelMaster).
Bool("cluster", cfg.Cluster).
Dur("readTimeout", cfg.ReadTimeout).
Dur("writeTimeout", cfg.WriteTimeout).
Dur("connectTimeout", cfg.ConnectTimeout).
Dur("peerLifetime", cfg.PeerLifetime)
}
// Validate sanity checks values set in a config and returns a new config with
// default values replacing anything that is invalid.
//
@@ -238,13 +226,7 @@ func (cfg Config) Connect() (con Connection, err error) {
_ = rs.Close()
rs = nil
}
cfg.Login, cfg.Password = "", ""
return Connection{rs, cfg}, err
}
// MarshalZerologObject writes configuration into zerolog event
func (ps *store) MarshalZerologObject(e *zerolog.Event) {
e.Str("type", Name).Object("config", ps.Config)
return Connection{rs}, err
}
func (ps *store) ScheduleGC(gcInterval, peerLifeTime time.Duration) {
@@ -301,7 +283,6 @@ func (ps *store) ScheduleStatisticsCollection(reportInterval time.Duration) {
// Connection is wrapper for redis.UniversalClient
type Connection struct {
redis.UniversalClient
Config
}
type store struct {
@@ -454,6 +435,9 @@ func (ps *store) GraduateLeecher(ih bittorrent.InfoHash, peer bittorrent.Peer) e
if err == nil {
err = tx.SAdd(context.TODO(), IHKey, ihSeederKey).Err()
}
if err == nil {
err = tx.HIncrBy(context.TODO(), CountDownloadsKey, infoHash, 1).Err()
}
return err
})
}
@@ -537,26 +521,28 @@ func (ps *Connection) countPeers(infoHashKey string, countFn getPeerCountFn) uin
return uint32(count)
}
// CountPeers calls provided countFn and returns seeders and leechers count for specified info hash
func (ps *Connection) CountPeers(ih bittorrent.InfoHash, countFn getPeerCountFn) (leechersCount, seedersCount uint32) {
// ScrapeIH calls provided countFn and returns seeders, leechers and downloads count for specified info hash
func (ps *Connection) ScrapeIH(ih bittorrent.InfoHash, countFn getPeerCountFn) (leechersCount, seedersCount, downloadsCount uint32) {
infoHash := ih.RawString()
leechersCount = ps.countPeers(InfoHashKey(infoHash, false, false), countFn) +
ps.countPeers(InfoHashKey(infoHash, false, true), countFn)
seedersCount = ps.countPeers(InfoHashKey(infoHash, true, false), countFn) +
ps.countPeers(InfoHashKey(infoHash, true, true), countFn)
d, err := ps.HGet(context.TODO(), CountDownloadsKey, infoHash).Uint64()
if err = AsNil(err); err != nil {
logger.Error().Err(err).Str("infoHash", infoHash).Msg("downloads count calculation failure")
}
downloadsCount = uint32(d)
return
}
func (ps *store) ScrapeSwarm(ih bittorrent.InfoHash) (leechers uint32, seeders uint32, snatched uint32) {
func (ps *store) ScrapeSwarm(ih bittorrent.InfoHash) (uint32, uint32, uint32) {
logger.Trace().
Stringer("infoHash", ih).
Msg("scrape swarm")
leechers, seeders = ps.CountPeers(ih, ps.HLen)
return
return ps.ScrapeIH(ih, ps.HLen)
}
const argNumErrorMsg = "ERR wrong number of arguments"
@@ -621,7 +607,7 @@ func (ps *Connection) Delete(ctx string, keys ...string) (err error) {
}
// Preservable - storage.DataStorage implementation
func (Connection) Preservable() bool {
func (*Connection) Preservable() bool {
return true
}
storage/storage.go
+37 -47
View File
@@ -3,12 +3,10 @@
package storage
import (
"errors"
"fmt"
"sync"
"time"
"github.com/rs/zerolog"
"github.com/sot-tech/mochi/bittorrent"
"github.com/sot-tech/mochi/pkg/conf"
"github.com/sot-tech/mochi/pkg/log"
@@ -25,9 +23,9 @@ const (
)
var (
logger = log.NewLogger("storage configurator")
driversM sync.RWMutex
drivers = make(map[string]Builder)
logger = log.NewLogger("storage")
driversMU sync.RWMutex
drivers = make(map[string]Driver)
)
// Config holds configuration for periodic execution tasks, which may or may not implement
@@ -83,19 +81,14 @@ type Entry struct {
Value []byte
}
// Builder is the function used to initialize a new type of PeerStorage.
type Builder func(cfg conf.MapConfig) (PeerStorage, error)
// Driver is the function used to initialize a new PeerStorage
// with provided configuration.
type Driver func(conf.MapConfig) (PeerStorage, error)
var (
// ErrResourceDoesNotExist is the error returned by all delete methods and the
// AnnouncePeers method of the PeerStorage interface if the requested resource
// does not exist.
ErrResourceDoesNotExist = bittorrent.ClientError("resource does not exist")
// ErrDriverDoesNotExist is the error returned by NewStorage when a peer
// store driver with that name does not exist.
ErrDriverDoesNotExist = errors.New("peer store driver with that name does not exist")
)
// ErrResourceDoesNotExist is the error returned by all delete methods and the
// AnnouncePeers method of the PeerStorage interface if the requested resource
// does not exist.
var ErrResourceDoesNotExist = bittorrent.ClientError("resource does not exist")
// DataStorage is the interface, used for implementing store for arbitrary data
type DataStorage interface {
@@ -216,86 +209,83 @@ type PeerStorage interface {
// Stopper is an interface that expects a Stop method to stop the PeerStorage.
// For more details see the documentation in the stop package.
stop.Stopper
// LogObjectMarshaler returns a loggable version of the data used to configure and
// operate a particular PeerStorage.
zerolog.LogObjectMarshaler
}
// RegisterBuilder makes a Builder available by the provided name.
// RegisterDriver makes a Driver available by the provided name.
//
// If called twice with the same name, the name is blank, or if the provided
// Driver is nil, this function panics.
func RegisterBuilder(name string, b Builder) {
func RegisterDriver(name string, d Driver) {
if name == "" {
panic("storage: could not register a Builder with an empty name")
panic("storage: could not register a Driver with an empty name")
}
if b == nil {
panic("storage: could not register a nil Builder")
if d == nil {
panic("storage: could not register a nil Driver")
}
driversM.Lock()
defer driversM.Unlock()
driversMU.Lock()
defer driversMU.Unlock()
if _, dup := drivers[name]; dup {
panic("storage: RegisterBuilder called twice for " + name)
panic("storage: RegisterDriver called twice for " + name)
}
drivers[name] = b
drivers[name] = d
}
// NewStorage attempts to initialize a new PeerStorage instance from
// the list of registered Drivers.
//
// If a builder does not exist, returns ErrDriverDoesNotExist.
func NewStorage(name string, cfg conf.MapConfig) (ps PeerStorage, err error) {
driversM.RLock()
defer driversM.RUnlock()
// the list of registered drivers.
func NewStorage(cfg conf.NamedMapConfig) (ps PeerStorage, err error) {
driversMU.RLock()
defer driversMU.RUnlock()
logger.Debug().Object("cfg", cfg).Msg("staring storage")
var b Builder
b, ok := drivers[name]
var b Driver
b, ok := drivers[cfg.Name]
if !ok {
return nil, ErrDriverDoesNotExist
return nil, fmt.Errorf("storage with name '%s' does not exists", cfg.Name)
}
c := new(Config)
if err = cfg.Unmarshal(c); err != nil {
if err = cfg.Config.Unmarshal(c); err != nil {
return
}
if ps, err = b(cfg); err != nil {
if ps, err = b(cfg.Config); err != nil {
return
}
if gc := ps.GCAware(); gc {
gcInterval, peerTTL := c.sanitizeGCConfig()
logger.Info().
Str("type", name).
Str("name", cfg.Name).
Dur("gcInterval", gcInterval).
Dur("peerTTL", peerTTL).
Msg("scheduling GC")
ps.ScheduleGC(gcInterval, peerTTL)
} else {
logger.Debug().
Str("type", name).
Str("name", cfg.Name).
Msg("storage does not support GC")
}
if st := ps.StatisticsAware(); st {
if statInterval := c.sanitizeStatisticsConfig(); statInterval > 0 {
logger.Info().
Str("type", name).
Str("name", cfg.Name).
Dur("statInterval", statInterval).
Msg("scheduling statistics collection")
ps.ScheduleStatisticsCollection(statInterval)
} else {
logger.Info().Str("type", name).Msg("statistics collection disabled because of zero reporting interval")
logger.Info().Str("name", cfg.Name).Msg("statistics collection disabled because of zero reporting interval")
}
} else {
logger.Debug().
Str("type", name).
Str("name", cfg.Name).
Msg("storage does not support statistics collection")
}
logger.Info().Str("name", cfg.Name).Msg("storage started")
return
}