diff --git a/LICENSE b/LICENSE index 494d9b7..4f8d4ff 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ -Chihaya is released under a BSD 2-Clause license, reproduced below. +Chihaya/MoChi is released under a BSD 2-Clause license, reproduced below. -Copyright (c) 2015, The Chihaya Authors +Copyright (c) 2015/2021, The Chihaya Authors and SOT-TECH All rights reserved. Redistribution and use in source and binary forms, with or without modification, @@ -21,8 +21,4 @@ ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -Modified Chihaya (MoChi) licensed under the same licence as original Chihaya by SOT-TECH. -Copyright (c) 2021, SOT-TECH -Some rights reserved. \ No newline at end of file +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/README.md b/README.md index db7bdff..e21e4d8 100644 --- a/README.md +++ b/README.md @@ -10,16 +10,23 @@ Modified version of [Chihaya](https://github.com/chihaya/chihaya), an open sourc ## Differences from the original project -* Support BittorrentV2 hashes (SHA-256 and _hybrid_ +* Supports BittorrentV2 hashes (SHA-256 and _hybrid_ SHA-256-to-160 [BEP52](https://www.bittorrent.org/beps/bep_0052.html), tested with qBittorrent); -* Support storage in middleware modules to persist useful data; -* Metrics can be turned off (not enabled till it really needed). +* Supports storage in middleware modules to persist useful data; +* Supports [KeyDB](https://keydb.dev) storage; +* Metrics can be turned off (not enabled till it really needed); +* Allows mixed peers: IPv4 requesters can fetch IPv6 peers or vice versa; +* Contains some internal improvements. + +_Note: From time to time MoChi fetch modifications from Chihaya but is not +fully compatible with original project (mainly in Redis storage structure), +so it cannot be mixed with Chihaya (i.e. it is impossible create MoChi-Chihaya cluster)._ ## Main goal The main goal of made modifications is to create semi-private tracker like [Hefur](https://github.com/sot-tech/hefur) but with cluster support (allowed torrents limited by pre-existent `list` middleware and another `directory` middleware -to limit registered torrents). +to [limit registered torrents](docs/middleware/torrent_approval.md)) and to maximize torrent swarm by providing maximum peers as possible (IPv4+IPv6). ## Notice diff --git a/bittorrent/bittorrent.go b/bittorrent/bittorrent.go index 4da9a25..50d2ce4 100644 --- a/bittorrent/bittorrent.go +++ b/bittorrent/bittorrent.go @@ -180,7 +180,9 @@ func (r AnnounceResponse) LogFields() log.Fields { // ScrapeRequest represents the parsed parameters from a scrape request. type ScrapeRequest struct { - Peer + // netip.Addr not used in internal logic, + // but MAY be used in middleware (per-ip block etc.) + netip.Addr InfoHashes []InfoHash Params Params } @@ -188,7 +190,7 @@ type ScrapeRequest struct { // LogFields renders the current response as a set of log fields. func (r ScrapeRequest) LogFields() log.Fields { return log.Fields{ - "peer": r.Peer, + "ip": r.Addr, "infoHashes": r.InfoHashes, "params": r.Params, } @@ -264,7 +266,7 @@ func (p Peer) String() string { // RawString generates concatenation of PeerID, net port and IP-address func (p Peer) RawString() string { - ip := p.Addr().Unmap() + ip := p.Addr() b := make([]byte, PeerIDLen+2+(ip.BitLen()/8)) copy(b[:PeerIDLen], p.ID[:]) binary.BigEndian.PutUint16(b[PeerIDLen:PeerIDLen+2], p.Port()) diff --git a/bittorrent/sanitize.go b/bittorrent/sanitize.go index c6bb42d..63ed49b 100644 --- a/bittorrent/sanitize.go +++ b/bittorrent/sanitize.go @@ -33,6 +33,7 @@ func SanitizeAnnounce(r *AnnounceRequest, maxNumWant, defaultNumWant uint32) err } log.Debug("sanitized announce", r, log.Fields{ + "ipPort": r.AddrPort, "maxNumWant": maxNumWant, "defaultNumWant": defaultNumWant, }) @@ -46,12 +47,13 @@ func SanitizeScrape(r *ScrapeRequest, maxScrapeInfoHashes uint32) error { r.InfoHashes = r.InfoHashes[:maxScrapeInfoHashes] } - r.AddrPort = netip.AddrPortFrom(r.Addr(), r.Port()) - if !r.Addr().IsValid() || r.Addr().IsUnspecified() { + r.Addr = r.Addr.Unmap() + if !r.Addr.IsValid() || r.Addr.IsUnspecified() { return ErrInvalidIP } log.Debug("sanitized scrape", r, log.Fields{ + "ip": r.Addr, "maxScrapeInfoHashes": maxScrapeInfoHashes, }) return nil diff --git a/docs/middleware/torrent_approval.md b/docs/middleware/torrent_approval.md index e3db3a1..488b172 100644 --- a/docs/middleware/torrent_approval.md +++ b/docs/middleware/torrent_approval.md @@ -21,9 +21,9 @@ If mode is **black list** (`invert` set to `true`), tracker will allow all hashe There are two sources of hashes: `list` and `directory`. -`list` is the static set of hashes, specified in configuration file. +* `list` is the static set of hashes, specified in configuration file. -`directory` will watch for `*.torrent` files in specified path and +* `directory` will watch for `*.torrent` files in specified path and append/delete records from storage. This source will parse all existing files at start and then watch for new files to add, or for delete events to remove hash from storage. diff --git a/docs/storage/keydb.md b/docs/storage/keydb.md new file mode 100644 index 0000000..4383f22 --- /dev/null +++ b/docs/storage/keydb.md @@ -0,0 +1,46 @@ +# KeyDB Storage + +This storage mainly the same as Redis and uses some of [redis](redis.md) store logic +with next exceptions: + +* peers stored in [sets](https://redis.io/docs/manual/data-types/#sets) +instead of [hashes](https://redis.io/docs/manual/data-types/#hashes); + +* keys such as `CHI_I`, `CHI_S_C` and `CHI_L_C` not used (at all); + +* peer TTL relies on KeyDB's [EXPIREMEMBER](https://docs.keydb.dev/docs/commands/#expiremember) +command, so MoChi does not need to periodically check peer expiration; + +* storage does not execute periodical statistics collection (peer/lecher/info hash count) +because: + * manual calculation (INC/DEC peers count) is not usable + * manual scan of all keys is quite expensive operation. + +## Use Case + +KeyDB is fork of Redis, which allows to create active-active cluster and set `set` member expiration, +so this type of backend can be used to create fully symmetric cluster of tracker nodes with minimum +overkill to garbage collection. + +## Configuration + +Configuration options are the same as [redis's](redis.md#Configuration), **BUT**: + +* `name` should be set to `keydb` instead of `redis`; + +* `gc_interval` and `prometheus_reporting_interval` don't have any sense. + +```yaml +mochi: + storage: + name: keydb + config: + ... +``` + +## Implementation + +KeyDB storage uses same key names as `redis` (`CHI_S4_`, `CHI_L6_`...) to store peers, +but it is **impossible** to switch between storage providers without deleting these keys. +You **can** use `redis` storage type with KeyDB instance (KeyDB supports all Redis commands), +but **not** `keydb` storage type with Redis instance. \ No newline at end of file diff --git a/frontend/http/frontend.go b/frontend/http/frontend.go index 1161de5..69df3bc 100644 --- a/frontend/http/frontend.go +++ b/frontend/http/frontend.go @@ -6,7 +6,6 @@ import ( "context" "crypto/tls" "errors" - "net" "net/http" "net/netip" "time" @@ -290,9 +289,7 @@ func (f *Frontend) announceRoute(w http.ResponseWriter, r *http.Request, ps http if f.EnableRequestTiming && metrics.Enabled() { start = time.Now() defer func() { - if f.EnableRequestTiming && metrics.Enabled() { - recordResponseDuration("announce", addr, err, time.Since(start)) - } + recordResponseDuration("announce", addr, err, time.Since(start)) }() } @@ -328,9 +325,7 @@ func (f *Frontend) scrapeRoute(w http.ResponseWriter, r *http.Request, ps httpro if f.EnableRequestTiming && metrics.Enabled() { start = time.Now() defer func() { - if f.EnableRequestTiming && metrics.Enabled() { - recordResponseDuration("scrape", addr, err, time.Since(start)) - } + recordResponseDuration("scrape", addr, err, time.Since(start)) }() } @@ -339,20 +334,7 @@ func (f *Frontend) scrapeRoute(w http.ResponseWriter, r *http.Request, ps httpro WriteError(w, err) return } - - host, _, err := net.SplitHostPort(r.RemoteAddr) - if err != nil { - log.Error("http: unable to determine remote address for scrape", log.Err(err)) - WriteError(w, err) - return - } - - addr, err = netip.ParseAddr(host) - if err != nil || addr.IsUnspecified() { - log.Error("http: invalid IP: neither v4 nor v6", log.Fields{"remoteAddr": r.RemoteAddr}) - WriteError(w, bittorrent.ErrInvalidIP) - return - } + addr = req.Addr ctx := injectRouteParamsToContext(context.Background(), ps) ctx, resp, err := f.logic.HandleScrape(ctx, req) diff --git a/frontend/http/parser.go b/frontend/http/parser.go index 43fbb5a..fd8b1ba 100644 --- a/frontend/http/parser.go +++ b/frontend/http/parser.go @@ -115,10 +115,7 @@ func ParseAnnounce(r *http.Request, opts ParseOptions) (*bittorrent.AnnounceRequ } // Parse the IP address where the client is listening. - ip, spoofed, err := requestedIP(r, qp, opts) - if err != nil { - return nil, bittorrent.ErrInvalidIP - } + ip, spoofed := requestedIP(r, qp, opts) request.Peer.AddrPort = netip.AddrPortFrom(ip, uint16(port)) request.IPProvided = spoofed @@ -141,9 +138,12 @@ func ParseScrape(r *http.Request, opts ParseOptions) (*bittorrent.ScrapeRequest, return nil, errNoInfoHash } + ip, _ := requestedIP(r, qp, opts) + request := &bittorrent.ScrapeRequest{ InfoHashes: infoHashes, Params: qp, + Addr: ip, } if err := bittorrent.SanitizeScrape(request, opts.MaxScrapeInfoHashes); err != nil { @@ -154,29 +154,22 @@ func ParseScrape(r *http.Request, opts ParseOptions) (*bittorrent.ScrapeRequest, } // requestedIP determines the IP address for a BitTorrent client request. -func requestedIP(r *http.Request, p bittorrent.Params, opts ParseOptions) (netip.Addr, bool, error) { +func requestedIP(r *http.Request, p bittorrent.Params, opts ParseOptions) (ip netip.Addr, spoofed bool) { if opts.AllowIPSpoofing { - if ipstr, ok := p.String("ip"); ok { - addr, err := netip.ParseAddr(ipstr) - return addr, true, err - } - - if ipstr, ok := p.String("ipv4"); ok { - addr, err := netip.ParseAddr(ipstr) - return addr, true, err - } - - if ipstr, ok := p.String("ipv6"); ok { - addr, err := netip.ParseAddr(ipstr) - return addr, true, err + for _, f := range []string{"ip", "ipv4", "ipv6"} { + if ipStr, ok := p.String(f); ok { + spoofed = true + ip, _ = netip.ParseAddr(ipStr) + return + } } } - if ipstr := r.Header.Get(opts.RealIPHeader); ipstr != "" && opts.RealIPHeader != "" { - addr, err := netip.ParseAddr(ipstr) - return addr, false, err + if ipStr := r.Header.Get(opts.RealIPHeader); ipStr != "" && opts.RealIPHeader != "" { + ip, _ = netip.ParseAddr(ipStr) } - addrPort, err := netip.ParseAddrPort(r.RemoteAddr) - return addrPort.Addr(), false, err + addrPort, _ := netip.ParseAddrPort(r.RemoteAddr) + ip = addrPort.Addr() + return } diff --git a/frontend/udp/parser.go b/frontend/udp/parser.go index 7e50582..a182bfc 100644 --- a/frontend/udp/parser.go +++ b/frontend/udp/parser.go @@ -243,7 +243,7 @@ func ParseScrape(r Request, opts ParseOptions) (*bittorrent.ScrapeRequest, error } if err == nil { // Sanitize the request. - request = &bittorrent.ScrapeRequest{InfoHashes: infoHashes} + request = &bittorrent.ScrapeRequest{InfoHashes: infoHashes, Addr: r.IP} err = bittorrent.SanitizeScrape(request, opts.MaxScrapeInfoHashes) } diff --git a/middleware/hooks.go b/middleware/hooks.go index 2946591..523aa2d 100644 --- a/middleware/hooks.go +++ b/middleware/hooks.go @@ -103,9 +103,9 @@ func (h *responseHook) HandleAnnounce(ctx context.Context, req *bittorrent.Annou } // Add the Scrape data to the response. - resp.Incomplete, resp.Complete, _ = h.store.ScrapeSwarm(req.InfoHash, req.Peer) + resp.Incomplete, resp.Complete, _ = h.store.ScrapeSwarm(req.InfoHash) if len(req.InfoHash) == bittorrent.InfoHashV2Len { - incomplete, complete, _ := h.store.ScrapeSwarm(req.InfoHash.TruncateV1(), req.Peer) + incomplete, complete, _ := h.store.ScrapeSwarm(req.InfoHash.TruncateV1()) resp.Incomplete, resp.Complete = resp.Incomplete+incomplete, resp.Complete+complete } @@ -175,9 +175,9 @@ func (h *responseHook) HandleScrape(ctx context.Context, req *bittorrent.ScrapeR for _, infoHash := range req.InfoHashes { scr := bittorrent.Scrape{InfoHash: infoHash} - scr.Incomplete, scr.Complete, scr.Snatches = h.store.ScrapeSwarm(infoHash, req.Peer) + scr.Incomplete, scr.Complete, scr.Snatches = h.store.ScrapeSwarm(infoHash) if len(infoHash) == bittorrent.InfoHashV2Len { - leechers, seeders, snatched := h.store.ScrapeSwarm(infoHash.TruncateV1(), req.Peer) + leechers, seeders, snatched := h.store.ScrapeSwarm(infoHash.TruncateV1()) scr.Incomplete, scr.Complete, scr.Snatches = scr.Incomplete+leechers, scr.Complete+seeders, scr.Snatches+snatched } diff --git a/storage/keydb/storage.go b/storage/keydb/storage.go index 1db0d9b..f1506c5 100644 --- a/storage/keydb/storage.go +++ b/storage/keydb/storage.go @@ -166,10 +166,9 @@ func (s store) AnnouncePeers(ih bittorrent.InfoHash, seeder bool, numWant int, p } // ScrapeSwarm is the same function as redis.ScrapeSwarm except `SCard` call instead of `HLen` -func (s store) ScrapeSwarm(ih bittorrent.InfoHash, peer bittorrent.Peer) (leechers uint32, seeders uint32, snatched uint32) { +func (s store) ScrapeSwarm(ih bittorrent.InfoHash) (leechers uint32, seeders uint32, snatched uint32) { log.Debug("storage: KeyDB ScrapeSwarm", log.Fields{ "infoHash": ih, - "peer": peer, }) leechers, seeders = s.CountPeers(ih, s.SCard) return diff --git a/storage/memory/storage.go b/storage/memory/storage.go index 5a24e1b..d73a7b0 100644 --- a/storage/memory/storage.go +++ b/storage/memory/storage.go @@ -404,7 +404,7 @@ func (ps *peerStore) countPeers(ih bittorrent.InfoHash, v6 bool) (leechers uint3 return } -func (ps *peerStore) ScrapeSwarm(ih bittorrent.InfoHash, _ bittorrent.Peer) (leechers uint32, seeders uint32, _ uint32) { +func (ps *peerStore) ScrapeSwarm(ih bittorrent.InfoHash) (leechers uint32, seeders uint32, snatched uint32) { select { case <-ps.closed: panic("attempted to interact with stopped memory store") diff --git a/storage/redis/storage.go b/storage/redis/storage.go index a011084..3e36cdc 100644 --- a/storage/redis/storage.go +++ b/storage/redis/storage.go @@ -564,10 +564,9 @@ func (ps Connection) CountPeers(ih bittorrent.InfoHash, countFn getPeerCountFn) return } -func (ps *store) ScrapeSwarm(ih bittorrent.InfoHash, peer bittorrent.Peer) (leechers uint32, seeders uint32, _ uint32) { +func (ps *store) ScrapeSwarm(ih bittorrent.InfoHash) (leechers uint32, seeders uint32, snatched uint32) { log.Debug("storage: Redis ScrapeSwarm", log.Fields{ "infoHash": ih, - "peer": peer, }) leechers, seeders = ps.CountPeers(ih, ps.HLen) diff --git a/storage/storage.go b/storage/storage.go index 711af6c..50c97d8 100644 --- a/storage/storage.go +++ b/storage/storage.go @@ -201,7 +201,7 @@ type PeerStorage interface { // filling the Snatches field is optional. // // If the Swarm does not exist, an empty Scrape and no error is returned. - ScrapeSwarm(infoHash bittorrent.InfoHash, peer bittorrent.Peer) (leechers uint32, seeders uint32, snatched uint32) + ScrapeSwarm(infoHash bittorrent.InfoHash) (leechers uint32, seeders uint32, snatched uint32) // Stopper is an interface that expects a Stop method to stop the PeerStorage. // For more details see the documentation in the stop package. diff --git a/storage/test/storage_bench.go b/storage/test/storage_bench.go index 0bf26a5..b8020fd 100644 --- a/storage/test/storage_bench.go +++ b/storage/test/storage_bench.go @@ -448,7 +448,7 @@ func (bh *benchHolder) AnnounceSeeder1kInfoHash(b *testing.B) { // ScrapeSwarm can run in parallel. func (bh *benchHolder) ScrapeSwarm(b *testing.B) { bh.runBenchmark(b, true, putPeers, func(i int, ps storage.PeerStorage, bd *benchData) error { - ps.ScrapeSwarm(bd.infoHashes[0], bd.peers[0]) + ps.ScrapeSwarm(bd.infoHashes[0]) return nil }) } @@ -458,7 +458,7 @@ func (bh *benchHolder) ScrapeSwarm(b *testing.B) { // ScrapeSwarm1kInfoHash can run in parallel. func (bh *benchHolder) ScrapeSwarm1kInfoHash(b *testing.B) { bh.runBenchmark(b, true, putPeers, func(i int, ps storage.PeerStorage, bd *benchData) error { - ps.ScrapeSwarm(bd.infoHashes[i%ihCount], bd.peers[0]) + ps.ScrapeSwarm(bd.infoHashes[i%ihCount]) return nil }) } diff --git a/storage/test/storage_test_base.go b/storage/test/storage_test_base.go index 3a2a9bd..5f1ad59 100644 --- a/storage/test/storage_test_base.go +++ b/storage/test/storage_test_base.go @@ -68,7 +68,7 @@ func (th *testHolder) AnnouncePeers(t *testing.T) { func (th *testHolder) ScrapeSwarm(t *testing.T) { for _, c := range testData { - l, s, n := th.st.ScrapeSwarm(c.ih, c.peer) + l, s, n := th.st.ScrapeSwarm(c.ih) require.Equal(t, uint32(0), s) require.Equal(t, uint32(0), l) require.Equal(t, uint32(0), n) @@ -93,7 +93,7 @@ func (th *testHolder) LeecherPutAnnounceDeleteAnnounce(t *testing.T) { require.Nil(t, err) require.True(t, containsPeer(peers, c.peer)) - l, s, _ := th.st.ScrapeSwarm(c.ih, c.peer) + l, s, _ := th.st.ScrapeSwarm(c.ih) require.Equal(t, uint32(2), l) require.Equal(t, uint32(0), s) @@ -123,7 +123,7 @@ func (th *testHolder) SeederPutAnnounceDeleteAnnounce(t *testing.T) { require.Nil(t, err) require.True(t, containsPeer(peers, c.peer)) - l, s, _ := th.st.ScrapeSwarm(c.ih, c.peer) + l, s, _ := th.st.ScrapeSwarm(c.ih) require.Equal(t, uint32(1), l) require.Equal(t, uint32(1), s)