(untested) sanitize code

* remove peer argument from scrape swarm storage call

* replace Peer field with netip.Addr in ScrapeRequest

* add man for keydb storage

* update readme

LICENSE

@@ -1,6 +1,6 @@
-Chihaya is released under a BSD 2-Clause license, reproduced below.
+Chihaya/MoChi is released under a BSD 2-Clause license, reproduced below.
 
-Copyright (c) 2015, The Chihaya Authors
+Copyright (c) 2015/2021, The Chihaya Authors and SOT-TECH
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without modification,
@@ -21,8 +21,4 @@ ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
 ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-Modified Chihaya (MoChi) licensed under the same licence as original Chihaya by SOT-TECH.
-Copyright (c) 2021, SOT-TECH
-Some rights reserved.
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

README.md

@@ -10,16 +10,23 @@ Modified version of [Chihaya](https://github.com/chihaya/chihaya), an open sourc
 
 ## Differences from the original project
 
-* Support BittorrentV2 hashes (SHA-256 and _hybrid_
+* Supports BittorrentV2 hashes (SHA-256 and _hybrid_
   SHA-256-to-160 [BEP52](https://www.bittorrent.org/beps/bep_0052.html), tested with qBittorrent);
-* Support storage in middleware modules to persist useful data;
-* Metrics can be turned off (not enabled till it really needed).
+* Supports storage in middleware modules to persist useful data;
+* Supports [KeyDB](https://keydb.dev) storage;
+* Metrics can be turned off (not enabled till it really needed);
+* Allows mixed peers: IPv4 requesters can fetch IPv6 peers or vice versa;
+* Contains some internal improvements.
+
+_Note: From time to time MoChi fetch modifications from Chihaya but is not 
+fully compatible with original project (mainly in Redis storage structure),
+so it cannot be mixed with Chihaya (i.e. it is impossible create MoChi-Chihaya cluster)._
 
 ## Main goal
 
 The main goal of made modifications is to create semi-private tracker like [Hefur](https://github.com/sot-tech/hefur)
 but with cluster support (allowed torrents limited by pre-existent `list` middleware and another `directory` middleware
-to limit registered torrents).
+to [limit registered torrents](docs/middleware/torrent_approval.md)) and to maximize torrent swarm by providing maximum peers as possible (IPv4+IPv6).
 
 ## Notice
 

bittorrent/bittorrent.go

@@ -180,7 +180,9 @@ func (r AnnounceResponse) LogFields() log.Fields {
 
 // ScrapeRequest represents the parsed parameters from a scrape request.
 type ScrapeRequest struct {
-	Peer
+	// netip.Addr not used in internal logic,
+	// but MAY be used in middleware (per-ip block etc.)
+	netip.Addr
 	InfoHashes []InfoHash
 	Params     Params
 }
@@ -188,7 +190,7 @@ type ScrapeRequest struct {
 // LogFields renders the current response as a set of log fields.
 func (r ScrapeRequest) LogFields() log.Fields {
 	return log.Fields{
-		"peer":       r.Peer,
+		"ip":         r.Addr,
 		"infoHashes": r.InfoHashes,
 		"params":     r.Params,
 	}
@@ -264,7 +266,7 @@ func (p Peer) String() string {
 
 // RawString generates concatenation of PeerID, net port and IP-address
 func (p Peer) RawString() string {
-	ip := p.Addr().Unmap()
+	ip := p.Addr()
 	b := make([]byte, PeerIDLen+2+(ip.BitLen()/8))
 	copy(b[:PeerIDLen], p.ID[:])
 	binary.BigEndian.PutUint16(b[PeerIDLen:PeerIDLen+2], p.Port())

bittorrent/sanitize.go

@@ -33,6 +33,7 @@ func SanitizeAnnounce(r *AnnounceRequest, maxNumWant, defaultNumWant uint32) err
 	}
 
 	log.Debug("sanitized announce", r, log.Fields{
+		"ipPort":         r.AddrPort,
 		"maxNumWant":     maxNumWant,
 		"defaultNumWant": defaultNumWant,
 	})
@@ -46,12 +47,13 @@ func SanitizeScrape(r *ScrapeRequest, maxScrapeInfoHashes uint32) error {
 		r.InfoHashes = r.InfoHashes[:maxScrapeInfoHashes]
 	}
 
-	r.AddrPort = netip.AddrPortFrom(r.Addr(), r.Port())
-	if !r.Addr().IsValid() || r.Addr().IsUnspecified() {
+	r.Addr = r.Addr.Unmap()
+	if !r.Addr.IsValid() || r.Addr.IsUnspecified() {
 		return ErrInvalidIP
 	}
 
 	log.Debug("sanitized scrape", r, log.Fields{
+		"ip":                  r.Addr,
 		"maxScrapeInfoHashes": maxScrapeInfoHashes,
 	})
 	return nil

docs/middleware/torrent_approval.md

@@ -21,9 +21,9 @@ If mode is **black list** (`invert` set to `true`), tracker will allow all hashe
 
 There are two sources of hashes: `list` and `directory`.
 
-`list` is the static set of hashes, specified in configuration file.
+* `list` is the static set of hashes, specified in configuration file.
 
-`directory` will watch for `*.torrent` files in specified path and
+* `directory` will watch for `*.torrent` files in specified path and
 append/delete records from storage. This source will parse all existing
 files at start and then watch for new files to add, or for delete events
 to remove hash from storage.

docs/storage/keydb.md

@@ -0,0 +1,46 @@
+# KeyDB Storage
+
+This storage mainly the same as Redis and uses some of [redis](redis.md) store logic
+with next exceptions:
+
+* peers stored in [sets](https://redis.io/docs/manual/data-types/#sets) 
+instead of [hashes](https://redis.io/docs/manual/data-types/#hashes);
+
+* keys such as `CHI_I`, `CHI_S_C` and `CHI_L_C` not used (at all);
+
+* peer TTL relies on KeyDB's [EXPIREMEMBER](https://docs.keydb.dev/docs/commands/#expiremember) 
+command, so MoChi does not need to periodically check peer expiration;
+
+* storage does not execute periodical statistics collection (peer/lecher/info hash count)
+because:
+  * manual calculation (INC/DEC peers count) is not usable 
+  * manual scan of all keys is quite expensive operation.
+
+## Use Case
+
+KeyDB is fork of Redis, which allows to create active-active cluster and set `set` member expiration,
+so this type of backend can be used to create fully symmetric cluster of tracker nodes with minimum
+overkill to garbage collection.
+
+## Configuration
+
+Configuration options are the same as [redis's](redis.md#Configuration), **BUT**:
+
+* `name` should be set to `keydb` instead of `redis`;
+
+* `gc_interval` and `prometheus_reporting_interval` don't have any sense.
+
+```yaml
+mochi:
+  storage:
+    name: keydb
+    config:
+	...
+```
+
+## Implementation
+
+KeyDB storage uses same key names as `redis` (`CHI_S4_<HASH>`, `CHI_L6_<HASH>`...) to store peers,
+but it is **impossible** to switch between storage providers without deleting these keys.
+You **can** use `redis` storage type with KeyDB instance (KeyDB supports all Redis commands),
+but **not** `keydb` storage type with Redis instance.

frontend/http/frontend.go

@@ -6,7 +6,6 @@ import (
 	"context"
 	"crypto/tls"
 	"errors"
-	"net"
 	"net/http"
 	"net/netip"
 	"time"
@@ -290,9 +289,7 @@ func (f *Frontend) announceRoute(w http.ResponseWriter, r *http.Request, ps http
 	if f.EnableRequestTiming && metrics.Enabled() {
 		start = time.Now()
 		defer func() {
-			if f.EnableRequestTiming && metrics.Enabled() {
-				recordResponseDuration("announce", addr, err, time.Since(start))
-			}
+			recordResponseDuration("announce", addr, err, time.Since(start))
 		}()
 	}
 
@@ -328,9 +325,7 @@ func (f *Frontend) scrapeRoute(w http.ResponseWriter, r *http.Request, ps httpro
 	if f.EnableRequestTiming && metrics.Enabled() {
 		start = time.Now()
 		defer func() {
-			if f.EnableRequestTiming && metrics.Enabled() {
-				recordResponseDuration("scrape", addr, err, time.Since(start))
-			}
+			recordResponseDuration("scrape", addr, err, time.Since(start))
 		}()
 	}
 
@@ -339,20 +334,7 @@ func (f *Frontend) scrapeRoute(w http.ResponseWriter, r *http.Request, ps httpro
 		WriteError(w, err)
 		return
 	}
-
-	host, _, err := net.SplitHostPort(r.RemoteAddr)
-	if err != nil {
-		log.Error("http: unable to determine remote address for scrape", log.Err(err))
-		WriteError(w, err)
-		return
-	}
-
-	addr, err = netip.ParseAddr(host)
-	if err != nil || addr.IsUnspecified() {
-		log.Error("http: invalid IP: neither v4 nor v6", log.Fields{"remoteAddr": r.RemoteAddr})
-		WriteError(w, bittorrent.ErrInvalidIP)
-		return
-	}
+	addr = req.Addr
 
 	ctx := injectRouteParamsToContext(context.Background(), ps)
 	ctx, resp, err := f.logic.HandleScrape(ctx, req)

frontend/http/parser.go

@@ -115,10 +115,7 @@ func ParseAnnounce(r *http.Request, opts ParseOptions) (*bittorrent.AnnounceRequ
 	}
 
 	// Parse the IP address where the client is listening.
-	ip, spoofed, err := requestedIP(r, qp, opts)
-	if err != nil {
-		return nil, bittorrent.ErrInvalidIP
-	}
+	ip, spoofed := requestedIP(r, qp, opts)
 	request.Peer.AddrPort = netip.AddrPortFrom(ip, uint16(port))
 	request.IPProvided = spoofed
 
@@ -141,9 +138,12 @@ func ParseScrape(r *http.Request, opts ParseOptions) (*bittorrent.ScrapeRequest,
 		return nil, errNoInfoHash
 	}
 
+	ip, _ := requestedIP(r, qp, opts)
+
 	request := &bittorrent.ScrapeRequest{
 		InfoHashes: infoHashes,
 		Params:     qp,
+		Addr:       ip,
 	}
 
 	if err := bittorrent.SanitizeScrape(request, opts.MaxScrapeInfoHashes); err != nil {
@@ -154,29 +154,22 @@ func ParseScrape(r *http.Request, opts ParseOptions) (*bittorrent.ScrapeRequest,
 }
 
 // requestedIP determines the IP address for a BitTorrent client request.
-func requestedIP(r *http.Request, p bittorrent.Params, opts ParseOptions) (netip.Addr, bool, error) {
+func requestedIP(r *http.Request, p bittorrent.Params, opts ParseOptions) (ip netip.Addr, spoofed bool) {
 	if opts.AllowIPSpoofing {
-		if ipstr, ok := p.String("ip"); ok {
-			addr, err := netip.ParseAddr(ipstr)
-			return addr, true, err
-		}
-
-		if ipstr, ok := p.String("ipv4"); ok {
-			addr, err := netip.ParseAddr(ipstr)
-			return addr, true, err
-		}
-
-		if ipstr, ok := p.String("ipv6"); ok {
-			addr, err := netip.ParseAddr(ipstr)
-			return addr, true, err
+		for _, f := range []string{"ip", "ipv4", "ipv6"} {
+			if ipStr, ok := p.String(f); ok {
+				spoofed = true
+				ip, _ = netip.ParseAddr(ipStr)
+				return
+			}
 		}
 	}
 
-	if ipstr := r.Header.Get(opts.RealIPHeader); ipstr != "" && opts.RealIPHeader != "" {
-		addr, err := netip.ParseAddr(ipstr)
-		return addr, false, err
+	if ipStr := r.Header.Get(opts.RealIPHeader); ipStr != "" && opts.RealIPHeader != "" {
+		ip, _ = netip.ParseAddr(ipStr)
 	}
 
-	addrPort, err := netip.ParseAddrPort(r.RemoteAddr)
-	return addrPort.Addr(), false, err
+	addrPort, _ := netip.ParseAddrPort(r.RemoteAddr)
+	ip = addrPort.Addr()
+	return
 }

frontend/udp/parser.go

@@ -243,7 +243,7 @@ func ParseScrape(r Request, opts ParseOptions) (*bittorrent.ScrapeRequest, error
 	}
 	if err == nil {
 		// Sanitize the request.
-		request = &bittorrent.ScrapeRequest{InfoHashes: infoHashes}
+		request = &bittorrent.ScrapeRequest{InfoHashes: infoHashes, Addr: r.IP}
 		err = bittorrent.SanitizeScrape(request, opts.MaxScrapeInfoHashes)
 	}
 

middleware/hooks.go

@@ -103,9 +103,9 @@ func (h *responseHook) HandleAnnounce(ctx context.Context, req *bittorrent.Annou
 	}
 
 	// Add the Scrape data to the response.
-	resp.Incomplete, resp.Complete, _ = h.store.ScrapeSwarm(req.InfoHash, req.Peer)
+	resp.Incomplete, resp.Complete, _ = h.store.ScrapeSwarm(req.InfoHash)
 	if len(req.InfoHash) == bittorrent.InfoHashV2Len {
-		incomplete, complete, _ := h.store.ScrapeSwarm(req.InfoHash.TruncateV1(), req.Peer)
+		incomplete, complete, _ := h.store.ScrapeSwarm(req.InfoHash.TruncateV1())
 		resp.Incomplete, resp.Complete = resp.Incomplete+incomplete, resp.Complete+complete
 	}
 
@@ -175,9 +175,9 @@ func (h *responseHook) HandleScrape(ctx context.Context, req *bittorrent.ScrapeR
 
 	for _, infoHash := range req.InfoHashes {
 		scr := bittorrent.Scrape{InfoHash: infoHash}
-		scr.Incomplete, scr.Complete, scr.Snatches = h.store.ScrapeSwarm(infoHash, req.Peer)
+		scr.Incomplete, scr.Complete, scr.Snatches = h.store.ScrapeSwarm(infoHash)
 		if len(infoHash) == bittorrent.InfoHashV2Len {
-			leechers, seeders, snatched := h.store.ScrapeSwarm(infoHash.TruncateV1(), req.Peer)
+			leechers, seeders, snatched := h.store.ScrapeSwarm(infoHash.TruncateV1())
 			scr.Incomplete, scr.Complete, scr.Snatches = scr.Incomplete+leechers, scr.Complete+seeders, scr.Snatches+snatched
 		}
 

storage/keydb/storage.go

@@ -166,10 +166,9 @@ func (s store) AnnouncePeers(ih bittorrent.InfoHash, seeder bool, numWant int, p
 }
 
 // ScrapeSwarm is the same function as redis.ScrapeSwarm except `SCard` call instead of `HLen`
-func (s store) ScrapeSwarm(ih bittorrent.InfoHash, peer bittorrent.Peer) (leechers uint32, seeders uint32, snatched uint32) {
+func (s store) ScrapeSwarm(ih bittorrent.InfoHash) (leechers uint32, seeders uint32, snatched uint32) {
 	log.Debug("storage: KeyDB ScrapeSwarm", log.Fields{
 		"infoHash": ih,
-		"peer":     peer,
 	})
 	leechers, seeders = s.CountPeers(ih, s.SCard)
 	return

storage/memory/storage.go

@@ -404,7 +404,7 @@ func (ps *peerStore) countPeers(ih bittorrent.InfoHash, v6 bool) (leechers uint3
 	return
 }
 
-func (ps *peerStore) ScrapeSwarm(ih bittorrent.InfoHash, _ bittorrent.Peer) (leechers uint32, seeders uint32, _ uint32) {
+func (ps *peerStore) ScrapeSwarm(ih bittorrent.InfoHash) (leechers uint32, seeders uint32, snatched uint32) {
 	select {
 	case <-ps.closed:
 		panic("attempted to interact with stopped memory store")

storage/redis/storage.go

@@ -564,10 +564,9 @@ func (ps Connection) CountPeers(ih bittorrent.InfoHash, countFn getPeerCountFn)
 	return
 }
 
-func (ps *store) ScrapeSwarm(ih bittorrent.InfoHash, peer bittorrent.Peer) (leechers uint32, seeders uint32, _ uint32) {
+func (ps *store) ScrapeSwarm(ih bittorrent.InfoHash) (leechers uint32, seeders uint32, snatched uint32) {
 	log.Debug("storage: Redis ScrapeSwarm", log.Fields{
 		"infoHash": ih,
-		"peer":     peer,
 	})
 
 	leechers, seeders = ps.CountPeers(ih, ps.HLen)

storage/storage.go

@@ -201,7 +201,7 @@ type PeerStorage interface {
 	// filling the Snatches field is optional.
 	//
 	// If the Swarm does not exist, an empty Scrape and no error is returned.
-	ScrapeSwarm(infoHash bittorrent.InfoHash, peer bittorrent.Peer) (leechers uint32, seeders uint32, snatched uint32)
+	ScrapeSwarm(infoHash bittorrent.InfoHash) (leechers uint32, seeders uint32, snatched uint32)
 
 	// Stopper is an interface that expects a Stop method to stop the PeerStorage.
 	// For more details see the documentation in the stop package.

storage/test/storage_bench.go

@@ -448,7 +448,7 @@ func (bh *benchHolder) AnnounceSeeder1kInfoHash(b *testing.B) {
 // ScrapeSwarm can run in parallel.
 func (bh *benchHolder) ScrapeSwarm(b *testing.B) {
 	bh.runBenchmark(b, true, putPeers, func(i int, ps storage.PeerStorage, bd *benchData) error {
-		ps.ScrapeSwarm(bd.infoHashes[0], bd.peers[0])
+		ps.ScrapeSwarm(bd.infoHashes[0])
 		return nil
 	})
 }
@@ -458,7 +458,7 @@ func (bh *benchHolder) ScrapeSwarm(b *testing.B) {
 // ScrapeSwarm1kInfoHash can run in parallel.
 func (bh *benchHolder) ScrapeSwarm1kInfoHash(b *testing.B) {
 	bh.runBenchmark(b, true, putPeers, func(i int, ps storage.PeerStorage, bd *benchData) error {
-		ps.ScrapeSwarm(bd.infoHashes[i%ihCount], bd.peers[0])
+		ps.ScrapeSwarm(bd.infoHashes[i%ihCount])
 		return nil
 	})
 }

storage/test/storage_test_base.go

@@ -68,7 +68,7 @@ func (th *testHolder) AnnouncePeers(t *testing.T) {
 
 func (th *testHolder) ScrapeSwarm(t *testing.T) {
 	for _, c := range testData {
-		l, s, n := th.st.ScrapeSwarm(c.ih, c.peer)
+		l, s, n := th.st.ScrapeSwarm(c.ih)
 		require.Equal(t, uint32(0), s)
 		require.Equal(t, uint32(0), l)
 		require.Equal(t, uint32(0), n)
@@ -93,7 +93,7 @@ func (th *testHolder) LeecherPutAnnounceDeleteAnnounce(t *testing.T) {
 		require.Nil(t, err)
 		require.True(t, containsPeer(peers, c.peer))
 
-		l, s, _ := th.st.ScrapeSwarm(c.ih, c.peer)
+		l, s, _ := th.st.ScrapeSwarm(c.ih)
 		require.Equal(t, uint32(2), l)
 		require.Equal(t, uint32(0), s)
 
@@ -123,7 +123,7 @@ func (th *testHolder) SeederPutAnnounceDeleteAnnounce(t *testing.T) {
 		require.Nil(t, err)
 		require.True(t, containsPeer(peers, c.peer))
 
-		l, s, _ := th.st.ScrapeSwarm(c.ih, c.peer)
+		l, s, _ := th.st.ScrapeSwarm(c.ih)
 		require.Equal(t, uint32(1), l)
 		require.Equal(t, uint32(1), s)