Mirror/mochi
1
0
Fork 0
mirror of https://github.com/sot-tech/mochi.git synced 2026-07-16 13:18:12 -07:00
Code Issues Packages Projects Releases Wiki Activity

(tested) rollback to separate v4 and v6 keys in redis/keydb

Browse Source 
* sanitize log fields
* remove miniredis dependency
* store/collect information about hybrid (v2to1) hashes
This commit is contained in:
Lawrence, Rendall
2022-04-24 20:28:41 +03:00
parent 22f459315b
commit ef03291efe
15 changed files with 324 additions and 281 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/build.yaml
+8 -3
View File
@@ -22,6 +22,11 @@ jobs:
unit: unit:
name: "Run Unit Tests" name: "Run Unit Tests"
runs-on: "ubuntu-latest" runs-on: "ubuntu-latest"
services:
redis:
image: "eqalpha/keydb"
ports: [ "6379:6379" ]
options: "--entrypoint keydb-server"
steps: steps:
- uses: "actions/checkout@v3" - uses: "actions/checkout@v3"
- uses: "actions/setup-go@v3" - uses: "actions/setup-go@v3"
@@ -54,9 +59,9 @@ jobs:
runs-on: "ubuntu-latest" runs-on: "ubuntu-latest"
services: services:
redis: redis:
image: "redis" image: "eqalpha/keydb"
ports: [ "6379:6379" ] ports: [ "6379:6379" ]
options: "--entrypoint redis-server" options: "--entrypoint keydb-server"
steps: steps:
- uses: "actions/checkout@v3" - uses: "actions/checkout@v3"
- uses: "actions/setup-go@v3" - uses: "actions/setup-go@v3"
@@ -67,7 +72,7 @@ jobs:
go install --tags e2e ./cmd/mochi go install --tags e2e ./cmd/mochi
curl -LO https://github.com/jzelinskie/faq/releases/download/0.0.6/faq-linux-amd64 curl -LO https://github.com/jzelinskie/faq/releases/download/0.0.6/faq-linux-amd64
chmod +x faq-linux-amd64 chmod +x faq-linux-amd64
./faq-linux-amd64 '.mochi.storage = {"config":{"gc_interval":"3m","peer_lifetime":"31m","prometheus_reporting_interval":"1s","redis_broker":"redis://127.0.0.1:6379/0","connect_timeout":"15s","read_timeout":"15s","write_timeout":"15s"},"name":"redis"}' ./dist/example_config.yaml > ./dist/example_redis_config.yaml ./faq-linux-amd64 '.mochi.storage = {"config":{"gc_interval":"3m","peer_lifetime":"31m","prometheus_reporting_interval":"1s","connect_timeout":"15s","read_timeout":"15s","write_timeout":"15s"},"name":"redis"}' ./dist/example_config.yaml > ./dist/example_redis_config.yaml
cat ./dist/example_redis_config.yaml cat ./dist/example_redis_config.yaml
- name: "Run end-to-end tests" - name: "Run end-to-end tests"
run: | run: |
bittorrent/bittorrent.go
+2 -2
View File
@@ -275,8 +275,8 @@ func (p Peer) RawString() string {
// LogFields renders the current peer as a set of Logrus fields. // LogFields renders the current peer as a set of Logrus fields.
func (p Peer) LogFields() log.Fields { func (p Peer) LogFields() log.Fields {
return log.Fields{ return log.Fields{
"ID": p.ID, "id": p.ID,
"IP": p.Addr(), "ip": p.Addr(),
"port": p.Port(), "port": p.Port(),
} }
} }
cmd/mochi/config.go
+1
View File
@@ -16,6 +16,7 @@ import (
_ "github.com/sot-tech/mochi/middleware/varinterval" _ "github.com/sot-tech/mochi/middleware/varinterval"
// Imports to register storage drivers. // Imports to register storage drivers.
_ "github.com/sot-tech/mochi/storage/keydb"
_ "github.com/sot-tech/mochi/storage/memory" _ "github.com/sot-tech/mochi/storage/memory"
_ "github.com/sot-tech/mochi/storage/redis" _ "github.com/sot-tech/mochi/storage/redis"
) )
docs/storage/redis.md
+10 -10
View File
@@ -58,24 +58,24 @@ time as value.
Here is an example: Here is an example:
``` ```
- CHI_4_I - CHI_I
- CHI_4_S_<HASH1> - CHI_S4_<HASH1>
- CHI_4_L_<HASH1> - CHI_L4_<HASH1>
- CHI_4_S_<HASH1> - CHI_S4_<HASH1>
- <peer 1 key>: <modification time in unix nanos> - <peer 1 key>: <modification time in unix nanos>
- <peer 2 key>: <modification time in unix nanos> - <peer 2 key>: <modification time in unix nanos>
- CHI_4_L_<HASH2> - CHI_L4_<HASH2>
- <peer 3 key>: <modification time in unix nanos> - <peer 3 key>: <modification time in unix nanos>
... ...
``` ```
In this case, prometheus would record two swarms, three seeders, and one leecher. These three keys per address family In this case, prometheus would record two swarms, three seeders, and one leecher. These two keys
are used to record the count of swarms, seeders, and leechers. are used to record the count of seeders and leechers.
``` ```
- CHI_4_S_C: "3" - CHI_S_C: "3"
- CHI_6_L_C: "1" - CHI_L_C: "1"
``` ```
Note: `CHI_4_I` set has a different meaning compared to the `memory` storage: Note: `CHI_I` set has a different meaning compared to the `memory` storage:
It represents info hashes reported by seeder, meaning that info hashes without seeders are not counted. It represents info hashes reported by seeder, meaning that info hashes without seeders are not counted.
frontend/http/frontend.go
+1 -1
View File
@@ -349,7 +349,7 @@ func (f *Frontend) scrapeRoute(w http.ResponseWriter, r *http.Request, ps httpro
addr, err = netip.ParseAddr(host) addr, err = netip.ParseAddr(host)
if err != nil || addr.IsUnspecified() { if err != nil || addr.IsUnspecified() {
log.Error("http: invalid IP: neither v4 nor v6", log.Fields{"RemoteAddr": r.RemoteAddr}) log.Error("http: invalid IP: neither v4 nor v6", log.Fields{"remoteAddr": r.RemoteAddr})
WriteError(w, bittorrent.ErrInvalidIP) WriteError(w, bittorrent.ErrInvalidIP)
return return
} }
frontend/udp/parser.go
+12 -17
View File
@@ -81,8 +81,9 @@ func ParseAnnounce(r Request, v6Action bool, opts ParseOptions) (*bittorrent.Ann
return nil, errMalformedPacket return nil, errMalformedPacket
} }
// XXX: pure V2 hashes will cause invalid parsing // XXX: pure V2 hashes will cause invalid parsing,
infohash := r.Packet[16:36] // but BEP-52 says, that V2 hashes SHOULD be truncated
infoHash := r.Packet[16:36]
peerIDBytes := r.Packet[36:56] peerIDBytes := r.Packet[36:56]
downloaded := binary.BigEndian.Uint64(r.Packet[56:64]) downloaded := binary.BigEndian.Uint64(r.Packet[56:64])
left := binary.BigEndian.Uint64(r.Packet[64:72]) left := binary.BigEndian.Uint64(r.Packet[64:72])
@@ -117,7 +118,7 @@ func ParseAnnounce(r Request, v6Action bool, opts ParseOptions) (*bittorrent.Ann
return nil, err return nil, err
} }
ih, err := bittorrent.NewInfoHash(infohash) ih, err := bittorrent.NewInfoHash(infoHash)
if err != nil { if err != nil {
return nil, errInvalidInfoHash return nil, errInvalidInfoHash
} }
@@ -222,33 +223,27 @@ func ParseScrape(r Request, opts ParseOptions) (*bittorrent.ScrapeRequest, error
// Skip past the initial headers and check that the bytes left equal the // Skip past the initial headers and check that the bytes left equal the
// length of a valid list of infohashes. // length of a valid list of infohashes.
r.Packet = r.Packet[16:] r.Packet = r.Packet[16:]
l := len(r.Packet) // Only V1 and V2to1 (truncated) allowed
isV1, isV2 := l%bittorrent.InfoHashV1Len == 0, l%bittorrent.InfoHashV2Len == 0 if len(r.Packet)%bittorrent.InfoHashV1Len != 0 {
if !(isV1 || isV2) {
return nil, errMalformedPacket return nil, errMalformedPacket
} }
// Allocate a list of infohashes and append it to the list until we're out. // Allocate a list of infohashes and append it to the list until we're out.
var infohashes []bittorrent.InfoHash var infoHashes []bittorrent.InfoHash
var err error var err error
var request *bittorrent.ScrapeRequest var request *bittorrent.ScrapeRequest
pageSize := bittorrent.InfoHashV1Len for len(r.Packet) >= bittorrent.InfoHashV1Len {
if isV2 {
pageSize = bittorrent.InfoHashV2Len
}
for len(r.Packet) >= pageSize {
var ih bittorrent.InfoHash var ih bittorrent.InfoHash
if ih, err = bittorrent.NewInfoHash(r.Packet[:pageSize]); err == nil { if ih, err = bittorrent.NewInfoHash(r.Packet[:bittorrent.InfoHashV1Len]); err == nil {
infohashes = append(infohashes, ih) infoHashes = append(infoHashes, ih)
r.Packet = r.Packet[pageSize:] r.Packet = r.Packet[bittorrent.InfoHashV1Len:]
} else { } else {
break break
} }
} }
if err == nil { if err == nil {
// Sanitize the request. // Sanitize the request.
request = &bittorrent.ScrapeRequest{InfoHashes: infohashes} request = &bittorrent.ScrapeRequest{InfoHashes: infoHashes}
err = bittorrent.SanitizeScrape(request, opts.MaxScrapeInfoHashes) err = bittorrent.SanitizeScrape(request, opts.MaxScrapeInfoHashes)
} }
go.mod
+4 -8
View File
@@ -4,13 +4,12 @@ go 1.18
require ( require (
github.com/SermoDigital/jose v0.9.2-0.20180104203859-803625baeddc github.com/SermoDigital/jose v0.9.2-0.20180104203859-803625baeddc
github.com/alicebob/miniredis v2.5.0+incompatible
github.com/anacrolix/torrent v1.42.0 github.com/anacrolix/torrent v1.42.0
github.com/go-redis/redis/v8 v8.11.5 github.com/go-redis/redis/v8 v8.11.5
github.com/julienschmidt/httprouter v1.3.0 github.com/julienschmidt/httprouter v1.3.0
github.com/mendsley/gojwk v0.0.0-20141217222730-4d5ec6e58103 github.com/mendsley/gojwk v0.0.0-20141217222730-4d5ec6e58103
github.com/minio/sha256-simd v1.0.0 github.com/minio/sha256-simd v1.0.0
github.com/mitchellh/mapstructure v1.4.3 github.com/mitchellh/mapstructure v1.5.0
github.com/pkg/errors v0.9.1 github.com/pkg/errors v0.9.1
github.com/prometheus/client_golang v1.12.1 github.com/prometheus/client_golang v1.12.1
github.com/sirupsen/logrus v1.8.1 github.com/sirupsen/logrus v1.8.1
@@ -20,7 +19,6 @@ require (
) )
require ( require (
github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a // indirect
github.com/anacrolix/dht/v2 v2.17.0 // indirect github.com/anacrolix/dht/v2 v2.17.0 // indirect
github.com/anacrolix/log v0.13.1 // indirect github.com/anacrolix/log v0.13.1 // indirect
github.com/anacrolix/missinggo v1.3.0 // indirect github.com/anacrolix/missinggo v1.3.0 // indirect
@@ -30,19 +28,17 @@ require (
github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/fsnotify/fsnotify v1.5.1 // indirect github.com/fsnotify/fsnotify v1.5.3 // indirect
github.com/golang/protobuf v1.5.2 // indirect github.com/golang/protobuf v1.5.2 // indirect
github.com/gomodule/redigo v1.8.8 // indirect
github.com/huandu/xstrings v1.3.2 // indirect github.com/huandu/xstrings v1.3.2 // indirect
github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect
github.com/klauspost/cpuid/v2 v2.0.12 // indirect github.com/klauspost/cpuid/v2 v2.0.12 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_model v0.2.0 // indirect github.com/prometheus/client_model v0.2.0 // indirect
github.com/prometheus/common v0.33.0 // indirect github.com/prometheus/common v0.34.0 // indirect
github.com/prometheus/procfs v0.7.3 // indirect github.com/prometheus/procfs v0.7.3 // indirect
github.com/spf13/pflag v1.0.5 // indirect github.com/spf13/pflag v1.0.5 // indirect
github.com/yuin/gopher-lua v0.0.0-20210529063254-f4c35e4016d9 // indirect golang.org/x/sys v0.0.0-20220422013727-9388b58f7150 // indirect
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad // indirect
google.golang.org/protobuf v1.28.0 // indirect google.golang.org/protobuf v1.28.0 // indirect
) )
go.sum
+8 -17
View File
@@ -47,10 +47,6 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a h1:HbKu58rmZpUGpz5+4FfNmIU+FmZg2P3Xaj2v2bfNWmk=
github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc=
github.com/alicebob/miniredis v2.5.0+incompatible h1:yBHoLpsyjupjz3NL3MhKMVkR41j82Yjf3KFv7ApYzUI=
github.com/alicebob/miniredis v2.5.0+incompatible/go.mod h1:8HZjEj4yU0dwhYHky+DxYx+6BMjkBbe5ONFIF1MXffk=
github.com/anacrolix/dht/v2 v2.17.0 h1:MrAS6XqVCqNyyskTwxZB2sqhU/GGUdecb2TNe2b2QjE= github.com/anacrolix/dht/v2 v2.17.0 h1:MrAS6XqVCqNyyskTwxZB2sqhU/GGUdecb2TNe2b2QjE=
github.com/anacrolix/dht/v2 v2.17.0/go.mod h1:osiyaNrMLG9dw7wUtVMaII/NdCjlXeHjUcYzXnmop68= github.com/anacrolix/dht/v2 v2.17.0/go.mod h1:osiyaNrMLG9dw7wUtVMaII/NdCjlXeHjUcYzXnmop68=
github.com/anacrolix/envpprof v0.0.0-20180404065416-323002cec2fa/go.mod h1:KgHhUaQMc8cC0+cEflSgCFNFbKwi5h54gqtVn8yhP7c= github.com/anacrolix/envpprof v0.0.0-20180404065416-323002cec2fa/go.mod h1:KgHhUaQMc8cC0+cEflSgCFNFbKwi5h54gqtVn8yhP7c=
@@ -115,8 +111,8 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/frankban/quicktest v1.14.0 h1:+cqqvzZV87b4adx/5ayVOaYZ2CrvM4ejQvUdBzPPUss= github.com/frankban/quicktest v1.14.0 h1:+cqqvzZV87b4adx/5ayVOaYZ2CrvM4ejQvUdBzPPUss=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= github.com/fsnotify/fsnotify v1.5.3 h1:vNFpj2z7YIbwh2bw7x35sqYpp2wfuq+pivKbWG09B8c=
github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= github.com/fsnotify/fsnotify v1.5.3/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU=
github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE=
github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE=
github.com/glycerine/go-unsnap-stream v0.0.0-20190901134440-81cf024a9e0a/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= github.com/glycerine/go-unsnap-stream v0.0.0-20190901134440-81cf024a9e0a/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE=
@@ -169,8 +165,6 @@ github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw
github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/gomodule/redigo v1.8.8 h1:f6cXq6RRfiyrOJEV7p3JhLDlmawGBVBBP1MggY8Mo4E=
github.com/gomodule/redigo v1.8.8/go.mod h1:7ArFNvsTjH8GMMzB4uy1snslv2BwmginuMs06a1uzZE=
github.com/google/btree v0.0.0-20180124185431-e89373fe6b4a/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v0.0.0-20180124185431-e89373fe6b4a/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
@@ -247,8 +241,8 @@ github.com/mendsley/gojwk v0.0.0-20141217222730-4d5ec6e58103 h1:Z/i1e+gTZrmcGeZy
github.com/mendsley/gojwk v0.0.0-20141217222730-4d5ec6e58103/go.mod h1:o9YPB5aGP8ob35Vy6+vyq3P3bWe7NQWzf+JLiXCiMaE= github.com/mendsley/gojwk v0.0.0-20141217222730-4d5ec6e58103/go.mod h1:o9YPB5aGP8ob35Vy6+vyq3P3bWe7NQWzf+JLiXCiMaE=
github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g=
github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM=
github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
@@ -294,8 +288,8 @@ github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8b
github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
github.com/prometheus/common v0.33.0 h1:rHgav/0a6+uYgGdNt3jwz8FNSesO/Hsang3O0T9A5SE= github.com/prometheus/common v0.34.0 h1:RBmGO9d/FVjqHT0yUGQwBJhkwKV+wPCn7KGpvfab0uE=
github.com/prometheus/common v0.33.0/go.mod h1:gB3sOl7P0TvJabZpLY5uQMpUqRCPPCyRLCZYc7JZTNE= github.com/prometheus/common v0.34.0/go.mod h1:gB3sOl7P0TvJabZpLY5uQMpUqRCPPCyRLCZYc7JZTNE=
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
@@ -342,8 +336,6 @@ github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPy
github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/gopher-lua v0.0.0-20210529063254-f4c35e4016d9 h1:k/gmLsJDWwWqbLCur2yWnJzwQEKRcAHXo6seXGuSwWw=
github.com/yuin/gopher-lua v0.0.0-20210529063254-f4c35e4016d9/go.mod h1:E1AXubJBdNmFERAOucpDIxNzeGfLzg0mYh+UfMWdChA=
go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
@@ -442,7 +434,6 @@ golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5h
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190204203706-41f3e6584952/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -481,8 +472,8 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad h1:ntjMns5wyP/fN65tdBD4g8J5w8n015+iIIs9rtjXkY0= golang.org/x/sys v0.0.0-20220422013727-9388b58f7150 h1:xHms4gcpe1YE7A3yIllJXP16CMAGuqwO2lX1mTyyRRc=
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
middleware/hooks.go
+34 -23
View File
@@ -30,38 +30,45 @@ type swarmInteractionHook struct {
store storage.PeerStorage store storage.PeerStorage
} }
func (h *swarmInteractionHook) HandleAnnounce(ctx context.Context, req *bittorrent.AnnounceRequest, _ *bittorrent.AnnounceResponse) (_ context.Context, err error) { func (h *swarmInteractionHook) HandleAnnounce(ctx context.Context, req *bittorrent.AnnounceRequest, _ *bittorrent.AnnounceResponse) (outCtx context.Context, err error) {
outCtx = ctx
if ctx.Value(SkipSwarmInteractionKey) != nil { if ctx.Value(SkipSwarmInteractionKey) != nil {
return ctx, nil return
} }
var storeFn func(bittorrent.InfoHash, bittorrent.Peer) error
switch { switch {
case req.Event == bittorrent.Stopped: case req.Event == bittorrent.Stopped:
err = h.store.DeleteSeeder(req.InfoHash, req.Peer) storeFn = func(hash bittorrent.InfoHash, peer bittorrent.Peer) error {
if err != nil && !errors.Is(err, storage.ErrResourceDoesNotExist) { err = h.store.DeleteSeeder(hash, peer)
return ctx, err if err != nil && !errors.Is(err, storage.ErrResourceDoesNotExist) {
} return err
}
err = h.store.DeleteLeecher(req.InfoHash, req.Peer) err = h.store.DeleteLeecher(hash, peer)
if err != nil && !errors.Is(err, storage.ErrResourceDoesNotExist) { if err != nil && !errors.Is(err, storage.ErrResourceDoesNotExist) {
return ctx, err return err
}
return nil
} }
case req.Event == bittorrent.Completed: case req.Event == bittorrent.Completed:
err = h.store.GraduateLeecher(req.InfoHash, req.Peer) storeFn = h.store.GraduateLeecher
return ctx, err
case req.Left == 0: case req.Left == 0:
// Completed events will also have Left == 0, but by making this // Completed events will also have Left == 0, but by making this
// an extra case we can treat "old" seeders differently from // an extra case we can treat "old" seeders differently from
// graduating leechers. (Calling PutSeeder is probably faster // graduating leechers. (Calling PutSeeder is probably faster
// than calling GraduateLeecher.) // than calling GraduateLeecher.)
err = h.store.PutSeeder(req.InfoHash, req.Peer) storeFn = h.store.PutSeeder
return ctx, err
default: default:
err = h.store.PutLeecher(req.InfoHash, req.Peer) storeFn = h.store.PutLeecher
return ctx, err
} }
return ctx, nil if err = storeFn(req.InfoHash, req.Peer); err == nil && len(req.InfoHash) == bittorrent.InfoHashV2Len {
err = storeFn(req.InfoHash.TruncateV1(), req.Peer)
}
return
} }
func (h *swarmInteractionHook) HandleScrape(ctx context.Context, _ *bittorrent.ScrapeRequest, _ *bittorrent.ScrapeResponse) (context.Context, error) { func (h *swarmInteractionHook) HandleScrape(ctx context.Context, _ *bittorrent.ScrapeRequest, _ *bittorrent.ScrapeResponse) (context.Context, error) {
@@ -97,6 +104,10 @@ func (h *responseHook) HandleAnnounce(ctx context.Context, req *bittorrent.Annou
// Add the Scrape data to the response. // Add the Scrape data to the response.
resp.Incomplete, resp.Complete, _ = h.store.ScrapeSwarm(req.InfoHash, req.Peer) resp.Incomplete, resp.Complete, _ = h.store.ScrapeSwarm(req.InfoHash, req.Peer)
if len(req.InfoHash) == bittorrent.InfoHashV2Len {
incomplete, complete, _ := h.store.ScrapeSwarm(req.InfoHash.TruncateV1(), req.Peer)
resp.Incomplete, resp.Complete = resp.Incomplete+incomplete, resp.Complete+complete
}
err = h.appendPeers(req, resp) err = h.appendPeers(req, resp)
return ctx, err return ctx, err
@@ -159,14 +170,14 @@ func (h *responseHook) HandleScrape(ctx context.Context, req *bittorrent.ScrapeR
} }
for _, infoHash := range req.InfoHashes { for _, infoHash := range req.InfoHashes {
leechers, seeders, snatched := h.store.ScrapeSwarm(infoHash, req.Peer) scr := bittorrent.Scrape{InfoHash: infoHash}
scr.Incomplete, scr.Complete, scr.Snatches = h.store.ScrapeSwarm(infoHash, req.Peer)
if len(infoHash) == bittorrent.InfoHashV2Len {
leechers, seeders, snatched := h.store.ScrapeSwarm(infoHash.TruncateV1(), req.Peer)
scr.Incomplete, scr.Complete, scr.Snatches = scr.Incomplete+leechers, scr.Complete+seeders, scr.Snatches+snatched
}
resp.Files = append(resp.Files, bittorrent.Scrape{ resp.Files = append(resp.Files, scr)
InfoHash: infoHash,
Snatches: snatched,
Complete: seeders,
Incomplete: leechers,
})
} }
return ctx, nil return ctx, nil
storage/keydb/storage.go
+40 -35
View File
@@ -35,10 +35,10 @@ var ErrNotKeyDB = errors.New("provided instance seems not KeyDB")
func init() { func init() {
// Register the storage driver. // Register the storage driver.
storage.RegisterBuilder(Name, Builder) storage.RegisterBuilder(Name, builder)
} }
func Builder(icfg conf.MapConfig) (storage.PeerStorage, error) { func builder(icfg conf.MapConfig) (storage.PeerStorage, error) {
var cfg r.Config var cfg r.Config
var err error var err error
@@ -46,10 +46,10 @@ func Builder(icfg conf.MapConfig) (storage.PeerStorage, error) {
return nil, err return nil, err
} }
return New(cfg) return newStore(cfg)
} }
func New(cfg r.Config) (*store, error) { func newStore(cfg r.Config) (*store, error) {
var err error var err error
if cfg, err = cfg.Validate(); err != nil { if cfg, err = cfg.Validate(); err != nil {
return nil, err return nil, err
@@ -70,7 +70,11 @@ func New(cfg r.Config) (*store, error) {
var st *store var st *store
if err == nil { if err == nil {
st = &store{rs, cfg.LogFields(), uint(cfg.PeerLifetime.Seconds())} st = &store{
Connection: rs,
logFields: cfg.LogFields(),
peerTTL: uint(cfg.PeerLifetime.Seconds()),
}
} }
return st, err return st, err
@@ -82,27 +86,27 @@ type store struct {
peerTTL uint peerTTL uint
} }
func (s store) setPeerTTL(infoHashKey, peerId string) error { func (s store) setPeerTTL(infoHashKey, peerID string) error {
return s.Process(context.TODO(), redis.NewCmd(context.TODO(), expireMemberCmd, infoHashKey, peerId, s.peerTTL)) return s.Process(context.TODO(), redis.NewCmd(context.TODO(), expireMemberCmd, infoHashKey, peerID, s.peerTTL))
} }
func (s store) addPeer(infoHashKey, peerId string) (err error) { func (s store) addPeer(infoHashKey, peerID string) (err error) {
log.Debug("storage: KeyDB: PutPeer", log.Fields{ log.Debug("storage: KeyDB: PutPeer", log.Fields{
"InfoHashKey": infoHashKey, "infoHashKey": infoHashKey,
"PeerId": peerId, "peerID": peerID,
}) })
if err = s.SAdd(context.TODO(), infoHashKey, peerId).Err(); err == nil { if err = s.SAdd(context.TODO(), infoHashKey, peerID).Err(); err == nil {
err = s.setPeerTTL(infoHashKey, peerId) err = s.setPeerTTL(infoHashKey, peerID)
} }
return return
} }
func (s store) delPeer(infoHashKey, peerId string) error { func (s store) delPeer(infoHashKey, peerID string) error {
log.Debug("storage: KeyDB: DeletePeer", log.Fields{ log.Debug("storage: KeyDB: DeletePeer", log.Fields{
"InfoHashKey": infoHashKey, "infoHashKey": infoHashKey,
"PeerId": peerId, "peerID": peerID,
}) })
deleted, err := s.SRem(context.TODO(), infoHashKey, peerId).Uint64() deleted, err := s.SRem(context.TODO(), infoHashKey, peerID).Uint64()
err = r.AsNil(err) err = r.AsNil(err)
if err == nil && deleted == 0 { if err == nil && deleted == 0 {
err = storage.ErrResourceDoesNotExist err = storage.ErrResourceDoesNotExist
@@ -112,34 +116,35 @@ func (s store) delPeer(infoHashKey, peerId string) error {
} }
func (s store) PutSeeder(ih bittorrent.InfoHash, peer bittorrent.Peer) error { func (s store) PutSeeder(ih bittorrent.InfoHash, peer bittorrent.Peer) error {
return s.addPeer(r.IHSeederKey+ih.RawString(), peer.RawString()) return s.addPeer(r.InfoHashKey(ih.RawString(), true, peer.Addr().Is6()), peer.RawString())
} }
func (s store) DeleteSeeder(ih bittorrent.InfoHash, peer bittorrent.Peer) error { func (s store) DeleteSeeder(ih bittorrent.InfoHash, peer bittorrent.Peer) error {
return s.delPeer(r.IHSeederKey+ih.RawString(), peer.RawString()) return s.delPeer(r.InfoHashKey(ih.RawString(), true, peer.Addr().Is6()), peer.RawString())
} }
func (s store) PutLeecher(ih bittorrent.InfoHash, peer bittorrent.Peer) error { func (s store) PutLeecher(ih bittorrent.InfoHash, peer bittorrent.Peer) error {
return s.addPeer(r.IHLeecherKey+ih.RawString(), peer.RawString()) return s.addPeer(r.InfoHashKey(ih.RawString(), false, peer.Addr().Is6()), peer.RawString())
} }
func (s store) DeleteLeecher(ih bittorrent.InfoHash, peer bittorrent.Peer) error { func (s store) DeleteLeecher(ih bittorrent.InfoHash, peer bittorrent.Peer) error {
return s.delPeer(r.IHLeecherKey+ih.RawString(), peer.RawString()) return s.delPeer(r.InfoHashKey(ih.RawString(), false, peer.Addr().Is6()), peer.RawString())
} }
func (s store) GraduateLeecher(ih bittorrent.InfoHash, peer bittorrent.Peer) (err error) { func (s store) GraduateLeecher(ih bittorrent.InfoHash, peer bittorrent.Peer) (err error) {
log.Debug("storage: KeyDB: GraduateLeecher", log.Fields{ log.Debug("storage: KeyDB: GraduateLeecher", log.Fields{
"InfoHash": ih, "infoHash": ih,
"Peer": peer, "peer": peer,
}) })
infoHash, peerId := ih.RawString(), peer.RawString() infoHash, peerID := ih.RawString(), peer.RawString()
ihSeederKey, ihLeecherKey := r.IHSeederKey+infoHash, r.IHLeecherKey+infoHash ihSeederKey := r.InfoHashKey(infoHash, true, peer.Addr().Is6())
ihLeecherKey := r.InfoHashKey(infoHash, false, peer.Addr().Is6())
var moved bool var moved bool
if moved, err = s.SMove(context.TODO(), ihLeecherKey, ihSeederKey, peerId).Result(); err == nil { if moved, err = s.SMove(context.TODO(), ihLeecherKey, ihSeederKey, peerID).Result(); err == nil {
if moved { if moved {
err = s.setPeerTTL(ihSeederKey, peerId) err = s.setPeerTTL(ihSeederKey, peerID)
} else { } else {
err = s.addPeer(ihSeederKey, peerId) err = s.addPeer(ihSeederKey, peerID)
} }
} }
return err return err
@@ -148,22 +153,22 @@ func (s store) GraduateLeecher(ih bittorrent.InfoHash, peer bittorrent.Peer) (er
// AnnouncePeers is the same function as redis.AnnouncePeers // AnnouncePeers is the same function as redis.AnnouncePeers
func (s store) AnnouncePeers(ih bittorrent.InfoHash, seeder bool, numWant int, peer bittorrent.Peer) ([]bittorrent.Peer, error) { func (s store) AnnouncePeers(ih bittorrent.InfoHash, seeder bool, numWant int, peer bittorrent.Peer) ([]bittorrent.Peer, error) {
log.Debug("storage: KeyDB: AnnouncePeers", log.Fields{ log.Debug("storage: KeyDB: AnnouncePeers", log.Fields{
"InfoHash": ih, "infoHash": ih,
"Seeder": seeder, "seeder": seeder,
"NumWant": numWant, "numWant": numWant,
"Peer": peer, "peer": peer,
}) })
return s.GetPeers(ih, seeder, numWant, peer, func(ctx context.Context, infoHashKey string) *redis.StringSliceCmd { return s.GetPeers(ih, seeder, numWant, peer, func(ctx context.Context, infoHashKey string, maxCount int) *redis.StringSliceCmd {
return s.SRandMemberN(context.TODO(), infoHashKey, int64(numWant)) return s.SRandMemberN(context.TODO(), infoHashKey, int64(maxCount))
}) })
} }
// ScrapeSwarm is the same function as redis.ScrapeSwarm except `SCard` call instead of `HLen` // ScrapeSwarm is the same function as redis.ScrapeSwarm except `SCard` call instead of `HLen`
func (s store) ScrapeSwarm(ih bittorrent.InfoHash, peer bittorrent.Peer) (leechers uint32, seeders uint32, snatched uint32) { func (s store) ScrapeSwarm(ih bittorrent.InfoHash, peer bittorrent.Peer) (leechers uint32, seeders uint32, snatched uint32) {
log.Debug("storage: KeyDB ScrapeSwarm", log.Fields{ log.Debug("storage: KeyDB ScrapeSwarm", log.Fields{
"InfoHash": ih, "infoHash": ih,
"Peer": peer, "peer": peer,
}) })
leechers, seeders = s.CountPeers(ih, s.SCard) leechers, seeders = s.CountPeers(ih, s.SCard)
return return
storage/keydb/storage_test.go
+1 -1
View File
@@ -21,7 +21,7 @@ var cfg = r.Config{
func createNew() s.PeerStorage { func createNew() s.PeerStorage {
var ps s.PeerStorage var ps s.PeerStorage
var err error var err error
ps, err = New(cfg) ps, err = newStore(cfg)
if err != nil { if err != nil {
panic(fmt.Sprint("Unable to create KeyDB connection: ", err, "\nThis driver needs real KeyDB instance")) panic(fmt.Sprint("Unable to create KeyDB connection: ", err, "\nThis driver needs real KeyDB instance"))
} }
storage/memory/storage.go
+7 -7
View File
@@ -30,10 +30,10 @@ const (
func init() { func init() {
// Register the storage driver. // Register the storage driver.
storage.RegisterBuilder(Name, Builder) storage.RegisterBuilder(Name, builder)
} }
func Builder(icfg conf.MapConfig) (storage.PeerStorage, error) { func builder(icfg conf.MapConfig) (storage.PeerStorage, error) {
var cfg Config var cfg Config
if err := icfg.Unmarshal(&cfg); err != nil { if err := icfg.Unmarshal(&cfg); err != nil {
return nil, err return nil, err
@@ -49,8 +49,8 @@ type Config struct {
// LogFields renders the current config as a set of Logrus fields. // LogFields renders the current config as a set of Logrus fields.
func (cfg Config) LogFields() log.Fields { func (cfg Config) LogFields() log.Fields {
return log.Fields{ return log.Fields{
"Name": Name, "name": Name,
"ShardCount": cfg.ShardCount, "shardCount": cfg.ShardCount,
} }
} }
@@ -64,9 +64,9 @@ func (cfg Config) Validate() Config {
if cfg.ShardCount <= 0 || cfg.ShardCount > (math.MaxInt/2) { if cfg.ShardCount <= 0 || cfg.ShardCount > (math.MaxInt/2) {
validcfg.ShardCount = defaultShardCount validcfg.ShardCount = defaultShardCount
log.Warn("falling back to default configuration", log.Fields{ log.Warn("falling back to default configuration", log.Fields{
"Name": Name + ".ShardCount", "name": Name + ".ShardCount",
"Provided": cfg.ShardCount, "provided": cfg.ShardCount,
"Default": validcfg.ShardCount, "default": validcfg.ShardCount,
}) })
} }
storage/redis/storage.go
+185 -134
View File
@@ -2,7 +2,7 @@
// BitTorrent tracker keeping peer data in redis with hash. // BitTorrent tracker keeping peer data in redis with hash.
// There two categories of hash: // There two categories of hash:
// //
// - CHI_{L,S}_<HASH> (hash type) // - CHI_{L,S}{4,6}_<HASH> (hash type)
// To save peers that hold the infohash, used for fast searching, // To save peers that hold the infohash, used for fast searching,
// deleting, and timeout handling // deleting, and timeout handling
// //
@@ -50,10 +50,14 @@ const (
PrefixKey = "CHI_" PrefixKey = "CHI_"
// IHKey redis hash key for all info hashes // IHKey redis hash key for all info hashes
IHKey = "CHI_I" IHKey = "CHI_I"
// IHSeederKey redis hash key prefix for seeders // IH4SeederKey redis hash key prefix for IPv4 seeders
IHSeederKey = "CHI_S_" IH4SeederKey = "CHI_S4_"
// IHLeecherKey redis hash key prefix for leechers // IH6SeederKey redis hash key prefix for IPv6 seeders
IHLeecherKey = "CHI_L_" IH6SeederKey = "CHI_S6_"
// IH4LeecherKey redis hash key prefix for IPv4 leechers
IH4LeecherKey = "CHI_L4_"
// IH6LeecherKey redis hash key prefix for IPv6 leechers
IH6LeecherKey = "CHI_L6_"
// CountSeederKey redis key for seeder count // CountSeederKey redis key for seeder count
CountSeederKey = "CHI_C_S" CountSeederKey = "CHI_C_S"
// CountLeecherKey redis key for leecher count // CountLeecherKey redis key for leecher count
@@ -65,10 +69,10 @@ var ErrSentinelAndClusterChecked = errors.New("unable to use both cluster and se
func init() { func init() {
// Register the storage builder. // Register the storage builder.
storage.RegisterBuilder(Name, Builder) storage.RegisterBuilder(Name, builder)
} }
func Builder(icfg conf.MapConfig) (storage.PeerStorage, error) { func builder(icfg conf.MapConfig) (storage.PeerStorage, error) {
// Unmarshal the bytes into the proper config type. // Unmarshal the bytes into the proper config type.
var cfg Config var cfg Config
@@ -76,10 +80,10 @@ func Builder(icfg conf.MapConfig) (storage.PeerStorage, error) {
return nil, err return nil, err
} }
return New(cfg) return newStore(cfg)
} }
func New(cfg Config) (*store, error) { func newStore(cfg Config) (*store, error) {
cfg, err := cfg.Validate() cfg, err := cfg.Validate()
if err != nil { if err != nil {
return nil, err return nil, err
@@ -117,17 +121,17 @@ type Config struct {
// LogFields renders the current config as a set of Logrus fields. // LogFields renders the current config as a set of Logrus fields.
func (cfg Config) LogFields() log.Fields { func (cfg Config) LogFields() log.Fields {
return log.Fields{ return log.Fields{
"Name": Name, "name": Name,
"PeerLifetime": cfg.PeerLifetime, "peerLifetime": cfg.PeerLifetime,
"Addresses": cfg.Addresses, "addresses": cfg.Addresses,
"DB": cfg.DB, "db": cfg.DB,
"PoolSize": cfg.PoolSize, "poolSize": cfg.PoolSize,
"Sentinel": cfg.Sentinel, "sentinel": cfg.Sentinel,
"SentinelMaster": cfg.SentinelMaster, "sentinelMaster": cfg.SentinelMaster,
"Cluster": cfg.Cluster, "cluster": cfg.Cluster,
"ReadTimeout": cfg.ReadTimeout, "readTimeout": cfg.ReadTimeout,
"WriteTimeout": cfg.WriteTimeout, "writeTimeout": cfg.WriteTimeout,
"ConnectTimeout": cfg.ConnectTimeout, "connectTimeout": cfg.ConnectTimeout,
} }
} }
@@ -154,42 +158,43 @@ func (cfg Config) Validate() (Config, error) {
if len(cfg.Addresses) == 0 { if len(cfg.Addresses) == 0 {
validCfg.Addresses = []string{defaultRedisAddress} validCfg.Addresses = []string{defaultRedisAddress}
log.Warn("falling back to default configuration", log.Fields{ log.Warn("falling back to default configuration", log.Fields{
"Name": Name + ".Addresses", "name": Name + ".Addresses",
"Provided": cfg.Addresses, "provided": cfg.Addresses,
"Default": validCfg.Addresses, "default": validCfg.Addresses,
}) })
} }
if cfg.ReadTimeout <= 0 { if cfg.ReadTimeout <= 0 {
validCfg.ReadTimeout = defaultReadTimeout validCfg.ReadTimeout = defaultReadTimeout
log.Warn("falling back to default configuration", log.Fields{ log.Warn("falling back to default configuration", log.Fields{
"Name": Name + ".ReadTimeout", "name": Name + ".ReadTimeout",
"Provided": cfg.ReadTimeout, "provided": cfg.ReadTimeout,
"Default": validCfg.ReadTimeout, "default": validCfg.ReadTimeout,
}) })
} }
if cfg.WriteTimeout <= 0 { if cfg.WriteTimeout <= 0 {
validCfg.WriteTimeout = defaultWriteTimeout validCfg.WriteTimeout = defaultWriteTimeout
log.Warn("falling back to default configuration", log.Fields{ log.Warn("falling back to default configuration", log.Fields{
"Name": Name + ".WriteTimeout", "name": Name + ".WriteTimeout",
"Provided": cfg.WriteTimeout, "provided": cfg.WriteTimeout,
"Default": validCfg.WriteTimeout, "default": validCfg.WriteTimeout,
}) })
} }
if cfg.ConnectTimeout <= 0 { if cfg.ConnectTimeout <= 0 {
validCfg.ConnectTimeout = defaultConnectTimeout validCfg.ConnectTimeout = defaultConnectTimeout
log.Warn("falling back to default configuration", log.Fields{ log.Warn("falling back to default configuration", log.Fields{
"Name": Name + ".ConnectTimeout", "name": Name + ".ConnectTimeout",
"Provided": cfg.ConnectTimeout, "provided": cfg.ConnectTimeout,
"Default": validCfg.ConnectTimeout, "default": validCfg.ConnectTimeout,
}) })
} }
return validCfg, nil return validCfg, nil
} }
// Connect creates redis client from configuration
func (cfg Config) Connect() (con Connection, err error) { func (cfg Config) Connect() (con Connection, err error) {
var rs redis.UniversalClient var rs redis.UniversalClient
switch { switch {
@@ -287,6 +292,7 @@ func (ps *store) ScheduleStatisticsCollection(reportInterval time.Duration) {
}() }()
} }
// Connection is wrapper for redis.UniversalClient
type Connection struct { type Connection struct {
redis.UniversalClient redis.UniversalClient
} }
@@ -309,7 +315,7 @@ func (ps *store) count(key string, getLength bool) (n uint64) {
if err != nil { if err != nil {
log.Error("storage: Redis: GET/SCARD failure", log.Fields{ log.Error("storage: Redis: GET/SCARD failure", log.Fields{
"key": key, "key": key,
"Error": err, "error": err,
}) })
} }
return return
@@ -336,6 +342,8 @@ func (ps *store) tx(txf func(tx redis.Pipeliner) error) (err error) {
return return
} }
// AsNil returns nil if provided err is redis.Nil
// otherwise returns err
func AsNil(err error) error { func AsNil(err error) error {
if err == nil || errors.Is(err, redis.Nil) { if err == nil || errors.Is(err, redis.Nil) {
return nil return nil
@@ -343,14 +351,37 @@ func AsNil(err error) error {
return err return err
} }
func (ps *store) putPeer(infoHashKey, peerCountKey, peerId string) error { // InfoHashKey generates redis key for provided hash and flags
func InfoHashKey(infoHash string, seeder, v6 bool) (infoHashKey string) {
var bm int
if seeder {
bm = 0b01
}
if v6 {
bm |= 0b10
}
switch bm {
case 0b11:
infoHashKey = IH6SeederKey
case 0b10:
infoHashKey = IH6LeecherKey
case 0b01:
infoHashKey = IH4SeederKey
case 0b00:
infoHashKey = IH4LeecherKey
}
infoHashKey += infoHash
return
}
func (ps *store) putPeer(infoHashKey, peerCountKey, peerID string) error {
log.Debug("storage: Redis: PutPeer", log.Fields{ log.Debug("storage: Redis: PutPeer", log.Fields{
"InfoHashKey": infoHashKey, "infoHashKey": infoHashKey,
"PeerCountKey": peerCountKey, "peerCountKey": peerCountKey,
"PeerId": peerId, "peerID": peerID,
}) })
return ps.tx(func(tx redis.Pipeliner) (err error) { return ps.tx(func(tx redis.Pipeliner) (err error) {
if err = tx.HSet(context.TODO(), infoHashKey, peerId, ps.getClock()).Err(); err != nil { if err = tx.HSet(context.TODO(), infoHashKey, peerID, ps.getClock()).Err(); err != nil {
return return
} }
if err = tx.Incr(context.TODO(), peerCountKey).Err(); err != nil { if err = tx.Incr(context.TODO(), peerCountKey).Err(); err != nil {
@@ -361,13 +392,13 @@ func (ps *store) putPeer(infoHashKey, peerCountKey, peerId string) error {
}) })
} }
func (ps *store) delPeer(infoHashKey, peerCountKey, peerId string) error { func (ps *store) delPeer(infoHashKey, peerCountKey, peerID string) error {
log.Debug("storage: Redis: DeletePeer", log.Fields{ log.Debug("storage: Redis: DeletePeer", log.Fields{
"InfoHashKey": infoHashKey, "infoHashKey": infoHashKey,
"PeerCountKey": peerCountKey, "peerCountKey": peerCountKey,
"PeerId": peerId, "peerID": peerID,
}) })
deleted, err := ps.HDel(context.TODO(), infoHashKey, peerId).Uint64() deleted, err := ps.HDel(context.TODO(), infoHashKey, peerID).Uint64()
err = AsNil(err) err = AsNil(err)
if err == nil { if err == nil {
if deleted == 0 { if deleted == 0 {
@@ -381,32 +412,32 @@ func (ps *store) delPeer(infoHashKey, peerCountKey, peerId string) error {
} }
func (ps *store) PutSeeder(ih bittorrent.InfoHash, peer bittorrent.Peer) error { func (ps *store) PutSeeder(ih bittorrent.InfoHash, peer bittorrent.Peer) error {
return ps.putPeer(IHSeederKey+ih.RawString(), CountSeederKey, peer.RawString()) return ps.putPeer(InfoHashKey(ih.RawString(), true, peer.Addr().Is6()), CountSeederKey, peer.RawString())
} }
func (ps *store) DeleteSeeder(ih bittorrent.InfoHash, peer bittorrent.Peer) error { func (ps *store) DeleteSeeder(ih bittorrent.InfoHash, peer bittorrent.Peer) error {
return ps.delPeer(IHSeederKey+ih.RawString(), CountSeederKey, peer.RawString()) return ps.delPeer(InfoHashKey(ih.RawString(), true, peer.Addr().Is6()), CountSeederKey, peer.RawString())
} }
func (ps *store) PutLeecher(ih bittorrent.InfoHash, peer bittorrent.Peer) error { func (ps *store) PutLeecher(ih bittorrent.InfoHash, peer bittorrent.Peer) error {
return ps.putPeer(IHLeecherKey+ih.RawString(), CountLeecherKey, peer.RawString()) return ps.putPeer(InfoHashKey(ih.RawString(), false, peer.Addr().Is6()), CountLeecherKey, peer.RawString())
} }
func (ps *store) DeleteLeecher(ih bittorrent.InfoHash, peer bittorrent.Peer) error { func (ps *store) DeleteLeecher(ih bittorrent.InfoHash, peer bittorrent.Peer) error {
return ps.delPeer(IHLeecherKey+ih.RawString(), CountLeecherKey, peer.RawString()) return ps.delPeer(InfoHashKey(ih.RawString(), false, peer.Addr().Is6()), CountLeecherKey, peer.RawString())
} }
func (ps *store) GraduateLeecher(ih bittorrent.InfoHash, peer bittorrent.Peer) error { func (ps *store) GraduateLeecher(ih bittorrent.InfoHash, peer bittorrent.Peer) error {
log.Debug("storage: Redis: GraduateLeecher", log.Fields{ log.Debug("storage: Redis: GraduateLeecher", log.Fields{
"InfoHash": ih, "infoHash": ih,
"Peer": peer, "peer": peer,
}) })
infoHash, peerId := ih.RawString(), peer.RawString() infoHash, peerID, isV6 := ih.RawString(), peer.RawString(), peer.Addr().Is6()
ihSeederKey, ihLeecherKey := IHSeederKey+infoHash, IHLeecherKey+infoHash ihSeederKey, ihLeecherKey := InfoHashKey(infoHash, true, isV6), InfoHashKey(infoHash, false, isV6)
return ps.tx(func(tx redis.Pipeliner) error { return ps.tx(func(tx redis.Pipeliner) error {
deleted, err := tx.HDel(context.TODO(), ihLeecherKey, peerId).Uint64() deleted, err := tx.HDel(context.TODO(), ihLeecherKey, peerID).Uint64()
err = AsNil(err) err = AsNil(err)
if err == nil { if err == nil {
if deleted > 0 { if deleted > 0 {
@@ -414,7 +445,7 @@ func (ps *store) GraduateLeecher(ih bittorrent.InfoHash, peer bittorrent.Peer) e
} }
} }
if err == nil { if err == nil {
err = tx.HSet(context.TODO(), ihSeederKey, peerId, ps.getClock()).Err() err = tx.HSet(context.TODO(), ihSeederKey, peerID, ps.getClock()).Err()
} }
if err == nil { if err == nil {
err = tx.Incr(context.TODO(), CountSeederKey).Err() err = tx.Incr(context.TODO(), CountSeederKey).Err()
@@ -426,19 +457,16 @@ func (ps *store) GraduateLeecher(ih bittorrent.InfoHash, peer bittorrent.Peer) e
}) })
} }
func (ps Connection) parsePeersList(peersResult *redis.StringSliceCmd, skipPeerId string, v4Only bool) (peers []bittorrent.Peer, err error) { func (ps Connection) parsePeersList(peersResult *redis.StringSliceCmd, skipPeerID string) (peers []bittorrent.Peer, err error) {
var peerIds []string var peerIds []string
peerIds, err = peersResult.Result() peerIds, err = peersResult.Result()
if err = AsNil(err); err == nil { if err = AsNil(err); err == nil {
for _, peerId := range peerIds { for _, peerID := range peerIds {
if peerId != skipPeerId { if peerID != skipPeerID {
if p, err := bittorrent.NewPeer(peerId); err == nil { if p, err := bittorrent.NewPeer(peerID); err == nil {
// If peer from request is V4 only, it won't receive V6 peers from DB peers = append(peers, p)
if !(v4Only && p.Addr().Is6()) {
peers = append(peers, p)
}
} else { } else {
log.Error("storage: Redis: unable to decode leecher", log.Fields{"PeerId": peerId}) log.Error("storage: Redis: unable to decode leecher", log.Fields{"peerID": peerID})
} }
} }
} }
@@ -446,31 +474,52 @@ func (ps Connection) parsePeersList(peersResult *redis.StringSliceCmd, skipPeerI
return return
} }
func (ps Connection) GetPeers(ih bittorrent.InfoHash, forSeeder bool, maxCount int, peer bittorrent.Peer, type getPeersFn func(context.Context, string, int) *redis.StringSliceCmd
membersFn func(context.Context, string) *redis.StringSliceCmd) (peers []bittorrent.Peer, err error) {
infoHash, peerId, isV4 := ih.RawString(), peer.RawString(), peer.Addr().Is4() // GetPeers retrieves peers for provided info hash by calling membersFn and
// converts result to bittorrent.Peer array.
// If forSeeder set to true - returns only leechers, if false -
// seeders and if maxCount not reached - leechers.
func (ps Connection) GetPeers(ih bittorrent.InfoHash, forSeeder bool, maxCount int, peer bittorrent.Peer, membersFn getPeersFn) (out []bittorrent.Peer, err error) {
infoHash, peerID, isV6 := ih.RawString(), peer.RawString(), peer.Addr().Is6()
var infoHashKeys []string
if forSeeder { if forSeeder {
peers, err = ps.parsePeersList(membersFn(context.TODO(), IHLeecherKey+infoHash), peerId, isV4) infoHashKeys = append(infoHashKeys, InfoHashKey(infoHash, false, isV6),
InfoHashKey(infoHash, false, !isV6))
} else { } else {
// Append as many seeders as possible. // Append as many peers as possible.
peers, err = ps.parsePeersList(membersFn(context.TODO(), IHSeederKey+infoHash), peerId, isV4) // Priority:
if err != nil { // same ip family seeders > same ip family leechers >
return // foreign ip family seeders > foreign ip family leechers
} infoHashKeys = append(infoHashKeys,
InfoHashKey(infoHash, true, isV6),
InfoHashKey(infoHash, false, isV6),
InfoHashKey(infoHash, true, !isV6),
InfoHashKey(infoHash, false, !isV6))
}
if maxCount -= len(peers); maxCount > 0 { for _, infoHashKey := range infoHashKeys {
if leechers, err := ps.parsePeersList(membersFn(context.TODO(), IHLeecherKey+infoHash), peerId, isV4); err == nil { var peers []bittorrent.Peer
peers = append(peers, leechers...) peers, err = ps.parsePeersList(membersFn(context.TODO(), infoHashKey, maxCount), peerID)
} else { maxCount -= len(peers)
log.Warn("storage: Redis: error occurred while receiving leechers", log.Fields{"InfoHash": ih, "Error": err}) out = append(out, peers...)
} if err != nil || maxCount <= 0 {
break
} }
}
if len(peers) == 0 && !isV4 { if l := len(out); err == nil {
if l == 0 {
err = storage.ErrResourceDoesNotExist err = storage.ErrResourceDoesNotExist
} }
} else if l > 0 {
err = nil
log.Warn("storage: Redis: error occurred while retrieving peers", log.Fields{
"infoHash": infoHash,
"error": err,
})
} }
return return
@@ -478,50 +527,47 @@ func (ps Connection) GetPeers(ih bittorrent.InfoHash, forSeeder bool, maxCount i
func (ps *store) AnnouncePeers(ih bittorrent.InfoHash, seeder bool, numWant int, peer bittorrent.Peer) ([]bittorrent.Peer, error) { func (ps *store) AnnouncePeers(ih bittorrent.InfoHash, seeder bool, numWant int, peer bittorrent.Peer) ([]bittorrent.Peer, error) {
log.Debug("storage: Redis: AnnouncePeers", log.Fields{ log.Debug("storage: Redis: AnnouncePeers", log.Fields{
"InfoHash": ih, "infoHash": ih,
"Seeder": seeder, "seeder": seeder,
"NumWant": numWant, "numWant": numWant,
"Peer": peer, "peer": peer,
}) })
return ps.GetPeers(ih, seeder, numWant, peer, func(ctx context.Context, infoHashKey string) *redis.StringSliceCmd { return ps.GetPeers(ih, seeder, numWant, peer, func(ctx context.Context, infoHashKey string, maxCount int) *redis.StringSliceCmd {
return ps.HRandField(ctx, infoHashKey, numWant, false) return ps.HRandField(ctx, infoHashKey, maxCount, false)
}) })
} }
func (ps Connection) CountPeers(ih bittorrent.InfoHash, countFn func(context.Context, string) *redis.IntCmd) (leechersCount, seedersCount uint32) { type getPeerCountFn func(context.Context, string) *redis.IntCmd
func (ps Connection) countPeers(infoHashKey string, countFn getPeerCountFn) uint32 {
count, err := countFn(context.TODO(), infoHashKey).Result()
err = AsNil(err)
if err != nil {
log.Error("storage: Redis: key size calculation failure", log.Fields{
"infoHashKey": infoHashKey,
"error": err,
})
}
return uint32(count)
}
// CountPeers calls provided countFn and returns seeders and leechers count for specified info hash
func (ps Connection) CountPeers(ih bittorrent.InfoHash, countFn getPeerCountFn) (leechersCount, seedersCount uint32) {
infoHash := ih.RawString() infoHash := ih.RawString()
ihSeederKey, ihLeecherKey := IHSeederKey+infoHash, IHLeecherKey+infoHash
count, err := countFn(context.TODO(), ihLeecherKey).Result() leechersCount = ps.countPeers(InfoHashKey(infoHash, false, false), countFn) +
err = AsNil(err) ps.countPeers(InfoHashKey(infoHash, false, true), countFn)
if err != nil { seedersCount = ps.countPeers(InfoHashKey(infoHash, true, false), countFn) +
log.Error("storage: Redis: key size calculation failure", log.Fields{ ps.countPeers(InfoHashKey(infoHash, true, true), countFn)
"InfoHashKey": ihLeecherKey,
"Error": err,
})
return
}
leechersCount = uint32(count)
count, err = countFn(context.TODO(), ihSeederKey).Result()
err = AsNil(err)
if err != nil {
log.Error("storage: Redis: key size calculation failure", log.Fields{
"InfoHashKey": ihSeederKey,
"Error": err,
})
return
}
seedersCount = uint32(count)
return return
} }
func (ps *store) ScrapeSwarm(ih bittorrent.InfoHash, peer bittorrent.Peer) (leechers uint32, seeders uint32, snatched uint32) { func (ps *store) ScrapeSwarm(ih bittorrent.InfoHash, peer bittorrent.Peer) (leechers uint32, seeders uint32, _ uint32) {
log.Debug("storage: Redis ScrapeSwarm", log.Fields{ log.Debug("storage: Redis ScrapeSwarm", log.Fields{
"InfoHash": ih, "infoHash": ih,
"Peer": peer, "peer": peer,
}) })
leechers, seeders = ps.CountPeers(ih, ps.HLen) leechers, seeders = ps.CountPeers(ih, ps.HLen)
@@ -531,6 +577,7 @@ func (ps *store) ScrapeSwarm(ih bittorrent.InfoHash, peer bittorrent.Peer) (leec
const argNumErrorMsg = "ERR wrong number of arguments" const argNumErrorMsg = "ERR wrong number of arguments"
// Put - storage.DataStorage implementation
func (ps Connection) Put(ctx string, values ...storage.Entry) (err error) { func (ps Connection) Put(ctx string, values ...storage.Entry) (err error) {
if l := len(values); l > 0 { if l := len(values); l > 0 {
if l == 1 { if l == 1 {
@@ -556,11 +603,13 @@ func (ps Connection) Put(ctx string, values ...storage.Entry) (err error) {
return return
} }
// Contains - storage.DataStorage implementation
func (ps Connection) Contains(ctx string, key string) (bool, error) { func (ps Connection) Contains(ctx string, key string) (bool, error) {
exist, err := ps.HExists(context.TODO(), PrefixKey+ctx, key).Result() exist, err := ps.HExists(context.TODO(), PrefixKey+ctx, key).Result()
return exist, AsNil(err) return exist, AsNil(err)
} }
// Load - storage.DataStorage implementation
func (ps Connection) Load(ctx string, key string) (v any, err error) { func (ps Connection) Load(ctx string, key string) (v any, err error) {
v, err = ps.HGet(context.TODO(), PrefixKey+ctx, key).Result() v, err = ps.HGet(context.TODO(), PrefixKey+ctx, key).Result()
if err != nil && errors.Is(err, redis.Nil) { if err != nil && errors.Is(err, redis.Nil) {
@@ -569,6 +618,7 @@ func (ps Connection) Load(ctx string, key string) (v any, err error) {
return return
} }
// Delete - storage.DataStorage implementation
func (ps Connection) Delete(ctx string, keys ...string) (err error) { func (ps Connection) Delete(ctx string, keys ...string) (err error) {
if len(keys) > 0 { if len(keys) > 0 {
err = AsNil(ps.HDel(context.TODO(), PrefixKey+ctx, keys...).Err()) err = AsNil(ps.HDel(context.TODO(), PrefixKey+ctx, keys...).Err())
@@ -586,6 +636,7 @@ func (ps Connection) Delete(ctx string, keys ...string) (err error) {
return return
} }
// Preservable - storage.DataStorage implementation
func (Connection) Preservable() bool { func (Connection) Preservable() bool {
return true return true
} }
@@ -645,12 +696,12 @@ func (ps *store) gc(cutoff time.Time) {
for _, infoHashKey := range infoHashKeys { for _, infoHashKey := range infoHashKeys {
var cntKey string var cntKey string
var seeder bool var seeder bool
if seeder = strings.HasPrefix(infoHashKey, IHSeederKey); seeder { if seeder = strings.HasPrefix(infoHashKey, IH4SeederKey) || strings.HasPrefix(infoHashKey, IH6SeederKey); seeder {
cntKey = CountSeederKey cntKey = CountSeederKey
} else if strings.HasPrefix(infoHashKey, IHLeecherKey) { } else if strings.HasPrefix(infoHashKey, IH4LeecherKey) || strings.HasPrefix(infoHashKey, IH6LeecherKey) {
cntKey = CountLeecherKey cntKey = CountLeecherKey
} else { } else {
log.Warn("storage: Redis: unexpected record found in info hash set", log.Fields{"InfoHashKey": infoHashKey}) log.Warn("storage: Redis: unexpected record found in info hash set", log.Fields{"infoHashKey": infoHashKey})
continue continue
} }
// list all (peer, timeout) pairs for the ih // list all (peer, timeout) pairs for the ih
@@ -658,18 +709,18 @@ func (ps *store) gc(cutoff time.Time) {
err = AsNil(err) err = AsNil(err)
if err == nil { if err == nil {
peersToRemove := make([]string, 0) peersToRemove := make([]string, 0)
for peerId, timeStamp := range peerList { for peerID, timeStamp := range peerList {
if mtime, err := strconv.ParseInt(timeStamp, 10, 64); err == nil { if mtime, err := strconv.ParseInt(timeStamp, 10, 64); err == nil {
if mtime <= cutoffNanos { if mtime <= cutoffNanos {
log.Debug("storage: Redis: adding peer to remove list", log.Fields{"PeerId": peerId}) log.Debug("storage: Redis: adding peer to remove list", log.Fields{"peerID": peerID})
peersToRemove = append(peersToRemove, peerId) peersToRemove = append(peersToRemove, peerID)
} }
} else { } else {
log.Error("storage: Redis: unable to decode peer timestamp", log.Fields{ log.Error("storage: Redis: unable to decode peer timestamp", log.Fields{
"InfoHashKey": infoHashKey, "infoHashKey": infoHashKey,
"PeerId": peerId, "peerID": peerID,
"Timestamp": timeStamp, "timestamp": timeStamp,
"Error": err, "error": err,
}) })
} }
} }
@@ -684,9 +735,9 @@ func (ps *store) gc(cutoff time.Time) {
err = AsNil(err) err = AsNil(err)
if err != nil { if err != nil {
log.Error("storage: Redis: unable to delete peer", log.Fields{ log.Error("storage: Redis: unable to delete peer", log.Fields{
"InfoHashKey": infoHashKey, "infoHashKey": infoHashKey,
"PeerId": k, "peerID": k,
"Error": err, "error": err,
}) })
} else { } else {
removedPeerCount += count removedPeerCount += count
@@ -694,18 +745,18 @@ func (ps *store) gc(cutoff time.Time) {
} }
} else { } else {
log.Error("storage: Redis: unable to delete peers", log.Fields{ log.Error("storage: Redis: unable to delete peers", log.Fields{
"InfoHashKey": infoHashKey, "infoHashKey": infoHashKey,
"PeerIds": peersToRemove, "peerIds": peersToRemove,
"Error": err, "error": err,
}) })
} }
} }
if removedPeerCount > 0 { // DECR seeder/leecher counter if removedPeerCount > 0 { // DECR seeder/leecher counter
if err = ps.DecrBy(context.Background(), cntKey, removedPeerCount).Err(); err != nil { if err = ps.DecrBy(context.Background(), cntKey, removedPeerCount).Err(); err != nil {
log.Error("storage: Redis: unable to decrement seeder/leecher peer count", log.Fields{ log.Error("storage: Redis: unable to decrement seeder/leecher peer count", log.Fields{
"InfoHashKey": infoHashKey, "infoHashKey": infoHashKey,
"CountKey": cntKey, "countKey": cntKey,
"Error": err, "error": err,
}) })
} }
} }
@@ -725,19 +776,19 @@ func (ps *store) gc(cutoff time.Time) {
}, infoHashKey)) }, infoHashKey))
if err != nil { if err != nil {
log.Error("storage: Redis: unable to clean info hash records", log.Fields{ log.Error("storage: Redis: unable to clean info hash records", log.Fields{
"InfoHashKey": infoHashKey, "infoHashKey": infoHashKey,
"Error": err, "error": err,
}) })
} }
} else { } else {
log.Error("storage: Redis: unable to fetch info hash peers", log.Fields{ log.Error("storage: Redis: unable to fetch info hash peers", log.Fields{
"InfoHashKey": infoHashKey, "infoHashKey": infoHashKey,
"Error": err, "error": err,
}) })
} }
} }
} else { } else {
log.Error("storage: Redis: unable to fetch info hash set", log.Fields{"HashSet": IHKey, "Error": err}) log.Error("storage: Redis: unable to fetch info hash set", log.Fields{"hashSet": IHKey, "error": err})
} }
} }
storage/redis/storage_test.go
+2 -14
View File
@@ -5,8 +5,6 @@ import (
"testing" "testing"
"time" "time"
"github.com/alicebob/miniredis"
s "github.com/sot-tech/mochi/storage" s "github.com/sot-tech/mochi/storage"
"github.com/sot-tech/mochi/storage/test" "github.com/sot-tech/mochi/storage/test"
) )
@@ -22,19 +20,9 @@ var cfg = Config{
func createNew() s.PeerStorage { func createNew() s.PeerStorage {
var ps s.PeerStorage var ps s.PeerStorage
var err error var err error
ps, err = New(cfg) ps, err = newStore(cfg)
if err != nil { if err != nil {
fmt.Println("unable to create real redis connection: ", err, " using simulator") panic(fmt.Sprint("Unable to create KeyDB connection: ", err, "\nThis driver needs real Redis instance"))
var rs *miniredis.Miniredis
rs, err = miniredis.Run()
if err != nil {
panic(err)
}
cfg.Addresses = []string{rs.Addr()}
ps, err = New(cfg)
}
if err != nil {
panic(err)
} }
return ps return ps
} }
storage/storage.go
+9 -9
View File
@@ -39,9 +39,9 @@ func (c Config) sanitizeGCConfig() (gcInterval, peerTTL time.Duration) {
if c.GarbageCollectionInterval <= 0 { if c.GarbageCollectionInterval <= 0 {
gcInterval = defaultGarbageCollectionInterval gcInterval = defaultGarbageCollectionInterval
log.Warn("falling back to default configuration", log.Fields{ log.Warn("falling back to default configuration", log.Fields{
"Name": "GarbageCollectionInterval", "name": "GarbageCollectionInterval",
"Provided": c.GarbageCollectionInterval, "provided": c.GarbageCollectionInterval,
"Default": defaultGarbageCollectionInterval, "default": defaultGarbageCollectionInterval,
}) })
} else { } else {
gcInterval = c.GarbageCollectionInterval gcInterval = c.GarbageCollectionInterval
@@ -49,9 +49,9 @@ func (c Config) sanitizeGCConfig() (gcInterval, peerTTL time.Duration) {
if c.PeerLifetime <= 0 { if c.PeerLifetime <= 0 {
peerTTL = defaultPeerLifetime peerTTL = defaultPeerLifetime
log.Warn("falling back to default configuration", log.Fields{ log.Warn("falling back to default configuration", log.Fields{
"Name": "PeerLifetime", "name": "PeerLifetime",
"Provided": c.PeerLifetime, "provided": c.PeerLifetime,
"Default": defaultPeerLifetime, "default": defaultPeerLifetime,
}) })
} else { } else {
peerTTL = c.PeerLifetime peerTTL = c.PeerLifetime
@@ -63,9 +63,9 @@ func (c Config) sanitizeStatisticsConfig() (statInterval time.Duration) {
if c.PrometheusReportingInterval < 0 { if c.PrometheusReportingInterval < 0 {
statInterval = defaultPrometheusReportingInterval statInterval = defaultPrometheusReportingInterval
log.Warn("falling back to default configuration", log.Fields{ log.Warn("falling back to default configuration", log.Fields{
"Name": "PrometheusReportingInterval", "name": "PrometheusReportingInterval",
"Provided": c.PrometheusReportingInterval, "provided": c.PrometheusReportingInterval,
"Default": defaultPrometheusReportingInterval, "default": defaultPrometheusReportingInterval,
}) })
} }
return return