Mirror/mochi
1
0
Fork 0
mirror of https://github.com/sot-tech/mochi.git synced 2026-07-05 08:08:12 -07:00
Code Issues Packages Projects Releases Wiki Activity

refactor code for context fallthrough

Browse Source
This commit is contained in:
Lawrence, Rendall
2022-11-01 17:02:24 +03:00
parent d6de38bdbd
commit d8d6f2cf4b
22 changed files with 312 additions and 493 deletions
Download Patch File Download Diff File Expand all files Collapse all files
storage/keydb/storage.go
+1 -27
View File
@@ -12,14 +12,12 @@ package keydb
import (
"context"
"errors"
"time"
"github.com/go-redis/redis/v8"
"github.com/sot-tech/mochi/bittorrent"
"github.com/sot-tech/mochi/pkg/conf"
"github.com/sot-tech/mochi/pkg/log"
"github.com/sot-tech/mochi/pkg/stop"
"github.com/sot-tech/mochi/storage"
r "github.com/sot-tech/mochi/storage/redis"
)
@@ -79,10 +77,7 @@ func newStore(cfg r.Config) (*store, error) {
var st *store
if err == nil {
st = &store{
Connection: rs,
peerTTL: uint(cfg.PeerLifetime.Seconds()),
}
st = &store{Connection: rs, peerTTL: uint(cfg.PeerLifetime.Seconds())}
}
return st, err
@@ -177,24 +172,3 @@ func (s *store) ScrapeSwarm(ctx context.Context, ih bittorrent.InfoHash) (uint32
Msg("scrape swarm")
return s.ScrapeIH(ctx, ih, s.SCard)
}
func (*store) GCAware() bool {
return false
}
func (*store) ScheduleGC(_, _ time.Duration) {}
func (*store) StatisticsAware() bool {
return false
}
func (*store) ScheduleStatisticsCollection(_ time.Duration) {}
func (s *store) Stop() stop.Result {
c := make(stop.Channel)
if s.UniversalClient != nil {
c.Done(s.UniversalClient.Close())
s.UniversalClient = nil
}
return c.Result()
}
storage/memory/storage.go
+10 -23
View File
@@ -14,7 +14,6 @@ import (
"github.com/sot-tech/mochi/pkg/conf"
"github.com/sot-tech/mochi/pkg/log"
"github.com/sot-tech/mochi/pkg/metrics"
"github.com/sot-tech/mochi/pkg/stop"
"github.com/sot-tech/mochi/pkg/timecache"
"github.com/sot-tech/mochi/storage"
)
@@ -96,8 +95,9 @@ type peerStore struct {
cfg Config
shards []*peerShard
closed chan struct{}
wg sync.WaitGroup
closed chan struct{}
wg sync.WaitGroup
onceCloser sync.Once
}
var _ storage.PeerStorage = &peerStore{}
@@ -468,17 +468,9 @@ func (ds *dataStore) Delete(_ context.Context, ctx string, keys ...string) error
return nil
}
func (*dataStore) Preservable() bool {
return false
}
func (*dataStore) Preservable() bool { return false }
func (*peerStore) GCAware() bool {
return true
}
func (*peerStore) StatisticsAware() bool {
return true
}
func (ds *dataStore) Close() error { return nil }
// GC deletes all Peers from the PeerStorage which are older than the
// cutoff time.
@@ -542,12 +534,9 @@ func (*peerStore) Ping(context.Context) error {
return nil
}
func (ps *peerStore) Stop() stop.Result {
c := make(stop.Channel)
go func() {
if ps.closed != nil {
close(ps.closed)
}
func (ps *peerStore) Close() error {
ps.onceCloser.Do(func() {
close(ps.closed)
ps.wg.Wait()
// Explicitly deallocate our storage.
@@ -556,9 +545,7 @@ func (ps *peerStore) Stop() stop.Result {
shards[i] = &peerShard{swarms: make(map[bittorrent.InfoHash]swarm)}
}
ps.shards = shards
})
c.Done()
}()
return c.Result()
return nil
}
storage/pg/storage.go
+23 -28
View File
@@ -20,7 +20,6 @@ import (
"github.com/sot-tech/mochi/pkg/conf"
"github.com/sot-tech/mochi/pkg/log"
"github.com/sot-tech/mochi/pkg/metrics"
"github.com/sot-tech/mochi/pkg/stop"
"github.com/sot-tech/mochi/pkg/timecache"
"github.com/sot-tech/mochi/storage"
)
@@ -76,7 +75,13 @@ func newStore(cfg Config) (storage.PeerStorage, error) {
return nil, err
}
return &store{Config: cfg, Pool: con, wg: sync.WaitGroup{}, closed: make(chan any)}, nil
return &store{
Config: cfg,
Pool: con,
wg: sync.WaitGroup{},
closed: make(chan any),
onceCloser: sync.Once{},
}, nil
}
type peerQueryConf struct {
@@ -215,8 +220,9 @@ func (cfg Config) Validate() (Config, error) {
type store struct {
Config
*pgxpool.Pool
wg sync.WaitGroup
closed chan any
wg sync.WaitGroup
closed chan any
onceCloser sync.Once
}
func (s *store) txBatch(ctx context.Context, batch *pgx.Batch) (err error) {
@@ -242,7 +248,7 @@ func (s *store) Put(ctx context.Context, storeCtx string, values ...storage.Entr
default:
var batch pgx.Batch
for _, v := range values {
batch.Queue(s.Data.AddQuery, pgx.NamedArgs{pCtx: ctx, pKey: []byte(v.Key), pValue: v.Value})
batch.Queue(s.Data.AddQuery, pgx.NamedArgs{pCtx: storeCtx, pKey: []byte(v.Key), pValue: v.Value})
}
err = s.txBatch(ctx, &batch)
}
@@ -281,11 +287,10 @@ func (s *store) Preservable() bool {
return true
}
func (s *store) GCAware() bool {
return len(s.GCQuery) > 0
}
func (s *store) ScheduleGC(gcInterval, peerLifeTime time.Duration) {
if len(s.GCQuery) == 0 {
return
}
s.wg.Add(1)
go func() {
defer s.wg.Done()
@@ -311,11 +316,10 @@ func (s *store) ScheduleGC(gcInterval, peerLifeTime time.Duration) {
}()
}
func (s *store) StatisticsAware() bool {
return len(s.InfoHashCountQuery) > 0
}
func (s *store) ScheduleStatisticsCollection(reportInterval time.Duration) {
if len(s.InfoHashCountQuery) == 0 {
return
}
s.wg.Add(1)
go func() {
defer s.wg.Done()
@@ -357,9 +361,7 @@ func (s *store) putPeer(ctx context.Context, ih bittorrent.InfoHash, peer bittor
pPort: peer.Port(),
pSeeder: seeder,
pV6: peer.Addr().Is6(),
}
if s.GCAware() {
args[pCreated] = timecache.Now()
pCreated: timecache.Now(),
}
_, err = s.Exec(ctx, s.Peer.AddQuery, args)
return
@@ -576,19 +578,12 @@ func (s *store) Ping(ctx context.Context) error {
return err
}
func (s *store) Stop() stop.Result {
c := make(stop.Channel)
func (s *store) Close() error {
go func() {
if s.closed != nil {
close(s.closed)
}
close(s.closed)
s.wg.Wait()
if s.Pool != nil {
logger.Info().Msg("pg exiting. mochi does not clear data in database when exiting.")
s.Close()
s.Pool = nil
}
c.Done()
logger.Info().Msg("pg exiting. mochi does not clear data in database when exiting.")
s.Pool.Close()
}()
return c.Result()
return nil
}
storage/redis/storage.go
+11 -32
View File
@@ -36,7 +36,6 @@ import (
"github.com/sot-tech/mochi/pkg/conf"
"github.com/sot-tech/mochi/pkg/log"
"github.com/sot-tech/mochi/pkg/metrics"
"github.com/sot-tech/mochi/pkg/stop"
"github.com/sot-tech/mochi/pkg/timecache"
"github.com/sot-tech/mochi/storage"
)
@@ -100,11 +99,7 @@ func newStore(cfg Config) (*store, error) {
return nil, err
}
return &store{
Connection: rs,
closed: make(chan any),
wg: sync.WaitGroup{},
}, nil
return &store{Connection: rs, closed: make(chan any)}, nil
}
// Config holds the configuration of a redis PeerStorage.
@@ -287,8 +282,9 @@ type Connection struct {
type store struct {
Connection
closed chan any
wg sync.WaitGroup
closed chan any
wg sync.WaitGroup
onceCloser sync.Once
}
func (ps *store) count(key string, getLength bool) (n uint64) {
@@ -611,14 +607,6 @@ func (*Connection) Preservable() bool {
return true
}
func (*store) GCAware() bool {
return true
}
func (*store) StatisticsAware() bool {
return true
}
// Ping sends `PING` request to Redis server
func (ps *Connection) Ping(ctx context.Context) error {
return ps.UniversalClient.Ping(ctx).Err()
@@ -769,21 +757,12 @@ func (ps *store) gc(cutoff time.Time) {
}
}
func (ps *store) Stop() stop.Result {
c := make(stop.Channel)
go func() {
if ps.closed != nil {
close(ps.closed)
}
func (ps *store) Close() (err error) {
ps.onceCloser.Do(func() {
close(ps.closed)
ps.wg.Wait()
var err error
if ps.UniversalClient != nil {
logger.Info().Msg("redis exiting. mochi does not clear data in redis when exiting. mochi keys have prefix " + PrefixKey)
err = ps.UniversalClient.Close()
ps.UniversalClient = nil
}
c.Done(err)
}()
return c.Result()
logger.Info().Msg("redis exiting. mochi does not clear data in redis when exiting. mochi keys have prefix " + PrefixKey)
err = ps.UniversalClient.Close()
})
return
}
storage/storage.go
+14 -17
View File
@@ -5,13 +5,13 @@ package storage
import (
"context"
"fmt"
"io"
"sync"
"time"
"github.com/sot-tech/mochi/bittorrent"
"github.com/sot-tech/mochi/pkg/conf"
"github.com/sot-tech/mochi/pkg/log"
"github.com/sot-tech/mochi/pkg/stop"
)
const (
@@ -93,6 +93,7 @@ var ErrResourceDoesNotExist = bittorrent.ClientError("resource does not exist")
// DataStorage is the interface, used for implementing store for arbitrary data
type DataStorage interface {
io.Closer
// Put used to place arbitrary k-v data with specified context
// into storage. storeCtx parameter used to group data
// (i.e. data only for specific middleware module: hash key, table name etc...)
@@ -189,27 +190,23 @@ type PeerStorage interface {
// Ping used for checks if storage is alive
// (connection could be established, enough space etc.)
Ping(ctx context.Context) error
}
// GCAware marks that this storage supports periodic
// peers collection
GCAware() bool
// GarbageCollector marks that this storage supports periodic
// stale peers collection
type GarbageCollector interface {
// ScheduleGC used to delete stale data, such as timed out seeders/leechers.
// Note: implementation must create subroutine by itself
ScheduleGC(gcInterval, peerLifeTime time.Duration)
}
// StatisticsAware marks that this storage supports periodic
// statistics collection
StatisticsAware() bool
// StatisticsCollector marks that this storage supports periodic
// statistics collection
type StatisticsCollector interface {
// ScheduleStatisticsCollection used to receive statistics information about hashes,
// seeders and leechers count.
// Note: implementation must create subroutine by itself
ScheduleStatisticsCollection(reportInterval time.Duration)
// Stopper is an interface that expects a Stop method to stop the PeerStorage.
// For more details see the documentation in the stop package.
stop.Stopper
}
// RegisterDriver makes a Driver available by the provided name.
@@ -256,27 +253,27 @@ func NewStorage(cfg conf.NamedMapConfig) (ps PeerStorage, err error) {
return
}
if gc := ps.GCAware(); gc {
if gc, isOk := ps.(GarbageCollector); isOk {
gcInterval, peerTTL := c.sanitizeGCConfig()
logger.Info().
Str("name", cfg.Name).
Dur("gcInterval", gcInterval).
Dur("peerTTL", peerTTL).
Msg("scheduling GC")
ps.ScheduleGC(gcInterval, peerTTL)
gc.ScheduleGC(gcInterval, peerTTL)
} else {
logger.Debug().
Str("name", cfg.Name).
Msg("storage does not support GC")
}
if st := ps.StatisticsAware(); st {
if st, isOk := ps.(StatisticsCollector); isOk {
if statInterval := c.sanitizeStatisticsConfig(); statInterval > 0 {
logger.Info().
Str("name", cfg.Name).
Dur("statInterval", statInterval).
Msg("scheduling statistics collection")
ps.ScheduleStatisticsCollection(statInterval)
st.ScheduleStatisticsCollection(statInterval)
} else {
logger.Info().Str("name", cfg.Name).Msg("statistics collection disabled because of zero reporting interval")
}
storage/test/storage_bench.go
+43 -43
View File
@@ -3,6 +3,7 @@
package test
import (
"context"
"math/rand"
"net"
"net/netip"
@@ -100,9 +101,8 @@ func (bh *benchHolder) runBenchmark(b *testing.B, parallel bool, sf benchSetupFu
}
b.StopTimer()
errChan := ps.Stop()
for err := range errChan {
b.Fatal(err)
if err := ps.Close(); err != nil {
b.Fatal()
}
}
@@ -125,7 +125,7 @@ func (bh *benchHolder) Nop(b *testing.B) {
// Put can run in parallel.
func (bh *benchHolder) Put(b *testing.B) {
bh.runBenchmark(b, true, nil, func(i int, ps storage.PeerStorage, bd *benchData) error {
return ps.PutSeeder(nil, bd.infoHashes[0], bd.peers[0])
return ps.PutSeeder(context.TODO(), bd.infoHashes[0], bd.peers[0])
})
}
@@ -135,7 +135,7 @@ func (bh *benchHolder) Put(b *testing.B) {
// Put1k can run in parallel.
func (bh *benchHolder) Put1k(b *testing.B) {
bh.runBenchmark(b, true, nil, func(i int, ps storage.PeerStorage, bd *benchData) error {
return ps.PutSeeder(nil, bd.infoHashes[0], bd.peers[i%peersCount])
return ps.PutSeeder(context.TODO(), bd.infoHashes[0], bd.peers[i%peersCount])
})
}
@@ -145,7 +145,7 @@ func (bh *benchHolder) Put1k(b *testing.B) {
// Put1kInfoHash can run in parallel.
func (bh *benchHolder) Put1kInfoHash(b *testing.B) {
bh.runBenchmark(b, true, nil, func(i int, ps storage.PeerStorage, bd *benchData) error {
return ps.PutSeeder(nil, bd.infoHashes[i%ihCount], bd.peers[0])
return ps.PutSeeder(context.TODO(), bd.infoHashes[i%ihCount], bd.peers[0])
})
}
@@ -155,7 +155,7 @@ func (bh *benchHolder) Put1kInfoHash(b *testing.B) {
// Put1kInfoHash1k can run in parallel.
func (bh *benchHolder) Put1kInfoHash1k(b *testing.B) {
bh.runBenchmark(b, true, nil, func(i int, ps storage.PeerStorage, bd *benchData) error {
err := ps.PutSeeder(nil, bd.infoHashes[i%ihCount], bd.peers[(i*3)%peersCount])
err := ps.PutSeeder(context.TODO(), bd.infoHashes[i%ihCount], bd.peers[(i*3)%peersCount])
return err
})
}
@@ -166,11 +166,11 @@ func (bh *benchHolder) Put1kInfoHash1k(b *testing.B) {
// PutDelete can not run in parallel.
func (bh *benchHolder) PutDelete(b *testing.B) {
bh.runBenchmark(b, false, nil, func(i int, ps storage.PeerStorage, bd *benchData) error {
err := ps.PutSeeder(nil, bd.infoHashes[0], bd.peers[0])
err := ps.PutSeeder(context.TODO(), bd.infoHashes[0], bd.peers[0])
if err != nil {
return err
}
return ps.DeleteSeeder(nil, bd.infoHashes[0], bd.peers[0])
return ps.DeleteSeeder(context.TODO(), bd.infoHashes[0], bd.peers[0])
})
}
@@ -180,11 +180,11 @@ func (bh *benchHolder) PutDelete(b *testing.B) {
// PutDelete1k can not run in parallel.
func (bh *benchHolder) PutDelete1k(b *testing.B) {
bh.runBenchmark(b, false, nil, func(i int, ps storage.PeerStorage, bd *benchData) error {
err := ps.PutSeeder(nil, bd.infoHashes[0], bd.peers[i%peersCount])
err := ps.PutSeeder(context.TODO(), bd.infoHashes[0], bd.peers[i%peersCount])
if err != nil {
return err
}
return ps.DeleteSeeder(nil, bd.infoHashes[0], bd.peers[i%peersCount])
return ps.DeleteSeeder(context.TODO(), bd.infoHashes[0], bd.peers[i%peersCount])
})
}
@@ -194,11 +194,11 @@ func (bh *benchHolder) PutDelete1k(b *testing.B) {
// PutDelete1kInfoHash can not run in parallel.
func (bh *benchHolder) PutDelete1kInfoHash(b *testing.B) {
bh.runBenchmark(b, false, nil, func(i int, ps storage.PeerStorage, bd *benchData) error {
err := ps.PutSeeder(nil, bd.infoHashes[i%ihCount], bd.peers[0])
err := ps.PutSeeder(context.TODO(), bd.infoHashes[i%ihCount], bd.peers[0])
if err != nil {
return err
}
return ps.DeleteSeeder(nil, bd.infoHashes[i%ihCount], bd.peers[0])
return ps.DeleteSeeder(context.TODO(), bd.infoHashes[i%ihCount], bd.peers[0])
})
}
@@ -208,11 +208,11 @@ func (bh *benchHolder) PutDelete1kInfoHash(b *testing.B) {
// PutDelete1kInfoHash1k can not run in parallel.
func (bh *benchHolder) PutDelete1kInfoHash1k(b *testing.B) {
bh.runBenchmark(b, false, nil, func(i int, ps storage.PeerStorage, bd *benchData) error {
err := ps.PutSeeder(nil, bd.infoHashes[i%ihCount], bd.peers[(i*3)%peersCount])
err := ps.PutSeeder(context.TODO(), bd.infoHashes[i%ihCount], bd.peers[(i*3)%peersCount])
if err != nil {
return err
}
err = ps.DeleteSeeder(nil, bd.infoHashes[i%ihCount], bd.peers[(i*3)%peersCount])
err = ps.DeleteSeeder(context.TODO(), bd.infoHashes[i%ihCount], bd.peers[(i*3)%peersCount])
return err
})
}
@@ -223,7 +223,7 @@ func (bh *benchHolder) PutDelete1kInfoHash1k(b *testing.B) {
// DeleteNonexist can run in parallel.
func (bh *benchHolder) DeleteNonexist(b *testing.B) {
bh.runBenchmark(b, true, nil, func(i int, ps storage.PeerStorage, bd *benchData) error {
_ = ps.DeleteSeeder(nil, bd.infoHashes[0], bd.peers[0])
_ = ps.DeleteSeeder(context.TODO(), bd.infoHashes[0], bd.peers[0])
return nil
})
}
@@ -234,7 +234,7 @@ func (bh *benchHolder) DeleteNonexist(b *testing.B) {
// DeleteNonexist can run in parallel.
func (bh *benchHolder) DeleteNonexist1k(b *testing.B) {
bh.runBenchmark(b, true, nil, func(i int, ps storage.PeerStorage, bd *benchData) error {
_ = ps.DeleteSeeder(nil, bd.infoHashes[0], bd.peers[i%peersCount])
_ = ps.DeleteSeeder(context.TODO(), bd.infoHashes[0], bd.peers[i%peersCount])
return nil
})
}
@@ -245,7 +245,7 @@ func (bh *benchHolder) DeleteNonexist1k(b *testing.B) {
// DeleteNonexist1kInfoHash can run in parallel.
func (bh *benchHolder) DeleteNonexist1kInfoHash(b *testing.B) {
bh.runBenchmark(b, true, nil, func(i int, ps storage.PeerStorage, bd *benchData) error {
_ = ps.DeleteSeeder(nil, bd.infoHashes[i%ihCount], bd.peers[0])
_ = ps.DeleteSeeder(context.TODO(), bd.infoHashes[i%ihCount], bd.peers[0])
return nil
})
}
@@ -256,7 +256,7 @@ func (bh *benchHolder) DeleteNonexist1kInfoHash(b *testing.B) {
// DeleteNonexist1kInfoHash1k can run in parallel.
func (bh *benchHolder) DeleteNonexist1kInfoHash1k(b *testing.B) {
bh.runBenchmark(b, true, nil, func(i int, ps storage.PeerStorage, bd *benchData) error {
_ = ps.DeleteSeeder(nil, bd.infoHashes[i%ihCount], bd.peers[(i*3)%peersCount])
_ = ps.DeleteSeeder(context.TODO(), bd.infoHashes[i%ihCount], bd.peers[(i*3)%peersCount])
return nil
})
}
@@ -267,7 +267,7 @@ func (bh *benchHolder) DeleteNonexist1kInfoHash1k(b *testing.B) {
// GradNonexist can run in parallel.
func (bh *benchHolder) GradNonexist(b *testing.B) {
bh.runBenchmark(b, true, nil, func(i int, ps storage.PeerStorage, bd *benchData) error {
_ = ps.GraduateLeecher(nil, bd.infoHashes[0], bd.peers[0])
_ = ps.GraduateLeecher(context.TODO(), bd.infoHashes[0], bd.peers[0])
return nil
})
}
@@ -278,7 +278,7 @@ func (bh *benchHolder) GradNonexist(b *testing.B) {
// GradNonexist1k can run in parallel.
func (bh *benchHolder) GradNonexist1k(b *testing.B) {
bh.runBenchmark(b, true, nil, func(i int, ps storage.PeerStorage, bd *benchData) error {
_ = ps.GraduateLeecher(nil, bd.infoHashes[0], bd.peers[i%peersCount])
_ = ps.GraduateLeecher(context.TODO(), bd.infoHashes[0], bd.peers[i%peersCount])
return nil
})
}
@@ -289,7 +289,7 @@ func (bh *benchHolder) GradNonexist1k(b *testing.B) {
// GradNonexist1kInfoHash can run in parallel.
func (bh *benchHolder) GradNonexist1kInfoHash(b *testing.B) {
bh.runBenchmark(b, true, nil, func(i int, ps storage.PeerStorage, bd *benchData) error {
_ = ps.GraduateLeecher(nil, bd.infoHashes[i%ihCount], bd.peers[0])
_ = ps.GraduateLeecher(context.TODO(), bd.infoHashes[i%ihCount], bd.peers[0])
return nil
})
}
@@ -301,7 +301,7 @@ func (bh *benchHolder) GradNonexist1kInfoHash(b *testing.B) {
// GradNonexist1kInfoHash1k can run in parallel.
func (bh *benchHolder) GradNonexist1kInfoHash1k(b *testing.B) {
bh.runBenchmark(b, true, nil, func(i int, ps storage.PeerStorage, bd *benchData) error {
_ = ps.GraduateLeecher(nil, bd.infoHashes[i%ihCount], bd.peers[(i*3)%peersCount])
_ = ps.GraduateLeecher(context.TODO(), bd.infoHashes[i%ihCount], bd.peers[(i*3)%peersCount])
return nil
})
}
@@ -313,15 +313,15 @@ func (bh *benchHolder) GradNonexist1kInfoHash1k(b *testing.B) {
// PutGradDelete can not run in parallel.
func (bh *benchHolder) PutGradDelete(b *testing.B) {
bh.runBenchmark(b, false, nil, func(i int, ps storage.PeerStorage, bd *benchData) error {
err := ps.PutLeecher(nil, bd.infoHashes[0], bd.peers[0])
err := ps.PutLeecher(context.TODO(), bd.infoHashes[0], bd.peers[0])
if err != nil {
return err
}
err = ps.GraduateLeecher(nil, bd.infoHashes[0], bd.peers[0])
err = ps.GraduateLeecher(context.TODO(), bd.infoHashes[0], bd.peers[0])
if err != nil {
return err
}
return ps.DeleteSeeder(nil, bd.infoHashes[0], bd.peers[0])
return ps.DeleteSeeder(context.TODO(), bd.infoHashes[0], bd.peers[0])
})
}
@@ -330,15 +330,15 @@ func (bh *benchHolder) PutGradDelete(b *testing.B) {
// PutGradDelete1k can not run in parallel.
func (bh *benchHolder) PutGradDelete1k(b *testing.B) {
bh.runBenchmark(b, false, nil, func(i int, ps storage.PeerStorage, bd *benchData) error {
err := ps.PutLeecher(nil, bd.infoHashes[0], bd.peers[i%peersCount])
err := ps.PutLeecher(context.TODO(), bd.infoHashes[0], bd.peers[i%peersCount])
if err != nil {
return err
}
err = ps.GraduateLeecher(nil, bd.infoHashes[0], bd.peers[i%peersCount])
err = ps.GraduateLeecher(context.TODO(), bd.infoHashes[0], bd.peers[i%peersCount])
if err != nil {
return err
}
return ps.DeleteSeeder(nil, bd.infoHashes[0], bd.peers[i%peersCount])
return ps.DeleteSeeder(context.TODO(), bd.infoHashes[0], bd.peers[i%peersCount])
})
}
@@ -348,15 +348,15 @@ func (bh *benchHolder) PutGradDelete1k(b *testing.B) {
// PutGradDelete1kInfoHash can not run in parallel.
func (bh *benchHolder) PutGradDelete1kInfoHash(b *testing.B) {
bh.runBenchmark(b, false, nil, func(i int, ps storage.PeerStorage, bd *benchData) error {
err := ps.PutLeecher(nil, bd.infoHashes[i%ihCount], bd.peers[0])
err := ps.PutLeecher(context.TODO(), bd.infoHashes[i%ihCount], bd.peers[0])
if err != nil {
return err
}
err = ps.GraduateLeecher(nil, bd.infoHashes[i%ihCount], bd.peers[0])
err = ps.GraduateLeecher(context.TODO(), bd.infoHashes[i%ihCount], bd.peers[0])
if err != nil {
return err
}
return ps.DeleteSeeder(nil, bd.infoHashes[i%ihCount], bd.peers[0])
return ps.DeleteSeeder(context.TODO(), bd.infoHashes[i%ihCount], bd.peers[0])
})
}
@@ -366,15 +366,15 @@ func (bh *benchHolder) PutGradDelete1kInfoHash(b *testing.B) {
// PutGradDelete1kInfoHash can not run in parallel.
func (bh *benchHolder) PutGradDelete1kInfoHash1k(b *testing.B) {
bh.runBenchmark(b, false, nil, func(i int, ps storage.PeerStorage, bd *benchData) error {
err := ps.PutLeecher(nil, bd.infoHashes[i%ihCount], bd.peers[(i*3)%peersCount])
err := ps.PutLeecher(context.TODO(), bd.infoHashes[i%ihCount], bd.peers[(i*3)%peersCount])
if err != nil {
return err
}
err = ps.GraduateLeecher(nil, bd.infoHashes[i%ihCount], bd.peers[(i*3)%peersCount])
err = ps.GraduateLeecher(context.TODO(), bd.infoHashes[i%ihCount], bd.peers[(i*3)%peersCount])
if err != nil {
return err
}
err = ps.DeleteSeeder(nil, bd.infoHashes[i%ihCount], bd.peers[(i*3)%peersCount])
err = ps.DeleteSeeder(context.TODO(), bd.infoHashes[i%ihCount], bd.peers[(i*3)%peersCount])
return err
})
}
@@ -385,9 +385,9 @@ func putPeers(ps storage.PeerStorage, bd *benchData) error {
for i, peer := range bd.peers {
var err error
if i < l/2 {
err = ps.PutLeecher(nil, ih, peer)
err = ps.PutLeecher(context.TODO(), ih, peer)
} else {
err = ps.PutSeeder(nil, ih, peer)
err = ps.PutSeeder(context.TODO(), ih, peer)
}
if err != nil {
return err
@@ -404,7 +404,7 @@ func putPeers(ps storage.PeerStorage, bd *benchData) error {
// AnnounceLeecher can run in parallel.
func (bh *benchHolder) AnnounceLeecher(b *testing.B) {
bh.runBenchmark(b, true, putPeers, func(i int, ps storage.PeerStorage, bd *benchData) error {
_, err := ps.AnnouncePeers(nil, bd.infoHashes[0], false, 50, bd.peers[0].Addr().Is6())
_, err := ps.AnnouncePeers(context.TODO(), bd.infoHashes[0], false, 50, bd.peers[0].Addr().Is6())
return err
})
}
@@ -415,7 +415,7 @@ func (bh *benchHolder) AnnounceLeecher(b *testing.B) {
// AnnounceLeecher1kInfoHash can run in parallel.
func (bh *benchHolder) AnnounceLeecher1kInfoHash(b *testing.B) {
bh.runBenchmark(b, true, putPeers, func(i int, ps storage.PeerStorage, bd *benchData) error {
_, err := ps.AnnouncePeers(nil, bd.infoHashes[i%ihCount], false, 50, bd.peers[0].Addr().Is6())
_, err := ps.AnnouncePeers(context.TODO(), bd.infoHashes[i%ihCount], false, 50, bd.peers[0].Addr().Is6())
return err
})
}
@@ -426,7 +426,7 @@ func (bh *benchHolder) AnnounceLeecher1kInfoHash(b *testing.B) {
// AnnounceSeeder can run in parallel.
func (bh *benchHolder) AnnounceSeeder(b *testing.B) {
bh.runBenchmark(b, true, putPeers, func(i int, ps storage.PeerStorage, bd *benchData) error {
_, err := ps.AnnouncePeers(nil, bd.infoHashes[0], true, 50, bd.peers[0].Addr().Is6())
_, err := ps.AnnouncePeers(context.TODO(), bd.infoHashes[0], true, 50, bd.peers[0].Addr().Is6())
return err
})
}
@@ -437,7 +437,7 @@ func (bh *benchHolder) AnnounceSeeder(b *testing.B) {
// AnnounceSeeder1kInfoHash can run in parallel.
func (bh *benchHolder) AnnounceSeeder1kInfoHash(b *testing.B) {
bh.runBenchmark(b, true, putPeers, func(i int, ps storage.PeerStorage, bd *benchData) error {
_, err := ps.AnnouncePeers(nil, bd.infoHashes[i%ihCount], true, 50, bd.peers[0].Addr().Is6())
_, err := ps.AnnouncePeers(context.TODO(), bd.infoHashes[i%ihCount], true, 50, bd.peers[0].Addr().Is6())
return err
})
}
@@ -448,7 +448,7 @@ func (bh *benchHolder) AnnounceSeeder1kInfoHash(b *testing.B) {
// ScrapeSwarm can run in parallel.
func (bh *benchHolder) ScrapeSwarm(b *testing.B) {
bh.runBenchmark(b, true, putPeers, func(i int, ps storage.PeerStorage, bd *benchData) error {
ps.ScrapeSwarm(nil, bd.infoHashes[0])
ps.ScrapeSwarm(context.TODO(), bd.infoHashes[0])
return nil
})
}
@@ -458,7 +458,7 @@ func (bh *benchHolder) ScrapeSwarm(b *testing.B) {
// ScrapeSwarm1kInfoHash can run in parallel.
func (bh *benchHolder) ScrapeSwarm1kInfoHash(b *testing.B) {
bh.runBenchmark(b, true, putPeers, func(i int, ps storage.PeerStorage, bd *benchData) error {
ps.ScrapeSwarm(nil, bd.infoHashes[i%ihCount])
ps.ScrapeSwarm(context.TODO(), bd.infoHashes[i%ihCount])
return nil
})
}
storage/test/storage_test_base.go
+40 -39
View File
@@ -3,6 +3,7 @@
package test
import (
"context"
"errors"
"testing"
@@ -40,7 +41,7 @@ type hashPeer struct {
func (th *testHolder) DeleteSeeder(t *testing.T) {
for _, c := range testData {
err := th.st.DeleteSeeder(nil, c.ih, c.peer)
err := th.st.DeleteSeeder(context.TODO(), c.ih, c.peer)
if errors.Is(err, storage.ErrResourceDoesNotExist) {
err = nil
}
@@ -54,14 +55,14 @@ func (th *testHolder) PutLeecher(t *testing.T) {
if c.peer.Addr().Is6() {
peer = v6Peer
}
err := th.st.PutLeecher(nil, c.ih, peer)
err := th.st.PutLeecher(context.TODO(), c.ih, peer)
require.Nil(t, err)
}
}
func (th *testHolder) DeleteLeecher(t *testing.T) {
for _, c := range testData {
err := th.st.DeleteLeecher(nil, c.ih, c.peer)
err := th.st.DeleteLeecher(context.TODO(), c.ih, c.peer)
if errors.Is(err, storage.ErrResourceDoesNotExist) {
err = nil
}
@@ -71,7 +72,7 @@ func (th *testHolder) DeleteLeecher(t *testing.T) {
func (th *testHolder) AnnouncePeers(t *testing.T) {
for _, c := range testData {
_, err := th.st.AnnouncePeers(nil, c.ih, false, 50, c.peer.Addr().Is6())
_, err := th.st.AnnouncePeers(context.TODO(), c.ih, false, 50, c.peer.Addr().Is6())
if errors.Is(err, storage.ErrResourceDoesNotExist) {
err = nil
}
@@ -81,7 +82,7 @@ func (th *testHolder) AnnouncePeers(t *testing.T) {
func (th *testHolder) ScrapeSwarm(t *testing.T) {
for _, c := range testData {
l, s, n := th.st.ScrapeSwarm(nil, c.ih)
l, s, n := th.st.ScrapeSwarm(context.TODO(), c.ih)
require.Equal(t, uint32(0), s)
require.Equal(t, uint32(0), l)
require.Equal(t, uint32(0), n)
@@ -91,26 +92,26 @@ func (th *testHolder) ScrapeSwarm(t *testing.T) {
func (th *testHolder) LeecherPutAnnounceDeleteAnnounce(t *testing.T) {
for _, c := range testData {
isV6 := c.peer.Addr().Is6()
err := th.st.PutLeecher(nil, c.ih, c.peer)
err := th.st.PutLeecher(context.TODO(), c.ih, c.peer)
require.Nil(t, err)
peers, err := th.st.AnnouncePeers(nil, c.ih, true, 50, isV6)
peers, err := th.st.AnnouncePeers(context.TODO(), c.ih, true, 50, isV6)
require.Nil(t, err)
require.True(t, containsPeer(peers, c.peer))
// non-seeder announce should still return the leecher
peers, err = th.st.AnnouncePeers(nil, c.ih, false, 50, isV6)
peers, err = th.st.AnnouncePeers(context.TODO(), c.ih, false, 50, isV6)
require.Nil(t, err)
require.True(t, containsPeer(peers, c.peer))
l, s, _ := th.st.ScrapeSwarm(nil, c.ih)
l, s, _ := th.st.ScrapeSwarm(context.TODO(), c.ih)
require.Equal(t, uint32(2), l)
require.Equal(t, uint32(0), s)
err = th.st.DeleteLeecher(nil, c.ih, c.peer)
err = th.st.DeleteLeecher(context.TODO(), c.ih, c.peer)
require.Nil(t, err)
peers, err = th.st.AnnouncePeers(nil, c.ih, true, 50, isV6)
peers, err = th.st.AnnouncePeers(context.TODO(), c.ih, true, 50, isV6)
if errors.Is(err, storage.ErrResourceDoesNotExist) {
err = nil
}
@@ -122,22 +123,22 @@ func (th *testHolder) LeecherPutAnnounceDeleteAnnounce(t *testing.T) {
func (th *testHolder) SeederPutAnnounceDeleteAnnounce(t *testing.T) {
for _, c := range testData {
isV6 := c.peer.Addr().Is6()
err := th.st.PutSeeder(nil, c.ih, c.peer)
err := th.st.PutSeeder(context.TODO(), c.ih, c.peer)
require.Nil(t, err)
// Should be leecher to see the seeder
peers, err := th.st.AnnouncePeers(nil, c.ih, false, 50, isV6)
peers, err := th.st.AnnouncePeers(context.TODO(), c.ih, false, 50, isV6)
require.Nil(t, err)
require.True(t, containsPeer(peers, c.peer))
l, s, _ := th.st.ScrapeSwarm(nil, c.ih)
l, s, _ := th.st.ScrapeSwarm(context.TODO(), c.ih)
require.Equal(t, uint32(1), l)
require.Equal(t, uint32(1), s)
err = th.st.DeleteSeeder(nil, c.ih, c.peer)
err = th.st.DeleteSeeder(context.TODO(), c.ih, c.peer)
require.Nil(t, err)
peers, err = th.st.AnnouncePeers(nil, c.ih, false, 50, isV6)
peers, err = th.st.AnnouncePeers(context.TODO(), c.ih, false, 50, isV6)
if errors.Is(err, storage.ErrResourceDoesNotExist) {
err = nil
}
@@ -153,44 +154,44 @@ func (th *testHolder) LeecherPutGraduateAnnounceDeleteAnnounce(t *testing.T) {
if isV6 {
peer = v6Peer
}
err := th.st.PutLeecher(nil, c.ih, c.peer)
err := th.st.PutLeecher(context.TODO(), c.ih, c.peer)
require.Nil(t, err)
err = th.st.GraduateLeecher(nil, c.ih, c.peer)
err = th.st.GraduateLeecher(context.TODO(), c.ih, c.peer)
require.Nil(t, err)
// Has to be leecher to see the graduated seeder
peers, err := th.st.AnnouncePeers(nil, c.ih, false, 50, isV6)
peers, err := th.st.AnnouncePeers(context.TODO(), c.ih, false, 50, isV6)
require.Nil(t, err)
require.True(t, containsPeer(peers, c.peer))
// Deleting the Peer as a Leecher should have no effect
err = th.st.DeleteLeecher(nil, c.ih, c.peer)
err = th.st.DeleteLeecher(context.TODO(), c.ih, c.peer)
if errors.Is(err, storage.ErrResourceDoesNotExist) {
err = nil
}
require.Nil(t, err)
// Verify it's still there
peers, err = th.st.AnnouncePeers(nil, c.ih, false, 50, isV6)
peers, err = th.st.AnnouncePeers(context.TODO(), c.ih, false, 50, isV6)
require.Nil(t, err)
require.True(t, containsPeer(peers, c.peer))
// Clean up
err = th.st.DeleteLeecher(nil, c.ih, peer)
err = th.st.DeleteLeecher(context.TODO(), c.ih, peer)
require.Nil(t, err)
// Test ErrDNE for missing leecher
err = th.st.DeleteLeecher(nil, c.ih, peer)
err = th.st.DeleteLeecher(context.TODO(), c.ih, peer)
if errors.Is(err, storage.ErrResourceDoesNotExist) {
err = nil
}
require.Nil(t, err)
err = th.st.DeleteSeeder(nil, c.ih, c.peer)
err = th.st.DeleteSeeder(context.TODO(), c.ih, c.peer)
require.Nil(t, err)
err = th.st.DeleteSeeder(nil, c.ih, c.peer)
err = th.st.DeleteSeeder(context.TODO(), c.ih, c.peer)
if errors.Is(err, storage.ErrResourceDoesNotExist) {
err = nil
}
@@ -200,35 +201,35 @@ func (th *testHolder) LeecherPutGraduateAnnounceDeleteAnnounce(t *testing.T) {
func (th *testHolder) CustomPutContainsLoadDelete(t *testing.T) {
for _, c := range testData {
err := th.st.Put(nil, kvStoreCtx, storage.Entry{Key: c.peer.String(), Value: []byte(c.ih.RawString())})
err := th.st.Put(context.TODO(), kvStoreCtx, storage.Entry{Key: c.peer.String(), Value: []byte(c.ih.RawString())})
require.Nil(t, err)
// check if exist in ctx we put
contains, err := th.st.Contains(nil, kvStoreCtx, c.peer.String())
contains, err := th.st.Contains(context.TODO(), kvStoreCtx, c.peer.String())
require.Nil(t, err)
require.True(t, contains)
// check if not exist in another ctx
contains, err = th.st.Contains(nil, "", c.peer.String())
contains, err = th.st.Contains(context.TODO(), "", c.peer.String())
require.Nil(t, err)
require.False(t, contains)
// check value and type in ctx we put
out, err := th.st.Load(nil, kvStoreCtx, c.peer.String())
out, err := th.st.Load(context.TODO(), kvStoreCtx, c.peer.String())
require.Nil(t, err)
ih, err := bittorrent.NewInfoHash(out)
require.Nil(t, err)
require.Equal(t, c.ih, ih)
// check value is nil in another ctx
dummy, err := th.st.Load(nil, "", c.peer.String())
dummy, err := th.st.Load(context.TODO(), "", c.peer.String())
require.Nil(t, err)
require.Nil(t, dummy)
err = th.st.Delete(nil, kvStoreCtx, c.peer.String())
err = th.st.Delete(context.TODO(), kvStoreCtx, c.peer.String())
require.Nil(t, err)
contains, err = th.st.Contains(nil, "", c.peer.String())
contains, err = th.st.Contains(context.TODO(), "", c.peer.String())
require.Nil(t, err)
require.False(t, contains)
}
@@ -245,29 +246,29 @@ func (th *testHolder) CustomBulkPutContainsLoadDelete(t *testing.T) {
Value: []byte(c.ih.RawString()),
})
}
err := th.st.Put(nil, kvStoreCtx, pairs...)
err := th.st.Put(context.TODO(), kvStoreCtx, pairs...)
require.Nil(t, err)
// check if exist in ctx we put
for _, k := range keys {
contains, err := th.st.Contains(nil, kvStoreCtx, k)
contains, err := th.st.Contains(context.TODO(), kvStoreCtx, k)
require.Nil(t, err)
require.True(t, contains)
}
// check value and type in ctx we put
for _, p := range pairs {
out, _ := th.st.Load(nil, kvStoreCtx, p.Key)
out, _ := th.st.Load(context.TODO(), kvStoreCtx, p.Key)
ih, err := bittorrent.NewInfoHash(out)
require.Nil(t, err)
require.Equal(t, p.Value, []byte(ih.RawString()))
}
err = th.st.Delete(nil, kvStoreCtx, keys...)
err = th.st.Delete(context.TODO(), kvStoreCtx, keys...)
require.Nil(t, err)
for _, k := range keys {
contains, err := th.st.Contains(nil, kvStoreCtx, k)
contains, err := th.st.Contains(context.TODO(), kvStoreCtx, k)
require.Nil(t, err)
require.False(t, contains)
}
@@ -304,8 +305,8 @@ func RunTests(t *testing.T, p storage.PeerStorage) {
t.Run("CustomPutContainsLoadDelete", th.CustomPutContainsLoadDelete)
t.Run("CustomBulkPutContainsLoadDelete", th.CustomBulkPutContainsLoadDelete)
e := th.st.Stop()
require.Nil(t, <-e)
e := th.st.Close()
require.Nil(t, e)
}
func containsPeer(peers []bittorrent.Peer, p bittorrent.Peer) bool {