Mirror/mochi
1
0
Fork 0
mirror of https://github.com/sot-tech/mochi.git synced 2026-07-29 02:28:10 -07:00
Code Issues Packages Projects Releases Wiki Activity

Implement simple PGDC methods for storage

Browse Source 
* sanitize code a little
* move e2e build to 'e2e' tag
This commit is contained in:
Širhoe Biazhkovič
2021-11-22 19:33:52 +03:00
committed by Lawrence, Rendall
parent 566d99fcd7
commit beb4736b86
37 changed files with 483 additions and 317 deletions
Download Patch File Download Diff File Expand all files Collapse all files
middleware/clientapproval/client_id_test.go
+5 -1
View File
@@ -47,7 +47,11 @@ func TestClientID(t *testing.T) {
t.Run(tt.peerID, func(t *testing.T) {
var clientID ClientID
copy(clientID[:], tt.clientID)
parsedID := NewClientID(bittorrent.NewPeerID([]byte(tt.peerID)))
peerId, err := bittorrent.NewPeerID([]byte(tt.peerID))
if err != nil {
t.Error(err)
}
parsedID := NewClientID(peerId)
if parsedID != clientID {
t.Error("Incorrectly parsed peer ID", tt.peerID, "as", parsedID)
}
middleware/clientapproval/clientapproval_test.go
+2 -2
View File
@@ -59,8 +59,8 @@ func TestHandleAnnounce(t *testing.T) {
req := &bittorrent.AnnounceRequest{}
resp := &bittorrent.AnnounceResponse{}
peerid := bittorrent.NewPeerID([]byte(tt.peerID))
peerid, err := bittorrent.NewPeerID([]byte(tt.peerID))
require.Nil(t, err)
req.Peer.ID = peerid
nctx, err := h.HandleAnnounce(ctx, req, resp)
middleware/hooks.go
+3 -3
View File
@@ -28,7 +28,7 @@ type swarmInteractionHook struct {
store storage.Storage
}
func (h *swarmInteractionHook) HandleAnnounce(ctx context.Context, req *bittorrent.AnnounceRequest, resp *bittorrent.AnnounceResponse) (_ context.Context, err error) {
func (h *swarmInteractionHook) HandleAnnounce(ctx context.Context, req *bittorrent.AnnounceRequest, _ *bittorrent.AnnounceResponse) (_ context.Context, err error) {
if ctx.Value(SkipSwarmInteractionKey) != nil {
return ctx, nil
}
@@ -75,14 +75,14 @@ type skipResponseHook struct{}
// skip.
var SkipResponseHookKey = skipResponseHook{}
type scrapeAddressType struct{}
//type scrapeAddressType struct{}
// ScrapeIsIPv6Key is the key under which to store whether or not the
// address used to request a scrape was an IPv6 address.
// The value is expected to be of type bool.
// A missing value or a value that is not a bool for this key is equivalent to
// it being set to false.
var ScrapeIsIPv6Key = scrapeAddressType{}
//var ScrapeIsIPv6Key = scrapeAddressType{}
type responseHook struct {
store storage.Storage
middleware/jwt/jwt.go
+1 -1
View File
@@ -22,7 +22,7 @@ import (
"github.com/SermoDigital/jose/jws"
"github.com/SermoDigital/jose/jwt"
"github.com/mendsley/gojwk"
yaml "gopkg.in/yaml.v2"
"gopkg.in/yaml.v2"
"github.com/chihaya/chihaya/bittorrent"
"github.com/chihaya/chihaya/middleware"
middleware/logic_test.go
+2 -2
View File
@@ -15,11 +15,11 @@ import (
// benchmarks.
type nopHook struct{}
func (h *nopHook) HandleAnnounce(ctx context.Context, req *bittorrent.AnnounceRequest, resp *bittorrent.AnnounceResponse) (context.Context, error) {
func (h *nopHook) HandleAnnounce(ctx context.Context, _ *bittorrent.AnnounceRequest, _ *bittorrent.AnnounceResponse) (context.Context, error) {
return ctx, nil
}
func (h *nopHook) HandleScrape(ctx context.Context, req *bittorrent.ScrapeRequest, resp *bittorrent.ScrapeResponse) (context.Context, error) {
func (h *nopHook) HandleScrape(ctx context.Context, _ *bittorrent.ScrapeRequest, _ *bittorrent.ScrapeResponse) (context.Context, error) {
return ctx, nil
}
middleware/middleware.go
+1 -1
View File
@@ -7,7 +7,7 @@ import (
"github.com/chihaya/chihaya/storage"
"sync"
yaml "gopkg.in/yaml.v2"
"gopkg.in/yaml.v2"
)
var (
middleware/torrentapproval/container/container.go
+4 -2
View File
@@ -7,7 +7,9 @@ import (
"sync"
)
type Builder func ([]byte, storage.Storage) (Container, error)
const DefaultStorageCtxName = "MW_APPROVAL"
type Builder func([]byte, storage.Storage) (Container, error)
var (
buildersMU sync.Mutex
@@ -30,7 +32,7 @@ func Register(n string, c Builder) {
}
type Container interface {
Contains(bittorrent.InfoHash) bool
Approved(bittorrent.InfoHash) bool
}
func GetContainer(name string, confBytes []byte, storage storage.Storage) (Container, error) {
middleware/torrentapproval/container/directory/directory.go
+48 -23
View File
@@ -4,63 +4,88 @@
package directory
import (
"crypto/sha256"
"fmt"
"github.com/anacrolix/torrent/metainfo"
"github.com/anacrolix/torrent/util/dirwatch"
"github.com/chihaya/chihaya/bittorrent"
"github.com/chihaya/chihaya/middleware/torrentapproval/container"
"github.com/chihaya/chihaya/middleware/torrentapproval/container/list"
"github.com/chihaya/chihaya/pkg/log"
"github.com/chihaya/chihaya/pkg/stop"
"github.com/chihaya/chihaya/storage"
"gopkg.in/yaml.v2"
"sync"
)
const Name = "directory"
func init() {
container.Register("directory", build)
container.Register(Name, build)
}
type Config struct {
WhitelistPath string `yaml:"whitelist_path"`
BlacklistPath string `yaml:"blacklist_path"`
list.Config
Path string `yaml:"path"`
}
// TODO: change sync map to provided storage
func build(confBytes []byte, storage storage.Storage) (container.Container, error) {
func build(confBytes []byte, st storage.Storage) (container.Container, error) {
c := new(Config)
if err := yaml.Unmarshal(confBytes, c); err != nil {
return nil, fmt.Errorf("unable to deserialise configuration: %v", err)
}
if len(c.WhitelistPath) > 0 && len(c.BlacklistPath) > 0 {
return nil, fmt.Errorf("using both whitelist and blacklist is invalid")
}
var err error
lst := &directory{
d := &directory{
List: list.List{
Hashes: sync.Map{},
Invert: len(c.WhitelistPath) == 0,
Invert: c.Invert,
Storage: st,
StorageCtx: c.StorageCtx,
},
watcher: nil,
}
dir := c.WhitelistPath
if lst.Invert {
dir = c.BlacklistPath
}
//FIXME: implement V2 torrent add/delete
var w *dirwatch.Instance
if w, err = dirwatch.New(dir); err != nil {
if w, err = dirwatch.New(c.Path); err != nil {
return nil, fmt.Errorf("unable to initialize directory watch: %v", err)
}
lst.watcher = w
d.watcher = w
if len(d.StorageCtx) == 0 {
log.Info("Storage context not set, using default value: " + container.DefaultStorageCtxName)
d.StorageCtx = container.DefaultStorageCtxName
}
go func() {
for event := range lst.watcher.Events {
for event := range d.watcher.Events {
switch event.Change {
case dirwatch.Added:
lst.Hashes.Store(event.InfoHash, list.DUMMY)
data := make([]storage.Pair, 1, 2)
data[0] = storage.Pair{Left: event.InfoHash[:], Right: list.DUMMY}
if v2ih, err := v2InfoHash(event.TorrentFilePath); err == nil {
data = append(data, storage.Pair{Left: v2ih, Right: list.DUMMY})
} else {
log.Err(err)
}
d.Storage.BulkPut(c.StorageCtx, data...)
case dirwatch.Removed:
lst.Hashes.Delete(event.InfoHash)
data := make([]interface{}, 1, 2)
data[0] = event.InfoHash[:]
if v2ih, err := v2InfoHash(event.TorrentFilePath); err == nil {
data = append(data, v2ih)
} else {
log.Err(err)
}
d.Storage.Delete(c.StorageCtx, data...)
}
}
}()
return lst, err
return d, err
}
func v2InfoHash(path string) (ih bittorrent.InfoHash, err error) {
var mi *metainfo.MetaInfo
if mi, err = metainfo.LoadFromFile(path); err == nil {
hash := sha256.New()
hash.Write(mi.InfoBytes)
ih, err = bittorrent.NewInfoHash(hash.Sum(nil))
}
return
}
type directory struct {
middleware/torrentapproval/container/list/list.go
+35 -30
View File
@@ -5,63 +5,68 @@ package list
import (
"encoding/hex"
"fmt"
bittorrent "github.com/chihaya/chihaya/bittorrent"
"github.com/chihaya/chihaya/bittorrent"
"github.com/chihaya/chihaya/middleware/torrentapproval/container"
"github.com/chihaya/chihaya/pkg/log"
"github.com/chihaya/chihaya/storage"
"gopkg.in/yaml.v2"
"sync"
)
const Name = "list"
func init() {
container.Register("list", build)
container.Register(Name, build)
}
type Config struct {
Whitelist []string `yaml:"whitelist"`
Blacklist []string `yaml:"blacklist"`
HashList []string `yaml:"hash_list"`
Invert bool `yaml:"invert"`
StorageCtx string `yaml:"storage_ctx"`
}
var DUMMY struct{}
const DUMMY = true
// TODO: change sync map to provided storage
func build(confBytes []byte, storage storage.Storage) (container.Container, error) {
func build(confBytes []byte, st storage.Storage) (container.Container, error) {
c := new(Config)
if err := yaml.Unmarshal(confBytes, c); err != nil {
return nil, fmt.Errorf("unable to deserialise configuration: %v", err)
}
if len(c.Whitelist) > 0 && len(c.Blacklist) > 0 {
return nil, fmt.Errorf("using both whitelist and blacklist is invalid")
}
l := &List{
Hashes: sync.Map{},
Invert: len(c.Whitelist) == 0,
Invert: c.Invert,
Storage: st,
StorageCtx: c.StorageCtx,
}
hashList := c.Whitelist
if l.Invert {
hashList = c.Blacklist
if len(l.StorageCtx) == 0 {
log.Info("Storage context not set, using default value: " + container.DefaultStorageCtxName)
l.StorageCtx = container.DefaultStorageCtxName
}
for _, hashString := range hashList {
hashBytes, err := hex.DecodeString(hashString)
if err != nil {
return nil, fmt.Errorf("whitelist : invalid hash %s, %v", hashString, err)
if len(c.HashList) > 0 {
init := make([]storage.Pair, 0, len(c.HashList))
for _, hashString := range c.HashList {
hashBytes, err := hex.DecodeString(hashString)
if err != nil {
return nil, fmt.Errorf("whitelist : invalid hash %s, %v", hashString, err)
}
ih, err := bittorrent.NewInfoHash(hashBytes)
if err != nil {
return nil, fmt.Errorf("whitelist : %s : %v", hashString, err)
}
init = append(init, storage.Pair{Left: ih, Right: DUMMY})
}
ih, err := bittorrent.NewInfoHash(hashBytes)
if err != nil {
return nil, fmt.Errorf("whitelist : %s : %v", hashString, err)
}
l.Hashes.Store(ih, DUMMY)
l.Storage.BulkPut(l.StorageCtx, init...)
}
return l, nil
}
type List struct {
Invert bool
Hashes sync.Map
Invert bool
Storage storage.Storage
StorageCtx string
}
func (l *List) Contains(hash bittorrent.InfoHash) bool {
_, result := l.Hashes.Load(hash)
return result != l.Invert
func (l *List) Approved(hash bittorrent.InfoHash) bool {
b := l.Storage.Contains(l.StorageCtx, hash)
return b != l.Invert
}
middleware/torrentapproval/torrentapproval.go
+1 -1
View File
@@ -62,7 +62,7 @@ type hook struct {
func (h *hook) HandleAnnounce(ctx context.Context, req *bittorrent.AnnounceRequest, _ *bittorrent.AnnounceResponse) (context.Context, error) {
var err error
if !h.hashContainer.Contains(req.InfoHash) {
if !h.hashContainer.Approved(req.InfoHash) {
err = ErrTorrentUnapproved
}
middleware/torrentapproval/torrentapproval_test.go
+13 -6
View File
@@ -5,6 +5,7 @@ import (
"encoding/hex"
"fmt"
"github.com/chihaya/chihaya/middleware"
"github.com/chihaya/chihaya/storage/memory"
"gopkg.in/yaml.v2"
"testing"
@@ -23,7 +24,7 @@ var cases = []struct {
middleware.Config{
Name: "list",
Options: map[string]interface{}{
"whitelist": []string{"3532cf2d327fad8448c075b4cb42c8136964a435"},
"hash_list": []string{"3532cf2d327fad8448c075b4cb42c8136964a435"},
},
},
"3532cf2d327fad8448c075b4cb42c8136964a435",
@@ -34,7 +35,7 @@ var cases = []struct {
middleware.Config{
Name: "list",
Options: map[string]interface{}{
"whitelist": []string{"3532cf2d327fad8448c075b4cb42c8136964a435"},
"hash_list": []string{"3532cf2d327fad8448c075b4cb42c8136964a435"},
},
},
"4532cf2d327fad8448c075b4cb42c8136964a435",
@@ -45,7 +46,8 @@ var cases = []struct {
middleware.Config{
Name: "list",
Options: map[string]interface{}{
"blacklist": []string{"3532cf2d327fad8448c075b4cb42c8136964a435"},
"hash_list": []string{"3532cf2d327fad8448c075b4cb42c8136964a435"},
"invert": true,
},
},
"4532cf2d327fad8448c075b4cb42c8136964a435",
@@ -56,7 +58,8 @@ var cases = []struct {
middleware.Config{
Name: "list",
Options: map[string]interface{}{
"blacklist": []string{"3532cf2d327fad8448c075b4cb42c8136964a435"},
"hash_list": []string{"3532cf2d327fad8448c075b4cb42c8136964a435"},
"invert": true,
},
},
"3532cf2d327fad8448c075b4cb42c8136964a435",
@@ -65,12 +68,15 @@ var cases = []struct {
}
func TestHandleAnnounce(t *testing.T) {
config := memory.Config{}.Validate()
storage, err := memory.New(config)
require.Nil(t, err)
for _, tt := range cases {
t.Run(fmt.Sprintf("testing hash %s", tt.ih), func(t *testing.T) {
d := driver{}
cfg, err := yaml.Marshal(tt.cfg)
require.Nil(t, err)
h, err := d.NewHook(cfg)
h, err := d.NewHook(cfg, storage)
require.Nil(t, err)
ctx := context.Background()
@@ -80,7 +86,8 @@ func TestHandleAnnounce(t *testing.T) {
hashbytes, err := hex.DecodeString(tt.ih)
require.Nil(t, err)
hashinfo := bittorrent.NewInfoHash(hashbytes)
hashinfo, err := bittorrent.NewInfoHash(hashbytes)
require.Nil(t, err)
req.InfoHash = hashinfo
middleware/varinterval/varinterval.go
+1 -1
View File
@@ -8,7 +8,7 @@ import (
"sync"
"time"
yaml "gopkg.in/yaml.v2"
"gopkg.in/yaml.v2"
"github.com/chihaya/chihaya/bittorrent"
"github.com/chihaya/chihaya/middleware"