From 495cba90aea69894115716fca7524b8c9caf4d3a Mon Sep 17 00:00:00 2001 From: Gabriel Simmer Date: Tue, 1 Apr 2025 12:57:19 +0100 Subject: [PATCH 01/13] S3 container for torrentapproval --- go.mod | 18 ++ go.sum | 36 ++++ middleware/torrentapproval/container/s3/s3.go | 202 ++++++++++++++++++ middleware/torrentapproval/torrentapproval.go | 1 + 4 files changed, 257 insertions(+) create mode 100644 middleware/torrentapproval/container/s3/s3.go diff --git a/go.mod b/go.mod index c6f0a8a..c47aa10 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,8 @@ require ( github.com/MicahParks/jwkset v0.8.0 github.com/MicahParks/keyfunc/v3 v3.3.10 github.com/PowerDNS/lmdb-go v1.9.3 + github.com/aws/aws-sdk-go-v2/config v1.29.12 + github.com/aws/aws-sdk-go-v2/service/s3 v1.79.0 github.com/cespare/xxhash/v2 v2.3.0 github.com/fasthttp/router v1.5.4 github.com/golang-jwt/jwt/v5 v5.2.2 @@ -26,6 +28,22 @@ require ( require ( github.com/andybalholm/brotli v1.1.1 // indirect + github.com/aws/aws-sdk-go-v2 v1.36.3 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.17.65 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.0 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.25.2 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.0 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.33.17 // indirect + github.com/aws/smithy-go v1.22.2 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect diff --git a/go.sum b/go.sum index bc8bf21..80af177 100644 --- a/go.sum +++ b/go.sum @@ -8,6 +8,42 @@ github.com/PowerDNS/lmdb-go v1.9.3 h1:AUMY2pZT8WRpkEv39I9Id3MuoHd+NZbTVpNhruVkPT github.com/PowerDNS/lmdb-go v1.9.3/go.mod h1:TE0l+EZK8Z1B4dx070ZxkWTlp8RG1mjN0/+FkFRQMtU= github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= +github.com/aws/aws-sdk-go-v2 v1.36.3 h1:mJoei2CxPutQVxaATCzDUjcZEjVRdpsiiXi2o38yqWM= +github.com/aws/aws-sdk-go-v2 v1.36.3/go.mod h1:LLXuLpgzEbD766Z5ECcRmi8AzSwfZItDtmABVkRLGzg= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 h1:zAybnyUQXIZ5mok5Jqwlf58/TFE7uvd3IAsa1aF9cXs= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10/go.mod h1:qqvMj6gHLR/EXWZw4ZbqlPbQUyenf4h82UQUlKc+l14= +github.com/aws/aws-sdk-go-v2/config v1.29.12 h1:Y/2a+jLPrPbHpFkpAAYkVEtJmxORlXoo5k2g1fa2sUo= +github.com/aws/aws-sdk-go-v2/config v1.29.12/go.mod h1:xse1YTjmORlb/6fhkWi8qJh3cvZi4JoVNhc+NbJt4kI= +github.com/aws/aws-sdk-go-v2/credentials v1.17.65 h1:q+nV2yYegofO/SUXruT+pn4KxkxmaQ++1B/QedcKBFM= +github.com/aws/aws-sdk-go-v2/credentials v1.17.65/go.mod h1:4zyjAuGOdikpNYiSGpsGz8hLGmUzlY8pc8r9QQ/RXYQ= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 h1:x793wxmUWVDhshP8WW2mlnXuFrO4cOd3HLBroh1paFw= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30/go.mod h1:Jpne2tDnYiFascUEs2AWHJL9Yp7A5ZVy3TNyxaAjD6M= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 h1:ZK5jHhnrioRkUNOc+hOgQKlUL5JeC3S6JgLxtQ+Rm0Q= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34/go.mod h1:p4VfIceZokChbA9FzMbRGz5OV+lekcVtHlPKEO0gSZY= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 h1:SZwFm17ZUNNg5Np0ioo/gq8Mn6u9w19Mri8DnJ15Jf0= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34/go.mod h1:dFZsC0BLo346mvKQLWmoJxT+Sjp+qcVR1tRVHQGOH9Q= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34 h1:ZNTqv4nIdE/DiBfUUfXcLZ/Spcuz+RjeziUtNJackkM= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34/go.mod h1:zf7Vcd1ViW7cPqYWEHLHJkS50X0JS2IKz9Cgaj6ugrs= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 h1:eAh2A4b5IzM/lum78bZ590jy36+d/aFLgKF/4Vd1xPE= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3/go.mod h1:0yKJC/kb8sAnmlYa6Zs3QVYqaC8ug2AbnNChv5Ox3uA= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.0 h1:lguz0bmOoGzozP9XfRJR1QIayEYo+2vP/No3OfLF0pU= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.0/go.mod h1:iu6FSzgt+M2/x3Dk8zhycdIcHjEFb36IS8HVUVFoMg0= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 h1:dM9/92u2F1JbDaGooxTq18wmmFzbJRfXfVfy96/1CXM= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15/go.mod h1:SwFBy2vjtA0vZbjjaFtfN045boopadnoVPhu4Fv66vY= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15 h1:moLQUoVq91LiqT1nbvzDukyqAlCv89ZmwaHw/ZFlFZg= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15/go.mod h1:ZH34PJUc8ApjBIfgQCFvkWcUDBtl/WTD+uiYHjd8igA= +github.com/aws/aws-sdk-go-v2/service/s3 v1.79.0 h1:OIw2nryEApESTYI5deCZGcq4Gvz8DBAt4tJlNyg3v5o= +github.com/aws/aws-sdk-go-v2/service/s3 v1.79.0/go.mod h1:U5SNqwhXB3Xe6F47kXvWihPl/ilGaEDe8HD/50Z9wxc= +github.com/aws/aws-sdk-go-v2/service/sso v1.25.2 h1:pdgODsAhGo4dvzC3JAG5Ce0PX8kWXrTZGx+jxADD+5E= +github.com/aws/aws-sdk-go-v2/service/sso v1.25.2/go.mod h1:qs4a9T5EMLl/Cajiw2TcbNt2UNo/Hqlyp+GiuG4CFDI= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.0 h1:90uX0veLKcdHVfvxhkWUQSCi5VabtwMLFutYiRke4oo= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.0/go.mod h1:MlYRNmYu/fGPoxBQVvBYr9nyr948aY/WLUvwBMBJubs= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.17 h1:PZV5W8yk4OtH1JAuhV2PXwwO9v5G5Aoj+eMCn4T+1Kc= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.17/go.mod h1:cQnB8CUnxbMU82JvlqjKR2HBOm3fe9pWorWBza6MBJ4= +github.com/aws/smithy-go v1.22.2 h1:6D9hW43xKFrRx/tXXfAlIZc4JI+yQe6snnWcQyxSyLQ= +github.com/aws/smithy-go v1.22.2/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= diff --git a/middleware/torrentapproval/container/s3/s3.go b/middleware/torrentapproval/container/s3/s3.go new file mode 100644 index 0000000..41c178b --- /dev/null +++ b/middleware/torrentapproval/container/s3/s3.go @@ -0,0 +1,202 @@ +package s3 + +import ( + "context" + "crypto/sha1" + "crypto/sha256" + "fmt" + "io" + "path/filepath" + "strings" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + awss3 "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/sot-tech/mochi/bittorrent" + "github.com/sot-tech/mochi/middleware/torrentapproval/container" + "github.com/sot-tech/mochi/middleware/torrentapproval/container/list" + "github.com/sot-tech/mochi/pkg/conf" + "github.com/sot-tech/mochi/pkg/log" + "github.com/sot-tech/mochi/pkg/str2bytes" + "github.com/sot-tech/mochi/storage" + "github.com/zeebo/bencode" +) + +var logger = log.NewLogger("middleware/torrent approval/s3") + +const ( + defaultPeriod = time.Minute + maxTorrentSize = 10 * 1024 * 1024 +) + +// Config - implementation of directory container configuration. +// Extends list.Config because uses the same storage and Approved function. +type Config struct { + list.Config + Bucket string + Path string + Period time.Duration +} + +type s3 struct { + list.List + closed chan bool +} + +func init() { + container.Register("s3", build) +} + +func build(conf conf.MapConfig, st storage.DataStorage) (container.Container, error) { + c := new(Config) + if err := conf.Unmarshal(c); err != nil { + return nil, fmt.Errorf("unable to deserialise configuration: %w", err) + } + var err error + s := &s3{ + List: list.List{ + Invert: c.Invert, + Storage: st, + StorageCtx: c.StorageCtx, + }, + closed: make(chan bool), + } + if len(s.StorageCtx) == 0 { + logger.Warn(). + Str("name", "StorageCtx"). + Str("provided", s.StorageCtx). + Str("default", container.DefaultStorageCtxName). + Msg("falling back to default configuration") + s.StorageCtx = container.DefaultStorageCtxName + } + if c.Period == 0 { + logger.Warn(). + Str("name", "Period"). + Dur("provided", 0). + Dur("default", defaultPeriod). + Msg("falling back to default configuration") + c.Period = defaultPeriod + } + + ctx := context.Background() + sdkConfig, err := config.LoadDefaultConfig(ctx) + if err != nil { + return nil, fmt.Errorf("unable load aws sdk configuration: %w", err) + } + s3Client := awss3.NewFromConfig(sdkConfig) + + go s.runScan(ctx, c.Bucket, c.Path, s3Client, c.Period) + return s, err +} + +// BencodeRawBytes wrapper for byte slice to get raw 'info' section from +// torrent file +type BencodeRawBytes []byte + +// UnmarshalBencode just appends raw byte slice to result +func (ba *BencodeRawBytes) UnmarshalBencode(in []byte) error { + *ba = append([]byte(nil), in...) + return nil +} + +type torrentRawInfoStruct struct { + Info BencodeRawBytes `bencode:"info"` +} + +type torrentNameInfoStruct struct { + Name string `bencode:"name"` +} + +func (s *s3) runScan(ctx context.Context, bucket, prefix string, s3Client *awss3.Client, period time.Duration) { + t := time.NewTicker(period) + defer t.Stop() + files := make(map[string][2]bittorrent.InfoHash) + tmpFiles := make(map[string]bool) + // nolint:gosec + s1, s2 := sha1.New(), sha256.New() + for { + select { + case <-s.closed: + return + case <-t.C: + logger.Debug().Msg("starting directory scan") + listObj := &awss3.ListObjectsV2Input{Bucket: &bucket, Prefix: &prefix} + if entries, err := s3Client.ListObjectsV2(ctx, listObj); err == nil { + for _, e := range entries.Contents { + if strings.ToLower(filepath.Ext(*e.Key)) == ".torrent" { + tmpFiles[filepath.Join(prefix, *e.Key)] = true + } + } + for p := range tmpFiles { + if _, exists := files[p]; !exists { + requestInput := &awss3.GetObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(p), + } + + result, err := s3Client.GetObject(ctx, requestInput) + if err != nil { + log.Print(err) + } + var info torrentRawInfoStruct + err = bencode.NewDecoder(io.LimitReader(result.Body, maxTorrentSize)).Decode(&info) + if err == nil { + s1.Write(info.Info) + h1, _ := bittorrent.NewInfoHash(s1.Sum(nil)) + s1.Reset() + + s2.Write(info.Info) + h2, _ := bittorrent.NewInfoHash(s2.Sum(nil)) + s2.Reset() + + files[p] = [2]bittorrent.InfoHash{h1, h2} + var name torrentNameInfoStruct + if err := bencode.DecodeBytes(info.Info, &name); err != nil { + logger.Warn(). + Err(err). + Str("file", p). + Msg("unable to unmarshal torrent info") + } + if len(name.Name) == 0 { + name.Name = list.DUMMY + } + bName := str2bytes.StringToBytes(name.Name) + logger.Err(s.Storage.Put(ctx, s.StorageCtx, storage.Entry{ + Key: h1.RawString(), + Value: bName, + }, storage.Entry{ + Key: h2.RawString(), + Value: bName, + }, storage.Entry{ + Key: h2.TruncateV1().RawString(), + Value: bName, + })). + Str("file", p). + Stringer("infoHash", h1). + Stringer("infoHashV2", h2). + Msg("added torrent to approval list") + } + } + if err != nil { + logger.Warn().Err(err).Str("file", p).Msg("unable to read file") + } + } + for p, ih := range files { + if _, isOk := tmpFiles[p]; !isOk { + delete(files, p) + logger.Err(s.Storage.Delete(ctx, s.StorageCtx, ih[0].RawString(), + ih[1].RawString(), ih[1].TruncateV1().RawString())). + Str("file", p). + Stringer("infoHash", ih[1]). + Stringer("infoHashV2", ih[1]). + Msg("deleted torrent from approval list") + } + } + clear(tmpFiles) + } else { + logger.Warn().Err(err).Msg("unable to get directory content") + } + } + } +} diff --git a/middleware/torrentapproval/torrentapproval.go b/middleware/torrentapproval/torrentapproval.go index 80962b1..7be410c 100644 --- a/middleware/torrentapproval/torrentapproval.go +++ b/middleware/torrentapproval/torrentapproval.go @@ -16,6 +16,7 @@ import ( // import directory watcher to enable appropriate support _ "github.com/sot-tech/mochi/middleware/torrentapproval/container/directory" + _ "github.com/sot-tech/mochi/middleware/torrentapproval/container/s3" // import static list to enable appropriate support _ "github.com/sot-tech/mochi/middleware/torrentapproval/container/list" From 4e8f0e29b640d7be8c48d0176f5053cc05fc8097 Mon Sep 17 00:00:00 2001 From: "Lawrence, Rendall" Date: Fri, 22 Aug 2025 14:39:44 +0300 Subject: [PATCH 02/13] (not tested) rework directory torrentapproval middleware to add s3 support partially rewrite https://github.com/gmemstr/mochi/tree/s3-approval-middleware --- .../container/directory/directory.go | 109 ++++++++--- middleware/torrentapproval/container/s3/s3.go | 185 +++++------------- 2 files changed, 124 insertions(+), 170 deletions(-) diff --git a/middleware/torrentapproval/container/directory/directory.go b/middleware/torrentapproval/container/directory/directory.go index 027b0fd..979fa43 100644 --- a/middleware/torrentapproval/container/directory/directory.go +++ b/middleware/torrentapproval/container/directory/directory.go @@ -10,6 +10,7 @@ import ( "crypto/sha256" "fmt" "io" + "iter" "os" "path/filepath" "strings" @@ -53,22 +54,6 @@ func build(conf conf.MapConfig, st storage.DataStorage) (container.Container, er return nil, fmt.Errorf("unable to deserialise configuration: %w", err) } var err error - d := &directory{ - List: list.List{ - Invert: c.Invert, - Storage: st, - StorageCtx: c.StorageCtx, - }, - closed: make(chan bool), - } - if len(d.StorageCtx) == 0 { - logger.Warn(). - Str("name", "StorageCtx"). - Str("provided", d.StorageCtx). - Str("default", container.DefaultStorageCtxName). - Msg("falling back to default configuration") - d.StorageCtx = container.DefaultStorageCtxName - } if c.Period == 0 { logger.Warn(). Str("name", "Period"). @@ -77,7 +62,12 @@ func build(conf conf.MapConfig, st storage.DataStorage) (container.Container, er Msg("falling back to default configuration") c.Period = defaultPeriod } - go d.runScan(c.Path, c.Period) + d := NewScanner(list.List{ + Invert: c.Invert, + Storage: st, + StorageCtx: c.StorageCtx, + }, path(c.Path)) + go d.Run(c.Period) return d, err } @@ -99,7 +89,71 @@ type torrentNameInfoStruct struct { Name string `bencode:"name"` } -func (d *directory) runScan(path string, period time.Duration) { +// PathReader - interface for abstract directory reader +type PathReader interface { + // ReadDir returns names of torrent entries. + // Implementation must return absolute names of entries + // to fetch torrent file-like data. + ReadDir() (it iter.Seq[string], err error) + // ReadData returns reader for entry data + ReadData(entry string) (io.ReadCloser, error) +} + +type path string + +var _ PathReader = path("") + +func (p path) ReadDir() (it iter.Seq[string], err error) { + var entries []os.DirEntry + dir := string(p) + if entries, err = os.ReadDir(dir); err == nil { + it = func(yield func(string) bool) { + for _, e := range entries { + if !e.IsDir() && strings.ToLower(filepath.Ext(e.Name())) == ".torrent" { + if !yield(filepath.Join(dir, e.Name())) { + return + } + } + } + } + } + return it, err +} + +func (p path) ReadData(entry string) (io.ReadCloser, error) { + return os.Open(entry) +} + +// NewScanner creates Scanner instance. +func NewScanner(list list.List, reader PathReader) *Scanner { + if len(list.StorageCtx) == 0 { + logger.Warn(). + Str("name", "StorageCtx"). + Str("provided", list.StorageCtx). + Str("default", container.DefaultStorageCtxName). + Msg("falling back to default configuration") + list.StorageCtx = container.DefaultStorageCtxName + } + return &Scanner{ + List: list, + reader: reader, + closed: make(chan bool), + } +} + +// Scanner holds list of approved/rejected torrents +type Scanner struct { + list.List + reader PathReader + closed chan bool +} + +// Run starts periodic directory scanning and blocks until Stop called +func (d *Scanner) Run(period time.Duration) { + if d.reader == nil { + log.Warn().Msg("reader not provided") + return + } t := time.NewTicker(period) defer t.Stop() files := make(map[string][2]bittorrent.InfoHash) @@ -112,16 +166,14 @@ func (d *directory) runScan(path string, period time.Duration) { return case <-t.C: logger.Debug().Msg("starting directory scan") - if entries, err := os.ReadDir(path); err == nil { - for _, e := range entries { - if !e.IsDir() && strings.ToLower(filepath.Ext(e.Name())) == ".torrent" { - tmpFiles[filepath.Join(path, e.Name())] = true - } + if entries, err := d.reader.ReadDir(); err == nil { + for e := range entries { + tmpFiles[e] = true } for p := range tmpFiles { if _, exists := files[p]; !exists { - var f *os.File - if f, err = os.Open(p); err == nil { + var f io.ReadCloser + if f, err = d.reader.ReadData(p); err == nil { var info torrentRawInfoStruct err = bencode.NewDecoder(io.LimitReader(f, maxTorrentSize)).Decode(&info) _ = f.Close() @@ -186,13 +238,8 @@ func (d *directory) runScan(path string, period time.Duration) { } } -type directory struct { - list.List - closed chan bool -} - // Close closes watching of torrent directory -func (d *directory) Close() error { +func (d *Scanner) Close() error { if d.closed != nil { close(d.closed) } diff --git a/middleware/torrentapproval/container/s3/s3.go b/middleware/torrentapproval/container/s3/s3.go index 41c178b..c3a127c 100644 --- a/middleware/torrentapproval/container/s3/s3.go +++ b/middleware/torrentapproval/container/s3/s3.go @@ -1,11 +1,13 @@ +// Package s3 implements container which +// checks if hash present in any of torrent file +// placed in S3-like storage. package s3 import ( "context" - "crypto/sha1" - "crypto/sha256" "fmt" "io" + "iter" "path/filepath" "strings" "time" @@ -13,37 +15,28 @@ import ( "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/config" awss3 "github.com/aws/aws-sdk-go-v2/service/s3" - "github.com/sot-tech/mochi/bittorrent" + "github.com/sot-tech/mochi/middleware/torrentapproval/container" + "github.com/sot-tech/mochi/middleware/torrentapproval/container/directory" "github.com/sot-tech/mochi/middleware/torrentapproval/container/list" "github.com/sot-tech/mochi/pkg/conf" "github.com/sot-tech/mochi/pkg/log" - "github.com/sot-tech/mochi/pkg/str2bytes" "github.com/sot-tech/mochi/storage" - "github.com/zeebo/bencode" ) var logger = log.NewLogger("middleware/torrent approval/s3") -const ( - defaultPeriod = time.Minute - maxTorrentSize = 10 * 1024 * 1024 -) +const defaultPeriod = time.Minute -// Config - implementation of directory container configuration. +// Config - implementation of S3 container configuration. // Extends list.Config because uses the same storage and Approved function. type Config struct { list.Config Bucket string - Path string + Prefix string Period time.Duration } -type s3 struct { - list.List - closed chan bool -} - func init() { container.Register("s3", build) } @@ -53,23 +46,6 @@ func build(conf conf.MapConfig, st storage.DataStorage) (container.Container, er if err := conf.Unmarshal(c); err != nil { return nil, fmt.Errorf("unable to deserialise configuration: %w", err) } - var err error - s := &s3{ - List: list.List{ - Invert: c.Invert, - Storage: st, - StorageCtx: c.StorageCtx, - }, - closed: make(chan bool), - } - if len(s.StorageCtx) == 0 { - logger.Warn(). - Str("name", "StorageCtx"). - Str("provided", s.StorageCtx). - Str("default", container.DefaultStorageCtxName). - Msg("falling back to default configuration") - s.StorageCtx = container.DefaultStorageCtxName - } if c.Period == 0 { logger.Warn(). Str("name", "Period"). @@ -79,124 +55,55 @@ func build(conf conf.MapConfig, st storage.DataStorage) (container.Container, er c.Period = defaultPeriod } - ctx := context.Background() - sdkConfig, err := config.LoadDefaultConfig(ctx) + sdkConfig, err := config.LoadDefaultConfig(context.Background()) if err != nil { - return nil, fmt.Errorf("unable load aws sdk configuration: %w", err) + return nil, fmt.Errorf("unable load AWS S3 SDK configuration: %w", err) } - s3Client := awss3.NewFromConfig(sdkConfig) - go s.runScan(ctx, c.Bucket, c.Path, s3Client, c.Period) + s := directory.NewScanner(list.List{ + Invert: c.Invert, + Storage: st, + StorageCtx: c.StorageCtx, + }, s3{client: awss3.NewFromConfig(sdkConfig), bucket: c.Bucket, prefix: c.Prefix}) + go s.Run(c.Period) + return s, err } -// BencodeRawBytes wrapper for byte slice to get raw 'info' section from -// torrent file -type BencodeRawBytes []byte - -// UnmarshalBencode just appends raw byte slice to result -func (ba *BencodeRawBytes) UnmarshalBencode(in []byte) error { - *ba = append([]byte(nil), in...) - return nil +type s3 struct { + client *awss3.Client + bucket, prefix string } -type torrentRawInfoStruct struct { - Info BencodeRawBytes `bencode:"info"` -} +var _ directory.PathReader = s3{} -type torrentNameInfoStruct struct { - Name string `bencode:"name"` -} - -func (s *s3) runScan(ctx context.Context, bucket, prefix string, s3Client *awss3.Client, period time.Duration) { - t := time.NewTicker(period) - defer t.Stop() - files := make(map[string][2]bittorrent.InfoHash) - tmpFiles := make(map[string]bool) - // nolint:gosec - s1, s2 := sha1.New(), sha256.New() - for { - select { - case <-s.closed: - return - case <-t.C: - logger.Debug().Msg("starting directory scan") - listObj := &awss3.ListObjectsV2Input{Bucket: &bucket, Prefix: &prefix} - if entries, err := s3Client.ListObjectsV2(ctx, listObj); err == nil { - for _, e := range entries.Contents { - if strings.ToLower(filepath.Ext(*e.Key)) == ".torrent" { - tmpFiles[filepath.Join(prefix, *e.Key)] = true +func (s s3) ReadDir() (it iter.Seq[string], err error) { + entries, err := s.client.ListObjectsV2(context.Background(), &awss3.ListObjectsV2Input{ + Bucket: &s.bucket, + Prefix: &s.prefix, + }) + if err == nil { + it = func(yield func(string) bool) { + for _, e := range entries.Contents { + if e.Key != nil && strings.ToLower(filepath.Ext(*e.Key)) == ".torrent" { + if !yield(filepath.Join(s.prefix, *e.Key)) { + return } } - for p := range tmpFiles { - if _, exists := files[p]; !exists { - requestInput := &awss3.GetObjectInput{ - Bucket: aws.String(bucket), - Key: aws.String(p), - } - - result, err := s3Client.GetObject(ctx, requestInput) - if err != nil { - log.Print(err) - } - var info torrentRawInfoStruct - err = bencode.NewDecoder(io.LimitReader(result.Body, maxTorrentSize)).Decode(&info) - if err == nil { - s1.Write(info.Info) - h1, _ := bittorrent.NewInfoHash(s1.Sum(nil)) - s1.Reset() - - s2.Write(info.Info) - h2, _ := bittorrent.NewInfoHash(s2.Sum(nil)) - s2.Reset() - - files[p] = [2]bittorrent.InfoHash{h1, h2} - var name torrentNameInfoStruct - if err := bencode.DecodeBytes(info.Info, &name); err != nil { - logger.Warn(). - Err(err). - Str("file", p). - Msg("unable to unmarshal torrent info") - } - if len(name.Name) == 0 { - name.Name = list.DUMMY - } - bName := str2bytes.StringToBytes(name.Name) - logger.Err(s.Storage.Put(ctx, s.StorageCtx, storage.Entry{ - Key: h1.RawString(), - Value: bName, - }, storage.Entry{ - Key: h2.RawString(), - Value: bName, - }, storage.Entry{ - Key: h2.TruncateV1().RawString(), - Value: bName, - })). - Str("file", p). - Stringer("infoHash", h1). - Stringer("infoHashV2", h2). - Msg("added torrent to approval list") - } - } - if err != nil { - logger.Warn().Err(err).Str("file", p).Msg("unable to read file") - } - } - for p, ih := range files { - if _, isOk := tmpFiles[p]; !isOk { - delete(files, p) - logger.Err(s.Storage.Delete(ctx, s.StorageCtx, ih[0].RawString(), - ih[1].RawString(), ih[1].TruncateV1().RawString())). - Str("file", p). - Stringer("infoHash", ih[1]). - Stringer("infoHashV2", ih[1]). - Msg("deleted torrent from approval list") - } - } - clear(tmpFiles) - } else { - logger.Warn().Err(err).Msg("unable to get directory content") } } } + return it, err +} + +func (s s3) ReadData(entry string) (data io.ReadCloser, err error) { + var result *awss3.GetObjectOutput + result, err = s.client.GetObject(context.Background(), &awss3.GetObjectInput{ + Bucket: aws.String(s.bucket), + Key: aws.String(entry), + }) + if err == nil { + data = result.Body + } + return } From 278e2e8b4d817c281570b291bbc336021a5e7942 Mon Sep 17 00:00:00 2001 From: "Lawrence, Rendall" Date: Fri, 19 Sep 2025 19:11:15 +0300 Subject: [PATCH 03/13] (WIP) s3 torrent approval test suite --- .../container/directory/directory.go | 47 +++-- .../container/directory/directory_test.go | 109 ++++++++++ middleware/torrentapproval/container/s3/s3.go | 77 +++++-- .../torrentapproval/container/s3/s3_test.go | 195 ++++++++++++++++++ 4 files changed, 391 insertions(+), 37 deletions(-) create mode 100644 middleware/torrentapproval/container/directory/directory_test.go create mode 100644 middleware/torrentapproval/container/s3/s3_test.go diff --git a/middleware/torrentapproval/container/directory/directory.go b/middleware/torrentapproval/container/directory/directory.go index 979fa43..a6231a0 100644 --- a/middleware/torrentapproval/container/directory/directory.go +++ b/middleware/torrentapproval/container/directory/directory.go @@ -53,46 +53,35 @@ func build(conf conf.MapConfig, st storage.DataStorage) (container.Container, er if err := conf.Unmarshal(c); err != nil { return nil, fmt.Errorf("unable to deserialise configuration: %w", err) } - var err error - if c.Period == 0 { - logger.Warn(). - Str("name", "Period"). - Dur("provided", 0). - Dur("default", defaultPeriod). - Msg("falling back to default configuration") - c.Period = defaultPeriod - } d := NewScanner(list.List{ Invert: c.Invert, Storage: st, StorageCtx: c.StorageCtx, - }, path(c.Path)) - go d.Run(c.Period) - return d, err + }, path(c.Path), c.Period) + go d.Run() + return d, nil } -// BencodeRawBytes wrapper for byte slice to get raw 'info' section from -// torrent file -type BencodeRawBytes []byte +type bencodeRawBytes []byte // UnmarshalBencode just appends raw byte slice to result -func (ba *BencodeRawBytes) UnmarshalBencode(in []byte) error { +func (ba *bencodeRawBytes) UnmarshalBencode(in []byte) error { *ba = append([]byte(nil), in...) return nil } type torrentRawInfoStruct struct { - Info BencodeRawBytes `bencode:"info"` + Info bencodeRawBytes `bencode:"info"` } type torrentNameInfoStruct struct { Name string `bencode:"name"` } -// PathReader - interface for abstract directory reader +// PathReader - interface for abstract directory-like reader type PathReader interface { // ReadDir returns names of torrent entries. - // Implementation must return absolute names of entries + // Implementation must return absolute paths of entries // to fetch torrent file-like data. ReadDir() (it iter.Seq[string], err error) // ReadData returns reader for entry data @@ -125,7 +114,7 @@ func (p path) ReadData(entry string) (io.ReadCloser, error) { } // NewScanner creates Scanner instance. -func NewScanner(list list.List, reader PathReader) *Scanner { +func NewScanner(list list.List, reader PathReader, period time.Duration) *Scanner { if len(list.StorageCtx) == 0 { logger.Warn(). Str("name", "StorageCtx"). @@ -134,9 +123,18 @@ func NewScanner(list list.List, reader PathReader) *Scanner { Msg("falling back to default configuration") list.StorageCtx = container.DefaultStorageCtxName } + if period == 0 { + logger.Warn(). + Str("name", "Period"). + Dur("provided", 0). + Dur("default", defaultPeriod). + Msg("falling back to default configuration") + period = defaultPeriod + } return &Scanner{ List: list, reader: reader, + period: period, closed: make(chan bool), } } @@ -145,16 +143,21 @@ func NewScanner(list list.List, reader PathReader) *Scanner { type Scanner struct { list.List reader PathReader + period time.Duration closed chan bool } // Run starts periodic directory scanning and blocks until Stop called -func (d *Scanner) Run(period time.Duration) { +func (d *Scanner) Run() { if d.reader == nil { log.Warn().Msg("reader not provided") return } - t := time.NewTicker(period) + if d.period == 0 { + log.Warn().Msg("period not provided") + return + } + t := time.NewTicker(d.period) defer t.Stop() files := make(map[string][2]bittorrent.InfoHash) tmpFiles := make(map[string]bool) diff --git a/middleware/torrentapproval/container/directory/directory_test.go b/middleware/torrentapproval/container/directory/directory_test.go new file mode 100644 index 0000000..4292700 --- /dev/null +++ b/middleware/torrentapproval/container/directory/directory_test.go @@ -0,0 +1,109 @@ +package directory + +import ( + "context" + "encoding/base64" + "encoding/hex" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/sot-tech/mochi/middleware/torrentapproval/container/list" + "github.com/sot-tech/mochi/pkg/conf" + "github.com/sot-tech/mochi/pkg/log" + "github.com/sot-tech/mochi/storage/memory" +) + +type testData struct { + name string + data []byte + hash string +} + +func unHEX(s string) string { + b, _ := hex.DecodeString(s) + return string(b) +} + +func unBase64(s string) []byte { + b, _ := base64.StdEncoding.DecodeString(s) + return b +} + +// contains created torrent file data from simple txt documents. +// data base64 encoded because `pieces` section in torrent file is raw bytes +var files = [2]testData{ + { + name: "test0.torrent", + data: unBase64(`ZDEwOmNyZWF0ZWQgYnkzMTpUcmFuc21pc3Npb24vNC4wLjYgKDM4YzE2NDkzM2UpMTM6Y3JlYXRp +b24gZGF0ZWkxNzU1ODYxOTI3ZTg6ZW5jb2Rpbmc1OlVURi04NDppbmZvZDY6bGVuZ3RoaTVlNDpu +YW1lODp0ZXN0LnR4dDEyOnBpZWNlIGxlbmd0aGkzMjc2OGU2OnBpZWNlczIwOk4SQ70ixm52wrqe +3cH5E5Tlf5+DZWU=`), + hash: unHEX("a10e8e9e81702bf8482f251551ff4fe011cba6a7"), + }, + { + name: "test1.torrent", + data: unBase64(`ZDEwOmNyZWF0ZWQgYnkzMTpUcmFuc21pc3Npb24vNC4wLjYgKDM4YzE2NDkzM2UpMTM6Y3JlYXRp +b24gZGF0ZWkxNzU2MTIzNzEwZTg6ZW5jb2Rpbmc1OlVURi04NDppbmZvZDY6bGVuZ3RoaTRlNDpu +YW1lOTp0ZXN0MC50eHQxMjpwaWVjZSBsZW5ndGhpMzI3NjhlNjpwaWVjZXMyMDqo/cIFqfGcwcdQ +emDE8BsT0R1/0GVl`), + hash: unHEX("e86d393bd458d2acc46d5467bc8cb8b30b1bfa77"), + }, +} + +func init() { + _ = log.ConfigureLogger("", "warn", false, false) +} + +func writeTmp() (string, error) { + tmpDir, err := os.MkdirTemp("", "") + if err != nil { + return "", err + } + for _, f := range files { + err = os.WriteFile(filepath.Join(tmpDir, f.name), f.data, 0644) + if err != nil { + return "", err + } + } + return tmpDir, err +} + +func TestScan(t *testing.T) { + tmpDir, err := writeTmp() + t.Cleanup(func() { + err := os.RemoveAll(tmpDir) + if err != nil { + t.Log(err) + } + }) + if err != nil { + t.Error(err) + return + } + st, _ := memory.Builder{}.NewDataStorage(make(conf.MapConfig)) + d := NewScanner(list.List{ + Invert: false, + Storage: st, + StorageCtx: "TEST", + }, path(tmpDir), time.Millisecond*10) + go d.Run() + t.Cleanup(func() { + _ = d.Close() + }) + time.Sleep(time.Millisecond * 100) + for _, f := range files { + contains, _ := d.List.Storage.Contains(context.Background(), "TEST", f.hash) + require.True(t, contains, "%s must present", f.name) + _ = os.Remove(filepath.Join(tmpDir, f.name)) + } + + time.Sleep(time.Millisecond * 100) + for _, f := range files { + contains, _ := d.List.Storage.Contains(context.Background(), "TEST", f.hash) + require.False(t, contains, "%s must absent", f.name) + } +} diff --git a/middleware/torrentapproval/container/s3/s3.go b/middleware/torrentapproval/container/s3/s3.go index c3a127c..586cd3a 100644 --- a/middleware/torrentapproval/container/s3/s3.go +++ b/middleware/torrentapproval/container/s3/s3.go @@ -12,9 +12,10 @@ import ( "strings" "time" - "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" awss3 "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/smithy-go/logging" "github.com/sot-tech/mochi/middleware/torrentapproval/container" "github.com/sot-tech/mochi/middleware/torrentapproval/container/directory" @@ -32,9 +33,14 @@ const defaultPeriod = time.Minute // Extends list.Config because uses the same storage and Approved function. type Config struct { list.Config - Bucket string - Prefix string - Period time.Duration + Endpoint string + Region string + KeyID string + KeySecret string + SessionToken string + Bucket string + Prefix string + Period time.Duration } func init() { @@ -55,38 +61,79 @@ func build(conf conf.MapConfig, st storage.DataStorage) (container.Container, er c.Period = defaultPeriod } - sdkConfig, err := config.LoadDefaultConfig(context.Background()) + awsCfg, err := config.LoadDefaultConfig(context.Background()) if err != nil { return nil, fmt.Errorf("unable load AWS S3 SDK configuration: %w", err) } + awsCfg.Logger = logging.LoggerFunc(func(classification logging.Classification, format string, v ...interface{}) { + if classification == logging.Debug { + logger.Debug().CallerSkipFrame(1).Msg(fmt.Sprintf(format, v...)) + } else if classification == logging.Warn { + logger.Warn().CallerSkipFrame(1).Msg(fmt.Sprintf(format, v...)) + } + }) + + if len(c.Endpoint) > 0 { + awsCfg.BaseEndpoint = &c.Endpoint + } + + if len(c.Region) > 0 { + awsCfg.Region = c.Region + } + + if len(c.KeyID) > 0 || len(c.KeySecret) > 0 || len(c.SessionToken) > 0 { + awsCfg.Credentials = credentials.NewStaticCredentialsProvider(c.KeyID, c.KeySecret, c.SessionToken) + } + s := directory.NewScanner(list.List{ Invert: c.Invert, Storage: st, StorageCtx: c.StorageCtx, - }, s3{client: awss3.NewFromConfig(sdkConfig), bucket: c.Bucket, prefix: c.Prefix}) - go s.Run(c.Period) + }, s3{ + client: awss3.NewFromConfig(awsCfg), + bucket: c.Bucket, + prefix: c.Prefix, + }, c.Period) + go s.Run() return s, err } +type s3Client interface { + ListObjectsV2( + ctx context.Context, input *awss3.ListObjectsV2Input, f ...func(*awss3.Options), + ) (*awss3.ListObjectsV2Output, error) + GetObject( + ctx context.Context, params *awss3.GetObjectInput, optFns ...func(*awss3.Options), + ) (*awss3.GetObjectOutput, error) +} + type s3 struct { - client *awss3.Client + client s3Client bucket, prefix string } var _ directory.PathReader = s3{} func (s s3) ReadDir() (it iter.Seq[string], err error) { - entries, err := s.client.ListObjectsV2(context.Background(), &awss3.ListObjectsV2Input{ - Bucket: &s.bucket, - Prefix: &s.prefix, - }) + search := &awss3.ListObjectsV2Input{Bucket: &s.bucket} + if len(s.prefix) > 0 { + search.Prefix = &s.prefix + } + entries, err := s.client.ListObjectsV2(context.Background(), search) if err == nil { it = func(yield func(string) bool) { for _, e := range entries.Contents { + logger.Trace().Any("content", e).Msg("read dir") if e.Key != nil && strings.ToLower(filepath.Ext(*e.Key)) == ".torrent" { - if !yield(filepath.Join(s.prefix, *e.Key)) { + var name string + if len(s.prefix) == 0 { + name = *e.Key + } else { + name = filepath.Join(s.prefix, *e.Key) + } + if !yield(name) { return } } @@ -99,8 +146,8 @@ func (s s3) ReadDir() (it iter.Seq[string], err error) { func (s s3) ReadData(entry string) (data io.ReadCloser, err error) { var result *awss3.GetObjectOutput result, err = s.client.GetObject(context.Background(), &awss3.GetObjectInput{ - Bucket: aws.String(s.bucket), - Key: aws.String(entry), + Bucket: &s.bucket, + Key: &entry, }) if err == nil { data = result.Body diff --git a/middleware/torrentapproval/container/s3/s3_test.go b/middleware/torrentapproval/container/s3/s3_test.go new file mode 100644 index 0000000..a5012cb --- /dev/null +++ b/middleware/torrentapproval/container/s3/s3_test.go @@ -0,0 +1,195 @@ +package s3 + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/hex" + "io" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + awss3 "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/stretchr/testify/require" + + "github.com/sot-tech/mochi/middleware/torrentapproval/container/directory" + "github.com/sot-tech/mochi/middleware/torrentapproval/container/list" + "github.com/sot-tech/mochi/pkg/conf" + "github.com/sot-tech/mochi/pkg/log" + "github.com/sot-tech/mochi/storage/memory" +) + +type testData struct { + data []byte + hash string +} + +func unHEX(s string) string { + b, _ := hex.DecodeString(s) + return string(b) +} + +func unBase64(s string) []byte { + b, _ := base64.StdEncoding.DecodeString(s) + return b +} + +// contains created torrent file data from simple txt documents. +// data base64 encoded because `pieces` section in torrent file is raw bytes +var files = map[string]testData{ + "test0.torrent": { + data: unBase64(`ZDEwOmNyZWF0ZWQgYnkzMTpUcmFuc21pc3Npb24vNC4wLjYgKDM4YzE2NDkzM2UpMTM6Y3JlYXRp +b24gZGF0ZWkxNzU1ODYxOTI3ZTg6ZW5jb2Rpbmc1OlVURi04NDppbmZvZDY6bGVuZ3RoaTVlNDpu +YW1lODp0ZXN0LnR4dDEyOnBpZWNlIGxlbmd0aGkzMjc2OGU2OnBpZWNlczIwOk4SQ70ixm52wrqe +3cH5E5Tlf5+DZWU=`), + hash: unHEX("a10e8e9e81702bf8482f251551ff4fe011cba6a7"), + }, + "test1.torrent": { + data: unBase64(`ZDEwOmNyZWF0ZWQgYnkzMTpUcmFuc21pc3Npb24vNC4wLjYgKDM4YzE2NDkzM2UpMTM6Y3JlYXRp +b24gZGF0ZWkxNzU2MTIzNzEwZTg6ZW5jb2Rpbmc1OlVURi04NDppbmZvZDY6bGVuZ3RoaTRlNDpu +YW1lOTp0ZXN0MC50eHQxMjpwaWVjZSBsZW5ndGhpMzI3NjhlNjpwaWVjZXMyMDqo/cIFqfGcwcdQ +emDE8BsT0R1/0GVl`), + hash: unHEX("e86d393bd458d2acc46d5467bc8cb8b30b1bfa77"), + }, +} + +func init() { + _ = log.ConfigureLogger("", "debug", false, false) +} + +type mockS3 struct { + objs []types.Object +} + +func (m *mockS3) ListObjectsV2( + context.Context, *awss3.ListObjectsV2Input, ...func(*awss3.Options), +) (*awss3.ListObjectsV2Output, error) { + return &awss3.ListObjectsV2Output{ + Contents: m.objs, + }, nil +} + +var _ s3Client = &mockS3{} + +func (m *mockS3) GetObject( + _ context.Context, params *awss3.GetObjectInput, _ ...func(*awss3.Options), +) (*awss3.GetObjectOutput, error) { + if params == nil || params.Key == nil { + return nil, nil + } + v := files[*params.Key] + return &awss3.GetObjectOutput{ + Body: io.NopCloser(bytes.NewReader(v.data)), + }, nil +} + +func TestScanMock(t *testing.T) { + cl := &mockS3{make([]types.Object, 0, len(files))} + for k := range files { + cl.objs = append(cl.objs, types.Object{Key: &k}) + } + st, _ := memory.Builder{}.NewDataStorage(make(conf.MapConfig)) + d := directory.NewScanner(list.List{ + Invert: false, + Storage: st, + StorageCtx: "TEST", + }, s3{ + client: cl, + }, time.Millisecond*10) + go d.Run() + t.Cleanup(func() { + _ = d.Close() + }) + + time.Sleep(time.Millisecond * 100) + for name, f := range files { + contains, _ := d.List.Storage.Contains(context.Background(), "TEST", f.hash) + require.True(t, contains, "%s must present", name) + for i := 0; i < len(cl.objs); i++ { + if *cl.objs[i].Key == name { + cl.objs = append(cl.objs[:i], cl.objs[i+1:]...) + } + } + } + + time.Sleep(time.Millisecond * 100) + for name, f := range files { + contains, _ := d.List.Storage.Contains(context.Background(), "TEST", f.hash) + require.False(t, contains, "%s must absent", name) + } +} + +var ( + minioEndpoint = "http://127.0.0.1:9000" + minioBucket = "test" +) + +const ( + minioKeyID = "minioadmin" + minioSecret = "minioadmin" + minioRegion = "us-east-1" +) + +// TestScanMinio requires real minio instance listening 127.0.0.1:9000 +// with default login/password (minioadmin/minioadmin) +func TestScanMinio(t *testing.T) { + st, _ := memory.Builder{}.NewDataStorage(make(conf.MapConfig)) + awsCfg, err := config.LoadDefaultConfig(context.Background()) + if err != nil { + t.Fatal(err) + } + awsCfg.BaseEndpoint = &minioEndpoint + awsCfg.Region = minioRegion + awsCfg.Credentials = credentials.NewStaticCredentialsProvider(minioKeyID, minioSecret, "") + cl := awss3.NewFromConfig(awsCfg) + + _, _ = cl.CreateBucket(context.Background(), &awss3.CreateBucketInput{Bucket: &minioBucket}) + + for name, data := range files { + _, err = cl.PutObject(context.Background(), &awss3.PutObjectInput{ + Bucket: &minioBucket, + Key: &name, + Body: bytes.NewReader(data.data), + }) + if err != nil { + t.Fatal(err) + } + } + + d := directory.NewScanner(list.List{ + Invert: false, + Storage: st, + StorageCtx: "TEST", + }, s3{ + client: cl, + bucket: minioBucket, + }, time.Millisecond*100) + go d.Run() + t.Cleanup(func() { + _ = d.Close() + }) + + time.Sleep(time.Millisecond * 200) + + for name, f := range files { + contains, _ := d.List.Storage.Contains(context.Background(), "TEST", f.hash) + require.True(t, contains, "%s must present", name) + _, err = cl.DeleteObject(context.Background(), &awss3.DeleteObjectInput{ + Bucket: &minioBucket, + Key: &name, + }) + if err != nil { + t.Fatal(err) + } + } + + time.Sleep(time.Millisecond * 200) + + for name, f := range files { + contains, _ := d.List.Storage.Contains(context.Background(), "TEST", f.hash) + require.False(t, contains, "%s must absent", name) + } +} From c6a5be08e929391d6968211d8c698425d3af28b1 Mon Sep 17 00:00:00 2001 From: "Lawrence, Rendall" Date: Mon, 22 Sep 2025 14:59:51 +0300 Subject: [PATCH 04/13] (pertially tested) remove S3 prefix concat to Key * add S3 test for prefix and suffix * update dependencies * add workflow image for minio --- .github/workflows/build.yaml | 7 + go.mod | 71 ++++---- go.sum | 158 +++++++++--------- middleware/torrentapproval/container/s3/s3.go | 19 +-- .../torrentapproval/container/s3/s3_test.go | 46 ++++- 5 files changed, 171 insertions(+), 130 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 2d8f237..fcf9edb 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -38,6 +38,13 @@ jobs: --health-interval 10s --health-timeout 5s --health-retries 5 + minio: + image: "minio/minio" + ports: [ "9000:9000" ] + options: >- + server + --address + 127.0.0.1:9000 steps: - uses: "actions/checkout@v4" - uses: "actions/setup-go@v5" diff --git a/go.mod b/go.mod index 504e2dd..59c664b 100644 --- a/go.mod +++ b/go.mod @@ -1,49 +1,47 @@ module github.com/sot-tech/mochi -go 1.23.0 - -toolchain go1.23.6 +go 1.24.0 require ( - code.cloudfoundry.org/go-diodes v0.0.0-20250819102816-d2f6d525a844 - github.com/MicahParks/jwkset v0.9.6 - github.com/MicahParks/keyfunc/v3 v3.6.1 + code.cloudfoundry.org/go-diodes v0.0.0-20250909124000-1dfc755f0d96 + github.com/MicahParks/jwkset v0.11.0 + github.com/MicahParks/keyfunc/v3 v3.6.2 github.com/PowerDNS/lmdb-go v1.9.3 - github.com/aws/aws-sdk-go-v2 v1.38.1 - github.com/aws/aws-sdk-go-v2/config v1.31.2 - github.com/aws/aws-sdk-go-v2/service/s3 v1.87.1 + github.com/aws/aws-sdk-go-v2/config v1.31.8 + github.com/aws/aws-sdk-go-v2/credentials v1.18.12 + github.com/aws/aws-sdk-go-v2/service/s3 v1.88.1 + github.com/aws/smithy-go v1.23.0 github.com/cespare/xxhash/v2 v2.3.0 github.com/fasthttp/router v1.5.4 github.com/golang-jwt/jwt/v5 v5.3.0 - github.com/jackc/pgx/v5 v5.7.5 + github.com/jackc/pgx/v5 v5.7.6 github.com/libp2p/go-reuseport v0.4.0 github.com/mitchellh/mapstructure v1.5.0 - github.com/prometheus/client_golang v1.23.0 - github.com/redis/go-redis/v9 v9.12.1 + github.com/prometheus/client_golang v1.23.2 + github.com/redis/go-redis/v9 v9.14.0 github.com/rs/zerolog v1.34.0 - github.com/stretchr/testify v1.10.0 - github.com/valyala/fasthttp v1.65.0 + github.com/stretchr/testify v1.11.1 + github.com/valyala/fasthttp v1.66.0 github.com/zeebo/bencode v1.0.0 gopkg.in/yaml.v3 v3.0.1 ) require ( github.com/andybalholm/brotli v1.2.0 // indirect - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.0 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.18.6 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.4 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.4 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.4 // indirect + github.com/aws/aws-sdk-go-v2 v1.39.0 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.1 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.7 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.7 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.7 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.4 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.0 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.8.4 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.4 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.4 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.28.2 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.33.2 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.38.0 // indirect - github.com/aws/smithy-go v1.22.5 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.7 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.1 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.8.7 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.7 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.7 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.29.3 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.34.4 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.38.4 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect @@ -57,15 +55,16 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.6.2 // indirect - github.com/prometheus/common v0.65.0 // indirect + github.com/prometheus/common v0.66.1 // indirect github.com/prometheus/procfs v0.17.0 // indirect github.com/savsgio/gotils v0.0.0-20250408102913-196191ec6287 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect - golang.org/x/crypto v0.41.0 // indirect - golang.org/x/net v0.43.0 // indirect - golang.org/x/sync v0.16.0 // indirect - golang.org/x/sys v0.35.0 // indirect - golang.org/x/text v0.28.0 // indirect - golang.org/x/time v0.12.0 // indirect - google.golang.org/protobuf v1.36.8 // indirect + go.yaml.in/yaml/v2 v2.4.3 // indirect + golang.org/x/crypto v0.42.0 // indirect + golang.org/x/net v0.44.0 // indirect + golang.org/x/sync v0.17.0 // indirect + golang.org/x/sys v0.36.0 // indirect + golang.org/x/text v0.29.0 // indirect + golang.org/x/time v0.13.0 // indirect + google.golang.org/protobuf v1.36.9 // indirect ) diff --git a/go.sum b/go.sum index 9c812fe..b3ecf7a 100644 --- a/go.sum +++ b/go.sum @@ -1,49 +1,51 @@ -code.cloudfoundry.org/go-diodes v0.0.0-20250819102816-d2f6d525a844 h1:Y1kyTNWp3C4NBcTOfwNYdtn4wIWKMuxjhNYaByo7pGs= -code.cloudfoundry.org/go-diodes v0.0.0-20250819102816-d2f6d525a844/go.mod h1:Kix9WmA7sHvHOhQ8vYalSSvwpZjZ1Vvr6H9RF0i4fAQ= -github.com/MicahParks/jwkset v0.9.6 h1:Tf8l2/MOby5Kh3IkrqzThPQKfLytMERoAsGZKlyYZxg= -github.com/MicahParks/jwkset v0.9.6/go.mod h1:U2oRhRaLgDCLjtpGL2GseNKGmZtLs/3O7p+OZaL5vo0= -github.com/MicahParks/keyfunc/v3 v3.6.1 h1:A8A5zGZ8XmRyxizSY7s5FLY/aSplrnEBLCOrC0D1ojM= -github.com/MicahParks/keyfunc/v3 v3.6.1/go.mod h1:y6Ed3dMgNKTcpxbaQHD8mmrYDUZWJAxteddA6OQj+ag= +code.cloudfoundry.org/go-diodes v0.0.0-20250909124000-1dfc755f0d96 h1:RhpatInI+K7hN/IxhT5z5Jy8obH5NVDmlpWAI4LxNoo= +code.cloudfoundry.org/go-diodes v0.0.0-20250909124000-1dfc755f0d96/go.mod h1:UenGpPnsmAK4LqD5O9Z9FYFv52ebeyWuc39hnVRGCtM= +github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= +github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/MicahParks/jwkset v0.11.0 h1:yc0zG+jCvZpWgFDFmvs8/8jqqVBG9oyIbmBtmjOhoyQ= +github.com/MicahParks/jwkset v0.11.0/go.mod h1:U2oRhRaLgDCLjtpGL2GseNKGmZtLs/3O7p+OZaL5vo0= +github.com/MicahParks/keyfunc/v3 v3.6.2 h1:82rre60MKw4r117ew5/T4m1AphgkpCOYry0RPbFUY3w= +github.com/MicahParks/keyfunc/v3 v3.6.2/go.mod h1:z66bkCviwqfg2YUp+Jcc/xRE9IXLcMq6DrgV/+Htru0= github.com/PowerDNS/lmdb-go v1.9.3 h1:AUMY2pZT8WRpkEv39I9Id3MuoHd+NZbTVpNhruVkPTg= github.com/PowerDNS/lmdb-go v1.9.3/go.mod h1:TE0l+EZK8Z1B4dx070ZxkWTlp8RG1mjN0/+FkFRQMtU= github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ= github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= -github.com/aws/aws-sdk-go-v2 v1.38.1 h1:j7sc33amE74Rz0M/PoCpsZQ6OunLqys/m5antM0J+Z8= -github.com/aws/aws-sdk-go-v2 v1.38.1/go.mod h1:9Q0OoGQoboYIAJyslFyF1f5K1Ryddop8gqMhWx/n4Wg= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.0 h1:6GMWV6CNpA/6fbFHnoAjrv4+LGfyTqZz2LtCHnspgDg= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.0/go.mod h1:/mXlTIVG9jbxkqDnr5UQNQxW1HRYxeGklkM9vAFeabg= -github.com/aws/aws-sdk-go-v2/config v1.31.2 h1:NOaSZpVGEH2Np/c1toSeW0jooNl+9ALmsUTZ8YvkJR0= -github.com/aws/aws-sdk-go-v2/config v1.31.2/go.mod h1:17ft42Yb2lF6OigqSYiDAiUcX4RIkEMY6XxEMJsrAes= -github.com/aws/aws-sdk-go-v2/credentials v1.18.6 h1:AmmvNEYrru7sYNJnp3pf57lGbiarX4T9qU/6AZ9SucU= -github.com/aws/aws-sdk-go-v2/credentials v1.18.6/go.mod h1:/jdQkh1iVPa01xndfECInp1v1Wnp70v3K4MvtlLGVEc= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.4 h1:lpdMwTzmuDLkgW7086jE94HweHCqG+uOJwHf3LZs7T0= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.4/go.mod h1:9xzb8/SV62W6gHQGC/8rrvgNXU6ZoYM3sAIJCIrXJxY= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.4 h1:IdCLsiiIj5YJ3AFevsewURCPV+YWUlOW8JiPhoAy8vg= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.4/go.mod h1:l4bdfCD7XyyZA9BolKBo1eLqgaJxl0/x91PL4Yqe0ao= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.4 h1:j7vjtr1YIssWQOMeOWRbh3z8g2oY/xPjnZH2gLY4sGw= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.4/go.mod h1:yDmJgqOiH4EA8Hndnv4KwAo8jCGTSnM5ASG1nBI+toA= +github.com/aws/aws-sdk-go-v2 v1.39.0 h1:xm5WV/2L4emMRmMjHFykqiA4M/ra0DJVSWUkDyBjbg4= +github.com/aws/aws-sdk-go-v2 v1.39.0/go.mod h1:sDioUELIUO9Znk23YVmIk86/9DOpkbyyVb1i/gUNFXY= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.1 h1:i8p8P4diljCr60PpJp6qZXNlgX4m2yQFpYk+9ZT+J4E= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.1/go.mod h1:ddqbooRZYNoJ2dsTwOty16rM+/Aqmk/GOXrK8cg7V00= +github.com/aws/aws-sdk-go-v2/config v1.31.8 h1:kQjtOLlTU4m4A64TsRcqwNChhGCwaPBt+zCQt/oWsHU= +github.com/aws/aws-sdk-go-v2/config v1.31.8/go.mod h1:QPpc7IgljrKwH0+E6/KolCgr4WPLerURiU592AYzfSY= +github.com/aws/aws-sdk-go-v2/credentials v1.18.12 h1:zmc9e1q90wMn8wQbjryy8IwA6Q4XlaL9Bx2zIqdNNbk= +github.com/aws/aws-sdk-go-v2/credentials v1.18.12/go.mod h1:3VzdRDR5u3sSJRI4kYcOSIBbeYsgtVk7dG5R/U6qLWY= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.7 h1:Is2tPmieqGS2edBnmOJIbdvOA6Op+rRpaYR60iBAwXM= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.7/go.mod h1:F1i5V5421EGci570yABvpIXgRIBPb5JM+lSkHF6Dq5w= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.7 h1:UCxq0X9O3xrlENdKf1r9eRJoKz/b0AfGkpp3a7FPlhg= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.7/go.mod h1:rHRoJUNUASj5Z/0eqI4w32vKvC7atoWR0jC+IkmVH8k= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.7 h1:Y6DTZUn7ZUC4th9FMBbo8LVE+1fyq3ofw+tRwkUd3PY= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.7/go.mod h1:x3XE6vMnU9QvHN/Wrx2s44kwzV2o2g5x/siw4ZUJ9g8= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.4 h1:BE/MNQ86yzTINrfxPPFS86QCBNQeLKY2A0KhDh47+wI= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.4/go.mod h1:SPBBhkJxjcrzJBc+qY85e83MQ2q3qdra8fghhkkyrJg= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.0 h1:6+lZi2JeGKtCraAj1rpoZfKqnQ9SptseRZioejfUOLM= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.0/go.mod h1:eb3gfbVIxIoGgJsi9pGne19dhCBpK6opTYpQqAmdy44= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.8.4 h1:Beh9oVgtQnBgR4sKKzkUBRQpf1GnL4wt0l4s8h2VCJ0= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.8.4/go.mod h1:b17At0o8inygF+c6FOD3rNyYZufPw62o9XJbSfQPgbo= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.4 h1:ueB2Te0NacDMnaC+68za9jLwkjzxGWm0KB5HTUHjLTI= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.4/go.mod h1:nLEfLnVMmLvyIG58/6gsSA03F1voKGaCfHV7+lR8S7s= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.4 h1:HVSeukL40rHclNcUqVcBwE1YoZhOkoLeBfhUqR3tjIU= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.4/go.mod h1:DnbBOv4FlIXHj2/xmrUQYtawRFC9L9ZmQPz+DBc6X5I= -github.com/aws/aws-sdk-go-v2/service/s3 v1.87.1 h1:2n6Pd67eJwAb/5KCX62/8RTU0aFAAW7V5XIGSghiHrw= -github.com/aws/aws-sdk-go-v2/service/s3 v1.87.1/go.mod h1:w5PC+6GHLkvMJKasYGVloB3TduOtROEMqm15HSuIbw4= -github.com/aws/aws-sdk-go-v2/service/sso v1.28.2 h1:ve9dYBB8CfJGTFqcQ3ZLAAb/KXWgYlgu/2R2TZL2Ko0= -github.com/aws/aws-sdk-go-v2/service/sso v1.28.2/go.mod h1:n9bTZFZcBa9hGGqVz3i/a6+NG0zmZgtkB9qVVFDqPA8= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.33.2 h1:pd9G9HQaM6UZAZh19pYOkpKSQkyQQ9ftnl/LttQOcGI= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.33.2/go.mod h1:eknndR9rU8UpE/OmFpqU78V1EcXPKFTTm5l/buZYgvM= -github.com/aws/aws-sdk-go-v2/service/sts v1.38.0 h1:iV1Ko4Em/lkJIsoKyGfc0nQySi+v0Udxr6Igq+y9JZc= -github.com/aws/aws-sdk-go-v2/service/sts v1.38.0/go.mod h1:bEPcjW7IbolPfK67G1nilqWyoxYMSPrDiIQ3RdIdKgo= -github.com/aws/smithy-go v1.22.5 h1:P9ATCXPMb2mPjYBgueqJNCA5S9UfktsW0tTxi+a7eqw= -github.com/aws/smithy-go v1.22.5/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.7 h1:BszAktdUo2xlzmYHjWMq70DqJ7cROM8iBd3f6hrpuMQ= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.7/go.mod h1:XJ1yHki/P7ZPuG4fd3f0Pg/dSGA2cTQBCLw82MH2H48= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.1 h1:oegbebPEMA/1Jny7kvwejowCaHz1FWZAQ94WXFNCyTM= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.1/go.mod h1:kemo5Myr9ac0U9JfSjMo9yHLtw+pECEHsFtJ9tqCEI8= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.8.7 h1:zmZ8qvtE9chfhBPuKB2aQFxW5F/rpwXUgmcVCgQzqRw= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.8.7/go.mod h1:vVYfbpd2l+pKqlSIDIOgouxNsGu5il9uDp0ooWb0jys= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.7 h1:mLgc5QIgOy26qyh5bvW+nDoAppxgn3J2WV3m9ewq7+8= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.7/go.mod h1:wXb/eQnqt8mDQIQTTmcw58B5mYGxzLGZGK8PWNFZ0BA= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.7 h1:u3VbDKUCWarWiU+aIUK4gjTr/wQFXV17y3hgNno9fcA= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.7/go.mod h1:/OuMQwhSyRapYxq6ZNpPer8juGNrB4P5Oz8bZ2cgjQE= +github.com/aws/aws-sdk-go-v2/service/s3 v1.88.1 h1:+RpGuaQ72qnU83qBKVwxkznewEdAGhIWo/PQCmkhhog= +github.com/aws/aws-sdk-go-v2/service/s3 v1.88.1/go.mod h1:xajPTguLoeQMAOE44AAP2RQoUhF8ey1g5IFHARv71po= +github.com/aws/aws-sdk-go-v2/service/sso v1.29.3 h1:7PKX3VYsZ8LUWceVRuv0+PU+E7OtQb1lgmi5vmUE9CM= +github.com/aws/aws-sdk-go-v2/service/sso v1.29.3/go.mod h1:Ql6jE9kyyWI5JHn+61UT/Y5Z0oyVJGmgmJbZD5g4unY= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.34.4 h1:e0XBRn3AptQotkyBFrHAxFB8mDhAIOfsG+7KyJ0dg98= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.34.4/go.mod h1:XclEty74bsGBCr1s0VSaA11hQ4ZidK4viWK7rRfO88I= +github.com/aws/aws-sdk-go-v2/service/sts v1.38.4 h1:PR00NXRYgY4FWHqOGx3fC3lhVKjsp1GdloDv2ynMSd8= +github.com/aws/aws-sdk-go-v2/service/sts v1.38.4/go.mod h1:Z+Gd23v97pX9zK97+tX4ppAgqCt3Z2dIXB02CtBncK8= +github.com/aws/smithy-go v1.23.0 h1:8n6I3gXzWJB2DxBDnfxgBaSX6oe0d/t10qGz7OKqMCE= +github.com/aws/smithy-go v1.23.0/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= @@ -61,8 +63,8 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/r github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/fasthttp/router v1.5.4 h1:oxdThbBwQgsDIYZ3wR1IavsNl6ZS9WdjKukeMikOnC8= github.com/fasthttp/router v1.5.4/go.mod h1:3/hysWq6cky7dTfzaaEPZGdptwjwx0qzTgFCKEWRjgc= -github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= -github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= @@ -70,14 +72,14 @@ github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9v github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= -github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 h1:BHT72Gu3keYf3ZEu2J0b1vyeLSOYI8bm5wbJM/8yDe8= -github.com/google/pprof v0.0.0-20250403155104-27863c87afa6/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= +github.com/google/pprof v0.0.0-20250820193118-f64d9cf942d6 h1:EEHtgt9IwisQ2AZ4pIsMjahcegHh6rmhqxzIRQIyepY= +github.com/google/pprof v0.0.0-20250820193118-f64d9cf942d6/go.mod h1:I6V7YzU0XDpsHqbsyrghnFZLO1gwK6NPTNvmetQIk9U= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgx/v5 v5.7.5 h1:JHGfMnQY+IEtGM63d+NGMjoRpysB2JBwDr5fsngwmJs= -github.com/jackc/pgx/v5 v5.7.5/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M= +github.com/jackc/pgx/v5 v5.7.6 h1:rWQc5FwZSPX58r1OQmkuaNicxdmExaEz5A2DO2hUuTk= +github.com/jackc/pgx/v5 v5.7.6/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= @@ -101,23 +103,23 @@ github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyua github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/onsi/ginkgo/v2 v2.23.4 h1:ktYTpKJAVZnDT4VjxSbiBenUjmlL/5QkBEocaWXiQus= -github.com/onsi/ginkgo/v2 v2.23.4/go.mod h1:Bt66ApGPBFzHyR+JO10Zbt0Gsp4uWxu5mIOTusL46e8= -github.com/onsi/gomega v1.38.0 h1:c/WX+w8SLAinvuKKQFh77WEucCnPk4j2OTUr7lt7BeY= -github.com/onsi/gomega v1.38.0/go.mod h1:OcXcwId0b9QsE7Y49u+BTrL4IdKOBOKnD6VQNTJEB6o= +github.com/onsi/ginkgo/v2 v2.25.3 h1:Ty8+Yi/ayDAGtk4XxmmfUy4GabvM+MegeB4cDLRi6nw= +github.com/onsi/ginkgo/v2 v2.25.3/go.mod h1:43uiyQC4Ed2tkOzLsEYm7hnrb7UJTWHYNsuy3bG/snE= +github.com/onsi/gomega v1.38.2 h1:eZCjf2xjZAqe+LeWvKb5weQ+NcPwX84kqJ0cZNxok2A= +github.com/onsi/gomega v1.38.2/go.mod h1:W2MJcYxRGV63b418Ai34Ud0hEdTVXq9NW9+Sx6uXf3k= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.23.0 h1:ust4zpdl9r4trLY/gSjlm07PuiBq2ynaXXlptpfy8Uc= -github.com/prometheus/client_golang v1.23.0/go.mod h1:i/o0R9ByOnHX0McrTMTyhYvKE4haaf2mW08I+jGAjEE= +github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= +github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= -github.com/prometheus/common v0.65.0 h1:QDwzd+G1twt//Kwj/Ww6E9FQq1iVMmODnILtW1t2VzE= -github.com/prometheus/common v0.65.0/go.mod h1:0gZns+BLRQ3V6NdaerOhMbwwRbNh9hkGINtQAsP5GS8= +github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs= +github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA= github.com/prometheus/procfs v0.17.0 h1:FuLQ+05u4ZI+SS/w9+BWEM2TXiHKsUQ9TADiRH7DuK0= github.com/prometheus/procfs v0.17.0/go.mod h1:oPQLaDAMRbA+u8H5Pbfq+dl3VDAvHxMUOVhe0wYB2zw= -github.com/redis/go-redis/v9 v9.12.1 h1:k5iquqv27aBtnTm2tIkROUDp8JBXhXZIVu1InSgvovg= -github.com/redis/go-redis/v9 v9.12.1/go.mod h1:huWgSWd8mW6+m0VPhJjSSQ+d6Nh1VICQ6Q5lHuCH/Iw= +github.com/redis/go-redis/v9 v9.14.0 h1:u4tNCjXOyzfgeLN+vAZaW1xUooqWDqVEsZN0U01jfAE= +github.com/redis/go-redis/v9 v9.14.0/go.mod h1:huWgSWd8mW6+m0VPhJjSSQ+d6Nh1VICQ6Q5lHuCH/Iw= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= @@ -128,12 +130,12 @@ github.com/savsgio/gotils v0.0.0-20250408102913-196191ec6287/go.mod h1:sM7Mt7uEo github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fasthttp v1.65.0 h1:j/u3uzFEGFfRxw79iYzJN+TteTJwbYkru9uDp3d0Yf8= -github.com/valyala/fasthttp v1.65.0/go.mod h1:P/93/YkKPMsKSnATEeELUCkG8a7Y+k99uxNHVbKINr4= +github.com/valyala/fasthttp v1.66.0 h1:M87A0Z7EayeyNaV6pfO3tUTUiYO0dZfEJnRGXTVNuyU= +github.com/valyala/fasthttp v1.66.0/go.mod h1:Y4eC+zwoocmXSVCB1JmhNbYtS7tZPRI2ztPB72EVObs= github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= github.com/zeebo/bencode v1.0.0 h1:zgop0Wu1nu4IexAZeCZ5qbsjU4O1vMrfCrVgUjbHVuA= @@ -142,25 +144,29 @@ go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs= go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= -golang.org/x/crypto v0.41.0 h1:WKYxWedPGCTVVl5+WHSSrOBT0O8lx32+zxmHxijgXp4= -golang.org/x/crypto v0.41.0/go.mod h1:pO5AFd7FA68rFak7rOAGVuygIISepHftHnr8dr6+sUc= -golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= -golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= -golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= -golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= +go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/crypto v0.42.0 h1:chiH31gIWm57EkTXpwnqf8qeuMUi0yekh6mT2AvFlqI= +golang.org/x/crypto v0.42.0/go.mod h1:4+rDnOTJhQCx2q7/j6rAN5XDw8kPjeaXEUR2eL94ix8= +golang.org/x/net v0.44.0 h1:evd8IRDyfNBMBTTY5XRF1vaZlD+EmWx6x8PkhR04H/I= +golang.org/x/net v0.44.0/go.mod h1:ECOoLqd5U3Lhyeyo/QDCEVQ4sNgYsqvCZ722XogGieY= +golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= +golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= -golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= -golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= -golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= -golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= -golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= -golang.org/x/tools v0.35.0 h1:mBffYraMEf7aa0sB+NuKnuCy8qI/9Bughn8dC2Gu5r0= -golang.org/x/tools v0.35.0/go.mod h1:NKdj5HkL/73byiZSJjqJgKn3ep7KjFkBOkR/Hps3VPw= -google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc= -google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= +golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k= +golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk= +golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4= +golang.org/x/time v0.13.0 h1:eUlYslOIt32DgYD6utsuUeHs4d7AsEYLuIAdg7FlYgI= +golang.org/x/time v0.13.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= +golang.org/x/tools v0.36.0 h1:kWS0uv/zsvHEle1LbV5LE8QujrxB3wfQyxHfhOk0Qkg= +golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s= +google.golang.org/protobuf v1.36.9 h1:w2gp2mA27hUeUzj9Ex9FBjsBm40zfaDtEWow293U7Iw= +google.golang.org/protobuf v1.36.9/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/middleware/torrentapproval/container/s3/s3.go b/middleware/torrentapproval/container/s3/s3.go index 586cd3a..9a688b3 100644 --- a/middleware/torrentapproval/container/s3/s3.go +++ b/middleware/torrentapproval/container/s3/s3.go @@ -8,7 +8,6 @@ import ( "fmt" "io" "iter" - "path/filepath" "strings" "time" @@ -40,6 +39,7 @@ type Config struct { SessionToken string Bucket string Prefix string + Suffix string Period time.Duration } @@ -94,6 +94,7 @@ func build(conf conf.MapConfig, st storage.DataStorage) (container.Container, er client: awss3.NewFromConfig(awsCfg), bucket: c.Bucket, prefix: c.Prefix, + suffix: c.Suffix, }, c.Period) go s.Run() @@ -110,8 +111,8 @@ type s3Client interface { } type s3 struct { - client s3Client - bucket, prefix string + client s3Client + bucket, prefix, suffix string } var _ directory.PathReader = s3{} @@ -125,15 +126,9 @@ func (s s3) ReadDir() (it iter.Seq[string], err error) { if err == nil { it = func(yield func(string) bool) { for _, e := range entries.Contents { - logger.Trace().Any("content", e).Msg("read dir") - if e.Key != nil && strings.ToLower(filepath.Ext(*e.Key)) == ".torrent" { - var name string - if len(s.prefix) == 0 { - name = *e.Key - } else { - name = filepath.Join(s.prefix, *e.Key) - } - if !yield(name) { + logger.Trace().Any("content", e).Msg("s3 read") + if e.Key != nil && strings.HasSuffix(*e.Key, s.suffix) { + if !yield(*e.Key) { return } } diff --git a/middleware/torrentapproval/container/s3/s3_test.go b/middleware/torrentapproval/container/s3/s3_test.go index a5012cb..ae9383e 100644 --- a/middleware/torrentapproval/container/s3/s3_test.go +++ b/middleware/torrentapproval/container/s3/s3_test.go @@ -125,6 +125,7 @@ func TestScanMock(t *testing.T) { var ( minioEndpoint = "http://127.0.0.1:9000" minioBucket = "test" + minioPrefix = "test/" ) const ( @@ -134,7 +135,7 @@ const ( ) // TestScanMinio requires real minio instance listening 127.0.0.1:9000 -// with default login/password (minioadmin/minioadmin) +// with default login, password and region (minioadmin/minioadmin, us-east-1) func TestScanMinio(t *testing.T) { st, _ := memory.Builder{}.NewDataStorage(make(conf.MapConfig)) awsCfg, err := config.LoadDefaultConfig(context.Background()) @@ -157,26 +158,59 @@ func TestScanMinio(t *testing.T) { if err != nil { t.Fatal(err) } + name = minioPrefix + name + _, err = cl.PutObject(context.Background(), &awss3.PutObjectInput{ + Bucket: &minioBucket, + Key: &name, + Body: bytes.NewReader(data.data), + }) + if err != nil { + t.Fatal(err) + } + name += "1" + _, err = cl.PutObject(context.Background(), &awss3.PutObjectInput{ + Bucket: &minioBucket, + Key: &name, + Body: bytes.NewReader(data.data), + }) + if err != nil { + t.Fatal(err) + } + } + + s3Dir := s3{ + client: cl, + bucket: minioBucket, + prefix: minioPrefix, + suffix: ".torrent", + } + + if content, err := s3Dir.ReadDir(); err == nil { + var i int + for range content { + i++ + } + require.Equal(t, len(files), i, "S3 content data not the same as test data") + } else { + t.Fatal(err) } d := directory.NewScanner(list.List{ Invert: false, Storage: st, StorageCtx: "TEST", - }, s3{ - client: cl, - bucket: minioBucket, - }, time.Millisecond*100) + }, s3Dir, time.Millisecond*100) + go d.Run() t.Cleanup(func() { _ = d.Close() }) - time.Sleep(time.Millisecond * 200) for name, f := range files { contains, _ := d.List.Storage.Contains(context.Background(), "TEST", f.hash) require.True(t, contains, "%s must present", name) + name = minioPrefix + name _, err = cl.DeleteObject(context.Background(), &awss3.DeleteObjectInput{ Bucket: &minioBucket, Key: &name, From 7b80ed199a7793efeb9539fdcf42e62514b4523c Mon Sep 17 00:00:00 2001 From: "Lawrence, Rendall" Date: Mon, 22 Sep 2025 15:02:33 +0300 Subject: [PATCH 05/13] update actions to go ver 1.24 --- .github/workflows/build.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index fcf9edb..62208ca 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -15,7 +15,7 @@ jobs: - uses: "actions/checkout@v4" - uses: "actions/setup-go@v5" with: - go-version: ">=1.23" + go-version: ">=1.24" - name: "Build" run: "go build ./cmd/..." @@ -49,7 +49,7 @@ jobs: - uses: "actions/checkout@v4" - uses: "actions/setup-go@v5" with: - go-version: "^1.23" + go-version: "^1.24" - name: "Run `go test`" run: "go test -race ./..." @@ -60,7 +60,7 @@ jobs: - uses: "actions/checkout@v4" - uses: "actions/setup-go@v5" with: - go-version: "^1.23" + go-version: "^1.24" - name: "Install and configure mochi" run: | go install ./cmd/mochi @@ -86,7 +86,7 @@ jobs: - uses: "actions/checkout@v4" - uses: "actions/setup-go@v5" with: - go-version: "^1.23" + go-version: "^1.24" - name: "Install and configure mochi" run: | go install ./cmd/mochi From febe60b069bc1045c448c4a0fb50365bc1e923f0 Mon Sep 17 00:00:00 2001 From: "Lawrence, Rendall" Date: Mon, 22 Sep 2025 15:49:50 +0300 Subject: [PATCH 06/13] add documentation about S3 container, fix S3 configration --- docs/middleware/torrent_approval.md | 21 +++++++++- middleware/torrentapproval/container/s3/s3.go | 38 +++++++++++++------ .../torrentapproval/container/s3/s3_test.go | 2 +- 3 files changed, 48 insertions(+), 13 deletions(-) diff --git a/docs/middleware/torrent_approval.md b/docs/middleware/torrent_approval.md index 22c6568..95e4a80 100644 --- a/docs/middleware/torrent_approval.md +++ b/docs/middleware/torrent_approval.md @@ -28,6 +28,11 @@ There are two sources of hashes: `list` and `directory`. files at start and then periodically watch for new files to add, or for delete events to remove hash from storage. +* `s3` will search for torrent files in specified S3-compatible storage (AWS S3, MinIO, etc.) and + append/delete records from storage. This source will parse all existing + files at start and then periodically watch for new files to add, or for delete events + to remove hash from storage. + Note: if storage is not `memory`, and `preserve` option set to `true`, records will be persisted in storage until _somebody_ or _something_ (different tool with access to storage) won't delete it. @@ -48,7 +53,21 @@ If `name` is empty or `internal` global storage will be used - `directory`: - `path` - directory to watch - `period` - time between two directory checks - - `invert` and `storage_ctx` has the same meanins as `list`'s options + - `invert` and `storage_ctx` has the same meaning as `list`'s options + - `s3`: + - `endpoint`* - base URL of S3 provider + - `region`* - S3 region to connect to + - `key_id`* - S3 access key ID + - `key_secret`* - S3 secret access key + - `session_token`* - S3 temporary security credential + - `bucket` - S3 bucket + - `prefix` - prefix path to search entries + - `suffix` - suffix to filter returned entries, such as extension (e.g. `.torrent`) + - `period` - time between two S3 checks + - `invert` and `storage_ctx` has the same meaning as `list`'s options + +Note: `s3` options marked with `*` and any other specific options can be omitted in MoChi and can be provided +with environment variables or in `$HOME/.aws/*` files (see [AWS SDK documentation](https://docs.aws.amazon.com/sdk-for-go/v2/developer-guide/configure-gosdk.html)). Configuration example: diff --git a/middleware/torrentapproval/container/s3/s3.go b/middleware/torrentapproval/container/s3/s3.go index 9a688b3..d9529be 100644 --- a/middleware/torrentapproval/container/s3/s3.go +++ b/middleware/torrentapproval/container/s3/s3.go @@ -5,6 +5,7 @@ package s3 import ( "context" + "errors" "fmt" "io" "iter" @@ -34,9 +35,9 @@ type Config struct { list.Config Endpoint string Region string - KeyID string - KeySecret string - SessionToken string + KeyID string `cfg:"key_id"` + KeySecret string `cfg:"key_secret"` + SessionToken string `cfg:"session_token"` Bucket string Prefix string Suffix string @@ -52,6 +53,11 @@ func build(conf conf.MapConfig, st storage.DataStorage) (container.Container, er if err := conf.Unmarshal(c); err != nil { return nil, fmt.Errorf("unable to deserialise configuration: %w", err) } + + if len(c.Bucket) == 0 { + return nil, errors.New("no bucket provided") + } + if c.Period == 0 { logger.Warn(). Str("name", "Period"). @@ -61,25 +67,35 @@ func build(conf conf.MapConfig, st storage.DataStorage) (container.Container, er c.Period = defaultPeriod } - awsCfg, err := config.LoadDefaultConfig(context.Background()) - if err != nil { - return nil, fmt.Errorf("unable load AWS S3 SDK configuration: %w", err) - } + modifiers := make([]func(*config.LoadOptions) error, 1, 4) - awsCfg.Logger = logging.LoggerFunc(func(classification logging.Classification, format string, v ...interface{}) { + modifiers[0] = config.WithLogger(logging.LoggerFunc(func( + classification logging.Classification, format string, v ...interface{}, + ) { if classification == logging.Debug { logger.Debug().CallerSkipFrame(1).Msg(fmt.Sprintf(format, v...)) } else if classification == logging.Warn { logger.Warn().CallerSkipFrame(1).Msg(fmt.Sprintf(format, v...)) } - }) + })) if len(c.Endpoint) > 0 { - awsCfg.BaseEndpoint = &c.Endpoint + modifiers = append(modifiers, config.WithBaseEndpoint(c.Endpoint)) } if len(c.Region) > 0 { - awsCfg.Region = c.Region + modifiers = append(modifiers, config.WithRegion(c.Endpoint)) + } + + if len(c.KeyID) > 0 || len(c.KeySecret) > 0 || len(c.SessionToken) > 0 { + modifiers = append(modifiers, config.WithCredentialsProvider( + credentials.NewStaticCredentialsProvider(c.KeyID, c.KeySecret, c.SessionToken)), + ) + } + + awsCfg, err := config.LoadDefaultConfig(context.Background(), modifiers...) + if err != nil { + return nil, fmt.Errorf("unable load AWS S3 SDK configuration: %w", err) } if len(c.KeyID) > 0 || len(c.KeySecret) > 0 || len(c.SessionToken) > 0 { diff --git a/middleware/torrentapproval/container/s3/s3_test.go b/middleware/torrentapproval/container/s3/s3_test.go index ae9383e..b26974f 100644 --- a/middleware/torrentapproval/container/s3/s3_test.go +++ b/middleware/torrentapproval/container/s3/s3_test.go @@ -57,7 +57,7 @@ emDE8BsT0R1/0GVl`), } func init() { - _ = log.ConfigureLogger("", "debug", false, false) + _ = log.ConfigureLogger("", "warn", false, false) } type mockS3 struct { From 3f78a01cf1171b8a97bb67d6e1eb8bdda193feb2 Mon Sep 17 00:00:00 2001 From: "Lawrence, Rendall" Date: Mon, 22 Sep 2025 16:03:04 +0300 Subject: [PATCH 07/13] (minor) fix lint warnings --- .../container/directory/directory_test.go | 9 ++++----- middleware/torrentapproval/container/s3/s3.go | 5 +++-- middleware/torrentapproval/container/s3/s3_test.go | 8 ++++---- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/middleware/torrentapproval/container/directory/directory_test.go b/middleware/torrentapproval/container/directory/directory_test.go index 4292700..387ba7b 100644 --- a/middleware/torrentapproval/container/directory/directory_test.go +++ b/middleware/torrentapproval/container/directory/directory_test.go @@ -64,9 +64,8 @@ func writeTmp() (string, error) { return "", err } for _, f := range files { - err = os.WriteFile(filepath.Join(tmpDir, f.name), f.data, 0644) - if err != nil { - return "", err + if err = os.WriteFile(filepath.Join(tmpDir, f.name), f.data, 0o600); err != nil { + break } } return tmpDir, err @@ -96,14 +95,14 @@ func TestScan(t *testing.T) { }) time.Sleep(time.Millisecond * 100) for _, f := range files { - contains, _ := d.List.Storage.Contains(context.Background(), "TEST", f.hash) + contains, _ := d.Storage.Contains(context.Background(), "TEST", f.hash) require.True(t, contains, "%s must present", f.name) _ = os.Remove(filepath.Join(tmpDir, f.name)) } time.Sleep(time.Millisecond * 100) for _, f := range files { - contains, _ := d.List.Storage.Contains(context.Background(), "TEST", f.hash) + contains, _ := d.Storage.Contains(context.Background(), "TEST", f.hash) require.False(t, contains, "%s must absent", f.name) } } diff --git a/middleware/torrentapproval/container/s3/s3.go b/middleware/torrentapproval/container/s3/s3.go index d9529be..67ae05b 100644 --- a/middleware/torrentapproval/container/s3/s3.go +++ b/middleware/torrentapproval/container/s3/s3.go @@ -72,9 +72,10 @@ func build(conf conf.MapConfig, st storage.DataStorage) (container.Container, er modifiers[0] = config.WithLogger(logging.LoggerFunc(func( classification logging.Classification, format string, v ...interface{}, ) { - if classification == logging.Debug { + switch classification { + case logging.Debug: logger.Debug().CallerSkipFrame(1).Msg(fmt.Sprintf(format, v...)) - } else if classification == logging.Warn { + case logging.Warn: logger.Warn().CallerSkipFrame(1).Msg(fmt.Sprintf(format, v...)) } })) diff --git a/middleware/torrentapproval/container/s3/s3_test.go b/middleware/torrentapproval/container/s3/s3_test.go index b26974f..c7b40ce 100644 --- a/middleware/torrentapproval/container/s3/s3_test.go +++ b/middleware/torrentapproval/container/s3/s3_test.go @@ -106,7 +106,7 @@ func TestScanMock(t *testing.T) { time.Sleep(time.Millisecond * 100) for name, f := range files { - contains, _ := d.List.Storage.Contains(context.Background(), "TEST", f.hash) + contains, _ := d.Storage.Contains(context.Background(), "TEST", f.hash) require.True(t, contains, "%s must present", name) for i := 0; i < len(cl.objs); i++ { if *cl.objs[i].Key == name { @@ -117,7 +117,7 @@ func TestScanMock(t *testing.T) { time.Sleep(time.Millisecond * 100) for name, f := range files { - contains, _ := d.List.Storage.Contains(context.Background(), "TEST", f.hash) + contains, _ := d.Storage.Contains(context.Background(), "TEST", f.hash) require.False(t, contains, "%s must absent", name) } } @@ -208,7 +208,7 @@ func TestScanMinio(t *testing.T) { time.Sleep(time.Millisecond * 200) for name, f := range files { - contains, _ := d.List.Storage.Contains(context.Background(), "TEST", f.hash) + contains, _ := d.Storage.Contains(context.Background(), "TEST", f.hash) require.True(t, contains, "%s must present", name) name = minioPrefix + name _, err = cl.DeleteObject(context.Background(), &awss3.DeleteObjectInput{ @@ -223,7 +223,7 @@ func TestScanMinio(t *testing.T) { time.Sleep(time.Millisecond * 200) for name, f := range files { - contains, _ := d.List.Storage.Contains(context.Background(), "TEST", f.hash) + contains, _ := d.Storage.Contains(context.Background(), "TEST", f.hash) require.False(t, contains, "%s must absent", name) } } From 3807371a6ce8393e834a1df6d918e686fe587edb Mon Sep 17 00:00:00 2001 From: "Lawrence, Rendall" Date: Mon, 22 Sep 2025 16:04:55 +0300 Subject: [PATCH 08/13] update golangci go version to 1.24 --- .golangci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.golangci.yaml b/.golangci.yaml index f95bb7c..3398a29 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -1,6 +1,6 @@ version: "2" run: - go: "1.23" + go: "1.24" linters: enable: - bidichk From 61f358c1b24eb14ad9747dec558429f63a093634 Mon Sep 17 00:00:00 2001 From: "Lawrence, Rendall" Date: Mon, 22 Sep 2025 16:36:33 +0300 Subject: [PATCH 09/13] fix minio github action --- .github/workflows/build.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 62208ca..47bbeb5 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -39,12 +39,12 @@ jobs: --health-timeout 5s --health-retries 5 minio: - image: "minio/minio" + image: "lazybit/minio" ports: [ "9000:9000" ] - options: >- - server - --address - 127.0.0.1:9000 + env: + MINIO_ACCESS_KEY: "minioadmin" + MINIO_SECRET_KEY: "minioadmin" + options: server /data steps: - uses: "actions/checkout@v4" - uses: "actions/setup-go@v5" From 5513f1357d00d71e0d763a85f67d517af8f3474a Mon Sep 17 00:00:00 2001 From: "Lawrence, Rendall" Date: Mon, 22 Sep 2025 17:11:05 +0300 Subject: [PATCH 10/13] fix lint warnings replace naked returns with arguments (gofumpt@v0.9.1) --- .github/workflows/build.yaml | 2 +- bittorrent/event.go | 4 +- bittorrent/request.go | 2 +- cmd/mochi/server_test.go | 2 +- frontend/frontend.go | 4 +- frontend/http/frontend.go | 6 +- frontend/http/frontend_test.go | 2 +- frontend/http/parser.go | 4 +- frontend/options.go | 6 +- frontend/udp/frontend.go | 20 +++---- middleware/hooks.go | 12 ++-- middleware/jwt/jwt.go | 4 +- middleware/jwt/jwt_test.go | 2 +- middleware/logic.go | 2 +- middleware/middleware.go | 2 +- middleware/torrentapproval/container/s3/s3.go | 2 +- middleware/torrentapproval/torrentapproval.go | 2 +- middleware/varinterval/varinterval.go | 4 +- pkg/conf/config.go | 2 +- pkg/timecache/timecache.go | 2 +- storage/keydb/storage.go | 2 +- storage/mdb/storage.go | 60 +++++++++---------- storage/memory/storage.go | 18 +++--- storage/pg/storage.go | 28 ++++----- storage/redis/storage.go | 40 ++++++------- storage/storage.go | 10 ++-- storage/test/storage_bench.go | 4 +- storage/test/storage_test_data.go | 4 +- 28 files changed, 126 insertions(+), 126 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 47bbeb5..855e050 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -39,7 +39,7 @@ jobs: --health-timeout 5s --health-retries 5 minio: - image: "lazybit/minio" + image: "minio/minio" ports: [ "9000:9000" ] env: MINIO_ACCESS_KEY: "minioadmin" diff --git a/bittorrent/event.go b/bittorrent/event.go index bb54ed1..08ac70b 100644 --- a/bittorrent/event.go +++ b/bittorrent/event.go @@ -52,7 +52,7 @@ func NewEvent(eventStr string) (evt Event, err error) { default: evt, err = None, ErrUnknownEvent } - return + return evt, err } // String implements Stringer for an event. @@ -69,5 +69,5 @@ func (e Event) String() (s string) { default: s = "" } - return + return s } diff --git a/bittorrent/request.go b/bittorrent/request.go index 57a46d5..576c20e 100644 --- a/bittorrent/request.go +++ b/bittorrent/request.go @@ -133,7 +133,7 @@ func (rp RequestPeer) Peers() (peers Peers) { AddrPort: netip.AddrPortFrom(a.Addr, rp.Port), }) } - return + return peers } // MarshalZerologObject writes fields into zerolog event diff --git a/cmd/mochi/server_test.go b/cmd/mochi/server_test.go index 757a11f..4b86ab2 100644 --- a/cmd/mochi/server_test.go +++ b/cmd/mochi/server_test.go @@ -242,7 +242,7 @@ func sendHTTPReq(u string) (err error) { return errors.New(r.Status) } } - return + return err } func BenchmarkServerHTTPAnnounce(b *testing.B) { diff --git a/frontend/frontend.go b/frontend/frontend.go index c2f5fda..85140e1 100644 --- a/frontend/frontend.go +++ b/frontend/frontend.go @@ -71,7 +71,7 @@ func NewFrontends(configs []conf.NamedMapConfig, logic *middleware.Logic) (fs [] fs = append(fs, f) logger.Info().Str("name", c.Name).Msg("frontend started") } - return + return fs, err } // CloseGroup simultaneously calls Close for each non-nil @@ -102,5 +102,5 @@ func CloseGroup(cls []io.Closer) (err error) { if sb.Len() > 0 { err = errors.New(sb.String()) } - return + return err } diff --git a/frontend/http/frontend.go b/frontend/http/frontend.go index b9e42f1..274d664 100644 --- a/frontend/http/frontend.go +++ b/frontend/http/frontend.go @@ -69,7 +69,7 @@ func (cfg Config) Validate() (validCfg Config, err error) { validCfg.ListenOptions = cfg.ListenOptions.Validate(logger) if cfg.UseTLS && (len(cfg.TLSCertPath) == 0 || len(cfg.TLSKeyPath) == 0) { err = errTLSNotProvided - return + return validCfg, err } if cfg.ReadTimeout <= 0 { @@ -118,7 +118,7 @@ func (cfg Config) Validate() (validCfg Config, err error) { Msg("falling back to default configuration") } validCfg.ParseOptions.ParseOptions = cfg.ParseOptions.Validate(logger) - return + return validCfg, err } type httpFE struct { @@ -232,7 +232,7 @@ func (f *httpFE) Close() (err error) { } }) - return + return err } // announceRoute parses and responds to an Announce. diff --git a/frontend/http/frontend_test.go b/frontend/http/frontend_test.go index 16fb041..9bba1d9 100644 --- a/frontend/http/frontend_test.go +++ b/frontend/http/frontend_test.go @@ -77,7 +77,7 @@ func runGet(u string, checkResponse bool) (err error) { return errors.New(r.Status) } } - return + return err } func BenchmarkPing(b *testing.B) { diff --git a/frontend/http/parser.go b/frontend/http/parser.go index 103d67b..89021d4 100644 --- a/frontend/http/parser.go +++ b/frontend/http/parser.go @@ -158,12 +158,12 @@ func requestedIPs(r *fasthttp.RequestCtx, p *queryParams, opts ParseOptions) (ad Provided: false, }) } - return + return addresses } func parseRequestAddress(s string, provided bool) (ra bittorrent.RequestAddress) { if addr, err := netip.ParseAddr(s); err == nil { ra.Addr, ra.Provided = addr, provided } - return + return ra } diff --git a/frontend/options.go b/frontend/options.go index 5344ec8..6c2a5f7 100644 --- a/frontend/options.go +++ b/frontend/options.go @@ -35,7 +35,7 @@ func (lo ListenOptions) Validate(logger *log.Logger) (validOptions ListenOptions Str("default", validOptions.Addr). Msg("falling back to default configuration") } - return + return validOptions } // ListenTCP listens at the given TCP Addr @@ -56,7 +56,7 @@ func (lo ListenOptions) ListenTCP() (conn *net.TCPListener, err error) { conn, err = net.ListenTCP("tcp", addr) } } - return + return conn, err } // ListenUDP listens at the given UDP Addr @@ -77,7 +77,7 @@ func (lo ListenOptions) ListenUDP() (conn *net.UDPConn, err error) { conn, err = net.ListenUDP("udp", addr) } } - return + return conn, err } // ParseOptions is the configuration used to parse an Announce Request. diff --git a/frontend/udp/frontend.go b/frontend/udp/frontend.go index 78a39c7..84b1d85 100644 --- a/frontend/udp/frontend.go +++ b/frontend/udp/frontend.go @@ -96,7 +96,7 @@ func (cfg Config) Validate() (validCfg Config) { validCfg.ParseOptions = cfg.ParseOptions.Validate(logger) - return + return validCfg } // udpFE holds the state of a UDP BitTorrent Frontend. @@ -174,7 +174,7 @@ func (f *udpFE) Close() (err error) { err = frontend.CloseGroup(cls) }) - return + return err } // serve blocks while listening and serving UDP BitTorrent requests @@ -257,7 +257,7 @@ func (f *udpFE) handleRequest(ctx context.Context, r Request, w ResponseWriter) // Malformed, no client packets are less than 16 bytes. // We explicitly return nothing in case this is a DoS attempt. err = errMalformedPacket - return + return actionName, err } // Parse the headers of the UDP packet. @@ -274,7 +274,7 @@ func (f *udpFE) handleRequest(ctx context.Context, r Request, w ResponseWriter) if actionID != connectActionID && !gen.Validate(connID, r.IP, timecache.Now()) { err = errBadConnectionID writeErrorResponse(w, txID, err) - return + return actionName, err } // Handle the requested action. @@ -284,7 +284,7 @@ func (f *udpFE) handleRequest(ctx context.Context, r Request, w ResponseWriter) if !bytes.Equal(connID, initialConnectionID) { err = errMalformedPacket - return + return actionName, err } writeConnectionID(w, txID, gen.Generate(r.IP, timecache.Now())) @@ -296,7 +296,7 @@ func (f *udpFE) handleRequest(ctx context.Context, r Request, w ResponseWriter) req, err = parseAnnounce(r, actionID == announceV6ActionID, f.ParseOptions) if err != nil { writeErrorResponse(w, txID, err) - return + return actionName, err } var resp *bittorrent.AnnounceResponse @@ -306,7 +306,7 @@ func (f *udpFE) handleRequest(ctx context.Context, r Request, w ResponseWriter) if !errors.Is(err, context.Canceled) { writeErrorResponse(w, txID, err) } - return + return actionName, err } if err = ctx.Err(); err == nil { @@ -323,7 +323,7 @@ func (f *udpFE) handleRequest(ctx context.Context, r Request, w ResponseWriter) req, err = parseScrape(r, f.ParseOptions) if err != nil { writeErrorResponse(w, txID, err) - return + return actionName, err } var resp *bittorrent.ScrapeResponse @@ -333,7 +333,7 @@ func (f *udpFE) handleRequest(ctx context.Context, r Request, w ResponseWriter) if !errors.Is(err, context.Canceled) { writeErrorResponse(w, txID, err) } - return + return actionName, err } if err = ctx.Err(); err == nil { @@ -348,5 +348,5 @@ func (f *udpFE) handleRequest(ctx context.Context, r Request, w ResponseWriter) writeErrorResponse(w, txID, err) } - return + return actionName, err } diff --git a/middleware/hooks.go b/middleware/hooks.go index f307110..d4e9364 100644 --- a/middleware/hooks.go +++ b/middleware/hooks.go @@ -43,7 +43,7 @@ type swarmInteractionHook struct { func (h *swarmInteractionHook) HandleAnnounce(ctx context.Context, req *bittorrent.AnnounceRequest, _ *bittorrent.AnnounceResponse) (outCtx context.Context, err error) { outCtx = ctx if ctx.Value(SkipSwarmInteractionKey) != nil { - return + return outCtx, err } var storeFn func(context.Context, bittorrent.InfoHash, bittorrent.Peer) error @@ -82,7 +82,7 @@ func (h *swarmInteractionHook) HandleAnnounce(ctx context.Context, req *bittorre } } - return + return outCtx, err } func (h *swarmInteractionHook) HandleScrape(ctx context.Context, _ *bittorrent.ScrapeRequest, _ *bittorrent.ScrapeResponse) (context.Context, error) { @@ -105,17 +105,17 @@ type responseHook struct { func (h *responseHook) scrape(ctx context.Context, ih bittorrent.InfoHash) (leechers uint32, seeders uint32, snatched uint32, err error) { leechers, seeders, snatched, err = h.store.ScrapeSwarm(ctx, ih) if err != nil { - return + return leechers, seeders, snatched, err } if len(ih) == bittorrent.InfoHashV2Len { var l, s, n uint32 l, s, n, err = h.store.ScrapeSwarm(ctx, ih.TruncateV1()) if err != nil { - return + return leechers, seeders, snatched, err } leechers, seeders, snatched = leechers+l, seeders+s, snatched+n } - return + return leechers, seeders, snatched, err } func (h *responseHook) HandleAnnounce(ctx context.Context, req *bittorrent.AnnounceRequest, resp *bittorrent.AnnounceResponse) (_ context.Context, err error) { @@ -209,7 +209,7 @@ func (h *responseHook) appendPeers(ctx context.Context, req *bittorrent.Announce } } - return + return err } func (h *responseHook) HandleScrape(ctx context.Context, req *bittorrent.ScrapeRequest, resp *bittorrent.ScrapeResponse) (_ context.Context, err error) { diff --git a/middleware/jwt/jwt.go b/middleware/jwt/jwt.go index f80ff22..5c91d4f 100644 --- a/middleware/jwt/jwt.go +++ b/middleware/jwt/jwt.go @@ -113,7 +113,7 @@ func build(config conf.MapConfig, _ storage.PeerStorage) (h middleware.Hook, err err = errJWKsNotSet } - return + return h, err } type announceClaims struct { @@ -237,5 +237,5 @@ func (h *hook) getJWTString(params bittorrent.Params) (jwt string) { } } } - return + return jwt } diff --git a/middleware/jwt/jwt_test.go b/middleware/jwt/jwt_test.go index 3271bfc..4c95742 100644 --- a/middleware/jwt/jwt_test.go +++ b/middleware/jwt/jwt_test.go @@ -56,7 +56,7 @@ type params map[string]string func (p params) GetString(key string) (out string, found bool) { out, found = p[key] - return + return out, found } func (params) MarshalZerologObject(*zerolog.Event) {} diff --git a/middleware/logic.go b/middleware/logic.go index 27d500c..9822c15 100644 --- a/middleware/logic.go +++ b/middleware/logic.go @@ -113,5 +113,5 @@ func (l *Logic) Ping(ctx context.Context) (err error) { break } } - return + return err } diff --git a/middleware/middleware.go b/middleware/middleware.go index 0f6c3b9..4b8c878 100644 --- a/middleware/middleware.go +++ b/middleware/middleware.go @@ -62,5 +62,5 @@ func NewHooks(configs []conf.NamedMapConfig, storage storage.PeerStorage) (hooks logger.Info().Str("name", c.Name).Msg("hook started") } - return + return hooks, err } diff --git a/middleware/torrentapproval/container/s3/s3.go b/middleware/torrentapproval/container/s3/s3.go index 67ae05b..6b4e6cc 100644 --- a/middleware/torrentapproval/container/s3/s3.go +++ b/middleware/torrentapproval/container/s3/s3.go @@ -164,5 +164,5 @@ func (s s3) ReadData(entry string) (data io.ReadCloser, err error) { if err == nil { data = result.Body } - return + return data, err } diff --git a/middleware/torrentapproval/torrentapproval.go b/middleware/torrentapproval/torrentapproval.go index 7be410c..18fe9a2 100644 --- a/middleware/torrentapproval/torrentapproval.go +++ b/middleware/torrentapproval/torrentapproval.go @@ -66,7 +66,7 @@ func build(config conf.MapConfig, st storage.PeerStorage) (h middleware.Hook, er } else if ds, err = storage.NewDataStorage(cfg.Storage); err == nil { dsc = ds } else { - return + return h, err } var c container.Container diff --git a/middleware/varinterval/varinterval.go b/middleware/varinterval/varinterval.go index 02b4f7e..73265c5 100644 --- a/middleware/varinterval/varinterval.go +++ b/middleware/varinterval/varinterval.go @@ -36,7 +36,7 @@ func build(config conf.MapConfig, _ storage.PeerStorage) (h middleware.Hook, err } } } - return + return h, err } var ( @@ -112,5 +112,5 @@ func deriveEntropyFromRequest(req *bittorrent.AnnounceRequest) (v0 uint64, v1 ui v0 = binary.BigEndian.Uint64([]byte(req.InfoHash[:8])) + binary.BigEndian.Uint64([]byte(req.InfoHash[8:16])) } v1 = binary.BigEndian.Uint64(req.ID[:8]) + binary.BigEndian.Uint64(req.ID[8:16]) - return + return v0, v1 } diff --git a/pkg/conf/config.go b/pkg/conf/config.go index 2b84325..f2fbb43 100644 --- a/pkg/conf/config.go +++ b/pkg/conf/config.go @@ -49,7 +49,7 @@ func (m MapConfig) Unmarshal(into any) (err error) { } else { err = ErrNilConfigMap } - return + return err } // NamedMapConfig encapsulates MapConfig with string Name diff --git a/pkg/timecache/timecache.go b/pkg/timecache/timecache.go index d7050fb..2714e4f 100644 --- a/pkg/timecache/timecache.go +++ b/pkg/timecache/timecache.go @@ -38,7 +38,7 @@ type TimeCache struct { func New() (tc *TimeCache) { tc = &TimeCache{closed: make(chan struct{})} tc.clock.Store(time.Now().UnixNano()) - return + return tc } // Run runs the TimeCache, updating the cached clock value once every interval diff --git a/storage/keydb/storage.go b/storage/keydb/storage.go index d5fddd0..2d1e582 100644 --- a/storage/keydb/storage.go +++ b/storage/keydb/storage.go @@ -108,7 +108,7 @@ func (s *store) addPeer(ctx context.Context, infoHashKey, peerID string) (err er if err = s.SAdd(ctx, infoHashKey, peerID).Err(); err == nil { err = s.Process(ctx, redis.NewCmd(ctx, expireMemberCmd, infoHashKey, peerID, s.peerTTL)) } - return + return err } func (s *store) delPeer(ctx context.Context, infoHashKey, peerID string) error { diff --git a/storage/mdb/storage.go b/storage/mdb/storage.go index b8b406c..f749362 100644 --- a/storage/mdb/storage.go +++ b/storage/mdb/storage.go @@ -181,14 +181,14 @@ func newStorage(cfg config) (*mdb, error) { dataDB, err = txn.OpenRoot(0) } if err != nil { - return + return err } if len(cfg.PeersDBName) > 0 { peersDB, err = txn.CreateDBI(cfg.PeersDBName) } else { peersDB, err = txn.OpenRoot(0) } - return + return err }); err != nil { _ = env.Close() return nil, err @@ -216,7 +216,7 @@ func (m *mdb) Close() (err error) { err = m.lmdbEnv.Close() } }) - return + return err } const keySeparator = '_' @@ -256,31 +256,31 @@ func (m *mdb) Put(_ context.Context, storeCtx string, values ...storage.Entry) ( break } } - return + return err }) } - return + return err } func (m *mdb) Contains(_ context.Context, storeCtx string, key string) (contains bool, err error) { err = m.View(func(txn *lmdb.Txn) (err error) { _, err = txn.Get(m.dataDB, composeKey(storeCtx, key)) - return + return err }) if err == nil { contains = true } else if lmdb.IsNotFound(err) { err = nil } - return + return contains, err } func (m *mdb) Load(_ context.Context, storeCtx string, key string) (v []byte, err error) { err = m.View(func(txn *lmdb.Txn) (err error) { v, err = ignoreNotFoundData(txn.Get(m.dataDB, composeKey(storeCtx, key))) - return + return err }) - return + return v, err } func (m *mdb) Delete(_ context.Context, storeCtx string, keys ...string) (err error) { @@ -291,10 +291,10 @@ func (m *mdb) Delete(_ context.Context, storeCtx string, keys ...string) (err er break } } - return + return err }) } - return + return err } const ( @@ -324,7 +324,7 @@ func unpackPeer(arr []byte) (peer bittorrent.Peer) { AddrPort: netip.AddrPortFrom(netip.AddrFrom16([ipLen]byte(arr[bittorrent.PeerIDLen:])).Unmap(), binary.BigEndian.Uint16(arr[bittorrent.PeerIDLen+ipLen:])), } - return + return peer } func composeIHKeyPrefix(ih []byte, seeder bool, v6 bool, suffixLen int) (ihKey []byte, suffixStart int) { @@ -343,13 +343,13 @@ func composeIHKeyPrefix(ih []byte, seeder bool, v6 bool, suffixLen int) (ihKey [ ihKey[2], ihKey[ihLen+3] = keySeparator, keySeparator copy(ihKey[3:], ih) suffixStart = len(ihKey) - suffixLen - return + return ihKey, suffixStart } func composeIHKey(ih bittorrent.InfoHash, peer bittorrent.Peer, seeder bool) (ihKey []byte) { ihKey, start := composeIHKeyPrefix(ih.Bytes(), seeder, peer.Addr().Is6(), packedPeerLen) packPeer(peer, ihKey[start:]) - return + return ihKey } func (m *mdb) putPeer(ih bittorrent.InfoHash, peer bittorrent.Peer, seeder bool) error { @@ -359,7 +359,7 @@ func (m *mdb) putPeer(ih bittorrent.InfoHash, peer bittorrent.Peer, seeder bool) if b, err = txn.PutReserve(m.peersDB, ihKey, 8, 0); err == nil { binary.BigEndian.PutUint64(b, uint64(timecache.NowUnix())) } - return + return err }) } @@ -390,12 +390,12 @@ func (m *mdb) GraduateLeecher(_ context.Context, ih bittorrent.InfoHash, peer bi ihKey := composeIHKey(ih, peer, false) return m.Update(func(txn *lmdb.Txn) (err error) { if err = ignoreNotFound(txn.Del(m.peersDB, ihKey, nil)); err != nil { - return + return err } ihKey[0] = seederPrefix var b []byte if b, err = txn.PutReserve(m.peersDB, ihKey, 8, 0); err != nil { - return + return err } binary.BigEndian.PutUint64(b, uint64(timecache.NowUnixNano())) @@ -403,7 +403,7 @@ func (m *mdb) GraduateLeecher(_ context.Context, ih bittorrent.InfoHash, peer bi ihPrefix[0], ihPrefix[1] = downloadedPrefix, countPrefix var v int if b, err = ignoreNotFoundData(txn.Get(m.peersDB, ihPrefix)); err != nil { - return + return err } if len(b) >= 4 { v = int(binary.BigEndian.Uint32(b)) @@ -412,7 +412,7 @@ func (m *mdb) GraduateLeecher(_ context.Context, ih bittorrent.InfoHash, peer bi if b, err = txn.PutReserve(m.peersDB, ihPrefix, 4, 0); err == nil { binary.BigEndian.PutUint32(b, uint32(v)) } - return + return err }) } @@ -450,11 +450,11 @@ func (m *mdb) scanPeers(ctx context.Context, prefix []byte, readRaw bool, fn fun err = scanner.Err() } scanner.Close() - return + return err }) m.wg.Done() - return + return err } func (m *mdb) AnnouncePeers(ctx context.Context, ih bittorrent.InfoHash, forSeeder bool, numWant int, v6 bool) (peers []bittorrent.Peer, err error) { @@ -474,7 +474,7 @@ func (m *mdb) AnnouncePeers(ctx context.Context, ih bittorrent.InfoHash, forSeed err = m.scanPeers(ctx, prefix, true, appendFn) } } - return + return peers, err } func (m *mdb) countPeers(ctx context.Context, scanPrefix []byte) (cnt uint32, err error) { @@ -509,35 +509,35 @@ func (m *mdb) countPeers(ctx context.Context, scanPrefix []byte) (cnt uint32, er } err = scanner.Err() scanner.Close() - return + return err }) m.wg.Done() - return + return cnt, err } func (m *mdb) ScrapeSwarm(ctx context.Context, ih bittorrent.InfoHash) (leechers uint32, seeders uint32, snatched uint32, err error) { scanPrefix, _ := composeIHKeyPrefix(ih.Bytes(), false, false, 0) if leechers, err = m.countPeers(ctx, scanPrefix); err != nil { - return + return leechers, seeders, snatched, err } scanPrefix[0], scanPrefix[1] = seederPrefix, ipv4Prefix if seeders, err = m.countPeers(ctx, scanPrefix); err != nil { - return + return leechers, seeders, snatched, err } scanPrefix[0], scanPrefix[1] = downloadedPrefix, countPrefix err = m.View(func(txn *lmdb.Txn) (err error) { var b []byte if b, err = ignoreNotFoundData(txn.Get(m.peersDB, scanPrefix)); err != nil { - return + return err } if len(b) >= 4 { snatched = binary.BigEndian.Uint32(b) } - return + return err }) - return + return leechers, seeders, snatched, err } const ( @@ -564,7 +564,7 @@ func (m *mdb) gc(cutoff time.Time) { break } } - return + return err }) } if err == nil { diff --git a/storage/memory/storage.go b/storage/memory/storage.go index d5f3d93..028b8e4 100644 --- a/storage/memory/storage.go +++ b/storage/memory/storage.go @@ -101,7 +101,7 @@ func (p *ihSwarm) get(k bittorrent.InfoHash) (v swarm, ok bool) { p.RLock() v, ok = p.m[k] p.RUnlock() - return + return v, ok } func (p *ihSwarm) getOrCreate(k bittorrent.InfoHash) (v swarm) { @@ -117,7 +117,7 @@ func (p *ihSwarm) getOrCreate(k bittorrent.InfoHash) (v swarm) { } p.Unlock() } - return + return v } func (p *ihSwarm) del(k bittorrent.InfoHash) (ok bool) { @@ -126,7 +126,7 @@ func (p *ihSwarm) del(k bittorrent.InfoHash) (ok bool) { delete(p.m, k) } p.Unlock() - return + return ok } func (p *ihSwarm) len() int { @@ -158,7 +158,7 @@ func (p *peers) get(k bittorrent.Peer) (v int64, ok bool) { p.RLock() v, ok = p.m[k] p.RUnlock() - return + return v, ok } func (p *peers) set(k bittorrent.Peer, v int64) { @@ -173,7 +173,7 @@ func (p *peers) del(k bittorrent.Peer) (ok bool) { delete(p.m, k) } p.Unlock() - return + return ok } func (p *peers) len() int { @@ -322,7 +322,7 @@ func (ps *peerStore) DeleteSeeder(_ context.Context, ih bittorrent.InfoHash, p b err = storage.ErrResourceDoesNotExist } - return + return err } func (ps *peerStore) PutLeecher(_ context.Context, ih bittorrent.InfoHash, p bittorrent.Peer) error { @@ -368,7 +368,7 @@ func (ps *peerStore) DeleteLeecher(_ context.Context, ih bittorrent.InfoHash, p err = storage.ErrResourceDoesNotExist } - return + return err } func (ps *peerStore) GraduateLeecher(_ context.Context, ih bittorrent.InfoHash, p bittorrent.Peer) error { @@ -427,7 +427,7 @@ func (ps *peerStore) AnnouncePeers(_ context.Context, ih bittorrent.InfoHash, fo } } - return + return peers, err } func (ps *peerStore) countPeers(ih bittorrent.InfoHash, v6 bool) (leechers, seeders uint32) { @@ -436,7 +436,7 @@ func (ps *peerStore) countPeers(ih bittorrent.InfoHash, v6 bool) (leechers, seed if sw, ok := shard.swarms.get(ih); ok { leechers, seeders = uint32(sw.leechers.len()), uint32(sw.seeders.len()) } - return + return leechers, seeders } func (ps *peerStore) ScrapeSwarm(_ context.Context, ih bittorrent.InfoHash) (leechers uint32, seeders uint32, snatched uint32, _ error) { diff --git a/storage/pg/storage.go b/storage/pg/storage.go index 3332b62..cb7d0ad 100644 --- a/storage/pg/storage.go +++ b/storage/pg/storage.go @@ -144,7 +144,7 @@ func checkParameter(p *string, name string) (err error) { if *p = strings.TrimSpace(*p); len(*p) == 0 { err = fmt.Errorf(errRequiredParameterNotSetMsg, name) } - return + return err } type config struct { @@ -268,7 +268,7 @@ func (s *store) txBatch(ctx context.Context, batch *pgx.Batch) (err error) { } } } - return + return err } func (s *store) Put(ctx context.Context, storeCtx string, values ...storage.Entry) (err error) { @@ -284,7 +284,7 @@ func (s *store) Put(ctx context.Context, storeCtx string, values ...storage.Entr } err = s.txBatch(ctx, &batch) } - return + return err } func (s *store) Contains(ctx context.Context, storeCtx string, key string) (contains bool, err error) { @@ -294,12 +294,12 @@ func (s *store) Contains(ctx context.Context, storeCtx string, key string) (cont contains = rows.Next() err = rows.Err() } - return + return contains, err } func (s *store) Load(ctx context.Context, storeCtx string, key string) (out []byte, err error) { err = noResultErr(s.QueryRow(ctx, s.Data.GetQuery, pgx.NamedArgs{pCtx: storeCtx, pKey: []byte(key)}).Scan(&out)) - return + return out, err } func (s *store) Delete(ctx context.Context, storeCtx string, keys ...string) (err error) { @@ -310,7 +310,7 @@ func (s *store) Delete(ctx context.Context, storeCtx string, keys ...string) (er } _, err = s.Exec(ctx, s.Data.DelQuery, pgx.NamedArgs{pCtx: storeCtx, pKey: baKeys}) } - return + return err } func (s *store) Preservable() bool { @@ -397,7 +397,7 @@ func (s *store) putPeer(ctx context.Context, ih []byte, peer bittorrent.Peer, se pV6: peer.Addr().Is6(), pCreated: timecache.Now(), }) - return + return err } func (s *store) delPeer(ctx context.Context, ih []byte, peer bittorrent.Peer, seeder bool) (err error) { @@ -412,7 +412,7 @@ func (s *store) delPeer(ctx context.Context, ih []byte, peer bittorrent.Peer, se pPort: peer.Port(), pSeeder: seeder, }) - return + return err } func (s *store) PutSeeder(ctx context.Context, ih bittorrent.InfoHash, peer bittorrent.Peer) error { @@ -475,7 +475,7 @@ func (s *store) getPeers(ctx context.Context, ih []byte, seeders bool, maxCount s.Announce.AddressColumn, s.Announce.PortColumn, }) - return + return peers, err } var maxIndex int switch { @@ -516,7 +516,7 @@ func (s *store) getPeers(ctx context.Context, ih []byte, seeders bool, maxCount } } } - return + return peers, err } func (s *store) AnnouncePeers(ctx context.Context, ih bittorrent.InfoHash, forSeeder bool, numWant int, v6 bool) (peers []bittorrent.Peer, err error) { @@ -546,7 +546,7 @@ func (s *store) AnnouncePeers(ctx context.Context, ih bittorrent.InfoHash, forSe logger.Warn().Err(err).Stringer("infoHash", ih).Msg("error occurred while retrieving peers") } - return + return peers, err } func (s *store) countPeers(ctx context.Context, ih []byte) (seeders uint32, leechers uint32, err error) { @@ -585,7 +585,7 @@ func (s *store) countPeers(ctx context.Context, ih []byte) (seeders uint32, leec } } } - return + return seeders, leechers, err } func (s *store) ScrapeSwarm(ctx context.Context, ih bittorrent.InfoHash) (leechers uint32, seeders uint32, snatched uint32, err error) { @@ -594,13 +594,13 @@ func (s *store) ScrapeSwarm(ctx context.Context, ih bittorrent.InfoHash) (leeche Msg("scrape swarm") ihb := ih.Bytes() if seeders, leechers, err = s.countPeers(ctx, ihb); err != nil { - return + return leechers, seeders, snatched, err } if len(s.Downloads.GetQuery) > 0 { err = noResultErr(s.QueryRow(ctx, s.Downloads.GetQuery, pgx.NamedArgs{pInfoHash: ihb}).Scan(&snatched)) } - return + return leechers, seeders, snatched, err } func (s *store) Ping(ctx context.Context) error { diff --git a/storage/redis/storage.go b/storage/redis/storage.go index d665d55..9bb8612 100644 --- a/storage/redis/storage.go +++ b/storage/redis/storage.go @@ -353,7 +353,7 @@ func (ps *store) count(key string, getLength bool) (n uint64) { if err != nil { logger.Error().Err(err).Str("key", key).Msg("GET/SCARD failure") } - return + return n } func (ps *store) getClock() int64 { @@ -374,7 +374,7 @@ func (ps *store) tx(ctx context.Context, txf func(tx redis.Pipeliner) error) (er } else { err = txErr } - return + return err } // NoResultErr returns nil if provided err is redis.Nil @@ -406,7 +406,7 @@ func InfoHashKey(infoHash string, seeder, v6 bool) (infoHashKey string) { infoHashKey = IH4LeecherKey } infoHashKey += infoHash - return + return infoHashKey } func (ps *store) putPeer(ctx context.Context, infoHashKey, peerCountKey, peerID string) error { @@ -416,13 +416,13 @@ func (ps *store) putPeer(ctx context.Context, infoHashKey, peerCountKey, peerID Msg("put peer") return ps.tx(ctx, func(tx redis.Pipeliner) (err error) { if err = tx.HSet(ctx, infoHashKey, peerID, ps.getClock()).Err(); err != nil { - return + return err } if err = tx.Incr(ctx, peerCountKey).Err(); err != nil { - return + return err } err = tx.SAdd(ctx, IHKey, infoHashKey).Err() - return + return err }) } @@ -513,7 +513,7 @@ var errInvalidPeerDataSize = fmt.Errorf("invalid peer data (must be at least %d func UnpackPeer(data string) (peer bittorrent.Peer, err error) { if len(data) < peerMinimumLen { err = errInvalidPeerDataSize - return + return peer, err } b := str2bytes.StringToBytes(data) peerID, _ := bittorrent.NewPeerID(b[:bittorrent.PeerIDLen]) @@ -529,7 +529,7 @@ func UnpackPeer(data string) (peer bittorrent.Peer, err error) { err = bittorrent.ErrInvalidIP } - return + return peer, err } func (ps *Connection) parsePeersList(peersResult *redis.StringSliceCmd) (peers []bittorrent.Peer, err error) { @@ -544,7 +544,7 @@ func (ps *Connection) parsePeersList(peersResult *redis.StringSliceCmd) (peers [ } } } - return + return peers, err } type getPeersFn func(context.Context, string, int) *redis.StringSliceCmd @@ -586,7 +586,7 @@ func (ps *Connection) GetPeers( logger.Warn().Err(err).Stringer("infoHash", ih).Msg("error occurred while retrieving peers") } - return + return out, err } func (ps *store) AnnouncePeers( @@ -613,26 +613,26 @@ func (ps *Connection) ScrapeIH(ctx context.Context, ih bittorrent.InfoHash, coun lc4, err = countFn(ctx, InfoHashKey(infoHash, false, false)).Result() if err = NoResultErr(err); err != nil { - return + return leechersCount, seedersCount, downloadsCount, err } lc6, err = countFn(ctx, InfoHashKey(infoHash, false, true)).Result() if err = NoResultErr(err); err != nil { - return + return leechersCount, seedersCount, downloadsCount, err } sc4, err = countFn(ctx, InfoHashKey(infoHash, true, false)).Result() if err = NoResultErr(err); err != nil { - return + return leechersCount, seedersCount, downloadsCount, err } sc6, err = countFn(ctx, InfoHashKey(infoHash, true, true)).Result() if err = NoResultErr(err); err != nil { - return + return leechersCount, seedersCount, downloadsCount, err } dc, err = ps.HGet(ctx, CountDownloadsKey, infoHash).Int64() if err = NoResultErr(err); err != nil { - return + return leechersCount, seedersCount, downloadsCount, err } leechersCount, seedersCount, downloadsCount = uint32(lc4+lc6), uint32(sc4+sc6), uint32(dc) - return + return leechersCount, seedersCount, downloadsCount, err } func (ps *store) ScrapeSwarm(ctx context.Context, ih bittorrent.InfoHash) (uint32, uint32, uint32, error) { @@ -667,7 +667,7 @@ func (ps *Connection) Put(ctx context.Context, storeCtx string, values ...storag } } } - return + return err } // Contains - storage.DataStorage implementation @@ -682,7 +682,7 @@ func (ps *Connection) Load(ctx context.Context, storeCtx string, key string) (v if err != nil && errors.Is(err, redis.Nil) { v, err = nil, nil } - return + return v, err } // Delete - storage.DataStorage implementation @@ -700,7 +700,7 @@ func (ps *Connection) Delete(ctx context.Context, storeCtx string, keys ...strin } } } - return + return err } // Preservable - storage.DataStorage implementation @@ -866,5 +866,5 @@ func (ps *store) Close() (err error) { logger.Info().Msg("redis exiting. mochi does not clear data in redis when exiting. mochi keys have prefix " + PrefixKey) err = ps.UniversalClient.Close() }) - return + return err } diff --git a/storage/storage.go b/storage/storage.go index a2d0279..a7bc42e 100644 --- a/storage/storage.go +++ b/storage/storage.go @@ -61,7 +61,7 @@ func (c Config) sanitizeGCConfig() (gcInterval, peerTTL time.Duration) { } else { peerTTL = c.PeerLifetime } - return + return gcInterval, peerTTL } func (c Config) sanitizeStatisticsConfig() (statInterval time.Duration) { @@ -73,7 +73,7 @@ func (c Config) sanitizeStatisticsConfig() (statInterval time.Duration) { Dur("default", DefaultPrometheusReportingInterval). Msg("falling back to default configuration") } - return + return statInterval } // Entry - some key-value pair, used for BulkPut @@ -269,11 +269,11 @@ func NewPeerStorage(cfg conf.NamedMapConfig) (ps PeerStorage, err error) { c := new(Config) if err = cfg.Config.Unmarshal(c); err != nil { - return + return ps, err } if ps, err = d.NewPeerStorage(cfg.Config); err != nil { - return + return ps, err } if gc, isOk := ps.(GarbageCollector); isOk { @@ -308,5 +308,5 @@ func NewPeerStorage(cfg conf.NamedMapConfig) (ps PeerStorage, err error) { logger.Info().Str("name", cfg.Name).Msg("storage started") - return + return ps, err } diff --git a/storage/test/storage_bench.go b/storage/test/storage_bench.go index 349a1f8..cc5b551 100644 --- a/storage/test/storage_bench.go +++ b/storage/test/storage_bench.go @@ -30,7 +30,7 @@ func generateInfoHashes() (a [ihCount]bittorrent.InfoHash) { for i := range a { a[i] = randIH(i < ihCount/2) } - return + return a } func generatePeers() (a [peersCount]bittorrent.Peer) { @@ -56,7 +56,7 @@ func generatePeers() (a [peersCount]bittorrent.Peer) { } } - return + return a } type ( diff --git a/storage/test/storage_test_data.go b/storage/test/storage_test_data.go index bbaa77c..87a1efa 100644 --- a/storage/test/storage_test_data.go +++ b/storage/test/storage_test_data.go @@ -27,7 +27,7 @@ func randIH(v2 bool) (ih bittorrent.InfoHash) { panic(err) } ih, _ = bittorrent.NewInfoHash(b) - return + return ih } func randPeerID() (ih bittorrent.PeerID) { @@ -36,7 +36,7 @@ func randPeerID() (ih bittorrent.PeerID) { panic(err) } ih, _ = bittorrent.NewPeerID(b) - return + return ih } func init() { From 04190a01b52e861d1c989500b46f0674c5e92820 Mon Sep 17 00:00:00 2001 From: "Lawrence, Rendall" Date: Mon, 22 Sep 2025 17:28:28 +0300 Subject: [PATCH 11/13] fix workflow for minio --- .github/workflows/build.yaml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 855e050..b03e7ed 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -39,12 +39,13 @@ jobs: --health-timeout 5s --health-retries 5 minio: - image: "minio/minio" + image: "minio/minio:latest" ports: [ "9000:9000" ] env: - MINIO_ACCESS_KEY: "minioadmin" - MINIO_SECRET_KEY: "minioadmin" - options: server /data + MINIO_ROOT_USER: minioadmin + MINIO_ROOT_PASSWORD: minioadmin + options: >- + --health-cmd "true" steps: - uses: "actions/checkout@v4" - uses: "actions/setup-go@v5" From 59a179f428714f7bc59813e3bf6119e85515ae73 Mon Sep 17 00:00:00 2001 From: "Lawrence, Rendall" Date: Mon, 22 Sep 2025 17:55:49 +0300 Subject: [PATCH 12/13] replace MinIO workflow image with download & run --- .github/workflows/build.yaml | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index b03e7ed..682e22b 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -38,19 +38,16 @@ jobs: --health-interval 10s --health-timeout 5s --health-retries 5 - minio: - image: "minio/minio:latest" - ports: [ "9000:9000" ] - env: - MINIO_ROOT_USER: minioadmin - MINIO_ROOT_PASSWORD: minioadmin - options: >- - --health-cmd "true" steps: - uses: "actions/checkout@v4" - uses: "actions/setup-go@v5" with: go-version: "^1.24" + - name: "Download & start MinIO" + run: | + wget --quiet -O /tmp/minio https://dl.min.io/server/minio/release/linux-amd64/minio + chmod +x /tmp/minio + /tmp/minio server /tmp/minio_data & - name: "Run `go test`" run: "go test -race ./..." From 02d343d5713873317fc6279dbbe2cbaf68b57f9a Mon Sep 17 00:00:00 2001 From: "Lawrence, Rendall" Date: Mon, 22 Sep 2025 18:09:51 +0300 Subject: [PATCH 13/13] fix data race in s3_test (mock test) --- .../torrentapproval/container/s3/s3_test.go | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/middleware/torrentapproval/container/s3/s3_test.go b/middleware/torrentapproval/container/s3/s3_test.go index c7b40ce..42dffb8 100644 --- a/middleware/torrentapproval/container/s3/s3_test.go +++ b/middleware/torrentapproval/container/s3/s3_test.go @@ -6,6 +6,7 @@ import ( "encoding/base64" "encoding/hex" "io" + "sync" "testing" "time" @@ -61,12 +62,15 @@ func init() { } type mockS3 struct { - objs []types.Object + objs []types.Object + objsMu *sync.Mutex } func (m *mockS3) ListObjectsV2( context.Context, *awss3.ListObjectsV2Input, ...func(*awss3.Options), ) (*awss3.ListObjectsV2Output, error) { + m.objsMu.Lock() + defer m.objsMu.Unlock() return &awss3.ListObjectsV2Output{ Contents: m.objs, }, nil @@ -87,7 +91,7 @@ func (m *mockS3) GetObject( } func TestScanMock(t *testing.T) { - cl := &mockS3{make([]types.Object, 0, len(files))} + cl := &mockS3{objs: make([]types.Object, 0, len(files)), objsMu: &sync.Mutex{}} for k := range files { cl.objs = append(cl.objs, types.Object{Key: &k}) } @@ -108,13 +112,12 @@ func TestScanMock(t *testing.T) { for name, f := range files { contains, _ := d.Storage.Contains(context.Background(), "TEST", f.hash) require.True(t, contains, "%s must present", name) - for i := 0; i < len(cl.objs); i++ { - if *cl.objs[i].Key == name { - cl.objs = append(cl.objs[:i], cl.objs[i+1:]...) - } - } } + cl.objsMu.Lock() + cl.objs = []types.Object{} + cl.objsMu.Unlock() + time.Sleep(time.Millisecond * 100) for name, f := range files { contains, _ := d.Storage.Contains(context.Background(), "TEST", f.hash)