mochi/middleware/torrentapproval/container/s3/s3_test.go

package s3

import (
	"bytes"
	"context"
	"encoding/base64"
	"encoding/hex"
	"io"
	"testing"
	"time"

	"github.com/aws/aws-sdk-go-v2/config"
	"github.com/aws/aws-sdk-go-v2/credentials"
	awss3 "github.com/aws/aws-sdk-go-v2/service/s3"
	"github.com/aws/aws-sdk-go-v2/service/s3/types"
	"github.com/stretchr/testify/require"

	"github.com/sot-tech/mochi/middleware/torrentapproval/container/directory"
	"github.com/sot-tech/mochi/middleware/torrentapproval/container/list"
	"github.com/sot-tech/mochi/pkg/conf"
	"github.com/sot-tech/mochi/pkg/log"
	"github.com/sot-tech/mochi/storage/memory"
)

type testData struct {
	data []byte
	hash string
}

func unHEX(s string) string {
	b, _ := hex.DecodeString(s)
	return string(b)
}

func unBase64(s string) []byte {
	b, _ := base64.StdEncoding.DecodeString(s)
	return b
}

// contains created torrent file data from simple txt documents.
// data base64 encoded because `pieces` section in torrent file is raw bytes
var files = map[string]testData{
	"test0.torrent": {
		data: unBase64(`ZDEwOmNyZWF0ZWQgYnkzMTpUcmFuc21pc3Npb24vNC4wLjYgKDM4YzE2NDkzM2UpMTM6Y3JlYXRp
b24gZGF0ZWkxNzU1ODYxOTI3ZTg6ZW5jb2Rpbmc1OlVURi04NDppbmZvZDY6bGVuZ3RoaTVlNDpu
YW1lODp0ZXN0LnR4dDEyOnBpZWNlIGxlbmd0aGkzMjc2OGU2OnBpZWNlczIwOk4SQ70ixm52wrqe
3cH5E5Tlf5+DZWU=`),
		hash: unHEX("a10e8e9e81702bf8482f251551ff4fe011cba6a7"),
	},
	"test1.torrent": {
		data: unBase64(`ZDEwOmNyZWF0ZWQgYnkzMTpUcmFuc21pc3Npb24vNC4wLjYgKDM4YzE2NDkzM2UpMTM6Y3JlYXRp
b24gZGF0ZWkxNzU2MTIzNzEwZTg6ZW5jb2Rpbmc1OlVURi04NDppbmZvZDY6bGVuZ3RoaTRlNDpu
YW1lOTp0ZXN0MC50eHQxMjpwaWVjZSBsZW5ndGhpMzI3NjhlNjpwaWVjZXMyMDqo/cIFqfGcwcdQ
emDE8BsT0R1/0GVl`),
		hash: unHEX("e86d393bd458d2acc46d5467bc8cb8b30b1bfa77"),
	},
}

func init() {
	_ = log.ConfigureLogger("", "warn", false, false)
}

type mockS3 struct {
	objs []types.Object
}

func (m *mockS3) ListObjectsV2(
	context.Context, *awss3.ListObjectsV2Input, ...func(*awss3.Options),
) (*awss3.ListObjectsV2Output, error) {
	return &awss3.ListObjectsV2Output{
		Contents: m.objs,
	}, nil
}

var _ s3Client = &mockS3{}

func (m *mockS3) GetObject(
	_ context.Context, params *awss3.GetObjectInput, _ ...func(*awss3.Options),
) (*awss3.GetObjectOutput, error) {
	if params == nil || params.Key == nil {
		return nil, nil
	}
	v := files[*params.Key]
	return &awss3.GetObjectOutput{
		Body: io.NopCloser(bytes.NewReader(v.data)),
	}, nil
}

func TestScanMock(t *testing.T) {
	cl := &mockS3{make([]types.Object, 0, len(files))}
	for k := range files {
		cl.objs = append(cl.objs, types.Object{Key: &k})
	}
	st, _ := memory.Builder{}.NewDataStorage(make(conf.MapConfig))
	d := directory.NewScanner(list.List{
		Invert:     false,
		Storage:    st,
		StorageCtx: "TEST",
	}, s3{
		client: cl,
	}, time.Millisecond*10)
	go d.Run()
	t.Cleanup(func() {
		_ = d.Close()
	})

	time.Sleep(time.Millisecond * 100)
	for name, f := range files {
		contains, _ := d.Storage.Contains(context.Background(), "TEST", f.hash)
		require.True(t, contains, "%s must present", name)
		for i := 0; i < len(cl.objs); i++ {
			if *cl.objs[i].Key == name {
				cl.objs = append(cl.objs[:i], cl.objs[i+1:]...)
			}
		}
	}

	time.Sleep(time.Millisecond * 100)
	for name, f := range files {
		contains, _ := d.Storage.Contains(context.Background(), "TEST", f.hash)
		require.False(t, contains, "%s must absent", name)
	}
}

var (
	minioEndpoint = "http://127.0.0.1:9000"
	minioBucket   = "test"
	minioPrefix   = "test/"
)

const (
	minioKeyID  = "minioadmin"
	minioSecret = "minioadmin"
	minioRegion = "us-east-1"
)

// TestScanMinio requires real minio instance listening 127.0.0.1:9000
// with default login, password and region (minioadmin/minioadmin, us-east-1)
func TestScanMinio(t *testing.T) {
	st, _ := memory.Builder{}.NewDataStorage(make(conf.MapConfig))
	awsCfg, err := config.LoadDefaultConfig(context.Background())
	if err != nil {
		t.Fatal(err)
	}
	awsCfg.BaseEndpoint = &minioEndpoint
	awsCfg.Region = minioRegion
	awsCfg.Credentials = credentials.NewStaticCredentialsProvider(minioKeyID, minioSecret, "")
	cl := awss3.NewFromConfig(awsCfg)

	_, _ = cl.CreateBucket(context.Background(), &awss3.CreateBucketInput{Bucket: &minioBucket})

	for name, data := range files {
		_, err = cl.PutObject(context.Background(), &awss3.PutObjectInput{
			Bucket: &minioBucket,
			Key:    &name,
			Body:   bytes.NewReader(data.data),
		})
		if err != nil {
			t.Fatal(err)
		}
		name = minioPrefix + name
		_, err = cl.PutObject(context.Background(), &awss3.PutObjectInput{
			Bucket: &minioBucket,
			Key:    &name,
			Body:   bytes.NewReader(data.data),
		})
		if err != nil {
			t.Fatal(err)
		}
		name += "1"
		_, err = cl.PutObject(context.Background(), &awss3.PutObjectInput{
			Bucket: &minioBucket,
			Key:    &name,
			Body:   bytes.NewReader(data.data),
		})
		if err != nil {
			t.Fatal(err)
		}
	}

	s3Dir := s3{
		client: cl,
		bucket: minioBucket,
		prefix: minioPrefix,
		suffix: ".torrent",
	}

	if content, err := s3Dir.ReadDir(); err == nil {
		var i int
		for range content {
			i++
		}
		require.Equal(t, len(files), i, "S3 content data not the same as test data")
	} else {
		t.Fatal(err)
	}

	d := directory.NewScanner(list.List{
		Invert:     false,
		Storage:    st,
		StorageCtx: "TEST",
	}, s3Dir, time.Millisecond*100)

	go d.Run()
	t.Cleanup(func() {
		_ = d.Close()
	})
	time.Sleep(time.Millisecond * 200)

	for name, f := range files {
		contains, _ := d.Storage.Contains(context.Background(), "TEST", f.hash)
		require.True(t, contains, "%s must present", name)
		name = minioPrefix + name
		_, err = cl.DeleteObject(context.Background(), &awss3.DeleteObjectInput{
			Bucket: &minioBucket,
			Key:    &name,
		})
		if err != nil {
			t.Fatal(err)
		}
	}

	time.Sleep(time.Millisecond * 200)

	for name, f := range files {
		contains, _ := d.Storage.Contains(context.Background(), "TEST", f.hash)
		require.False(t, contains, "%s must absent", name)
	}
}