mirror of
https://github.com/sot-tech/mochi.git
synced 2026-05-26 17:54:45 -07:00
change async torrent approval directory watch to periodic
This commit is contained in:
Lawrence, Rendall
@@ -1,8 +1,6 @@
|
||||
---
|
||||
run:
|
||||
# mochi in not written with generics (a.t.m),
|
||||
# so we can check with 1.17
|
||||
go: "1.17"
|
||||
go: "1.22"
|
||||
timeout: "5m"
|
||||
output:
|
||||
sort-results: true
|
||||
@@ -15,6 +13,7 @@ linters-settings:
|
||||
gosec:
|
||||
excludes:
|
||||
- "G505" # Allow SHA1 usage
|
||||
- "G115" # FIXME: remove after https://github.com/securego/gosec/issues/1187 resolve
|
||||
linters:
|
||||
enable:
|
||||
- "bidichk"
|
||||
|
||||
4
dist/example_config.yaml
vendored
4
dist/example_config.yaml
vendored
@@ -214,6 +214,10 @@ prehooks:
|
||||
# configuration:
|
||||
# hash_list:
|
||||
# - "a1b2c3d4e5a1b2c3d4e5a1b2c3d4e5a1b2c3d4e5"
|
||||
# Path to watch new torrent files (only for initial_source: 'directory')
|
||||
# path: "some/path"
|
||||
# Time between two directory checks
|
||||
# period: 5m
|
||||
# true - whitelist mode, false - blacklist
|
||||
# invert: false
|
||||
# Name of storage context where store hash list
|
||||
|
||||
@@ -25,7 +25,7 @@ There are two sources of hashes: `list` and `directory`.
|
||||
|
||||
* `directory` will watch for `*.torrent` files in specified path and
|
||||
append/delete records from storage. This source will parse all existing
|
||||
files at start and then watch for new files to add, or for delete events
|
||||
files at start and then periodically watch for new files to add, or for delete events
|
||||
to remove hash from storage.
|
||||
|
||||
Note: if storage is not `memory`, and `preserve` option set to `true`, records
|
||||
@@ -37,7 +37,8 @@ to storage) won't delete it.
|
||||
This middleware provides the following parameters for configuration:
|
||||
|
||||
- `initial_source` - source type: `list` or `directory`
|
||||
- `preserve`: - save source provided data into storage
|
||||
- `storage` - storage configuration to store data, structure is same as global `storage` section.
|
||||
If `name` is empty or `internal` global storage will be used
|
||||
- `configuration` - options for specified source
|
||||
- `list`:
|
||||
- `hash_list` - list of HEX encoded hashes
|
||||
@@ -46,6 +47,7 @@ This middleware provides the following parameters for configuration:
|
||||
It may be redis hash key, DB table name etc.
|
||||
- `directory`:
|
||||
- `path` - directory to watch
|
||||
- `period` - time between two directory checks
|
||||
- `invert` and `storage_ctx` has the same meanins as `list`'s options
|
||||
|
||||
Configuration example:
|
||||
@@ -58,9 +60,13 @@ mochi:
|
||||
- name: torrent approval
|
||||
options:
|
||||
initial_source: list
|
||||
preserve: true
|
||||
storage:
|
||||
name: internal
|
||||
config:
|
||||
configuration:
|
||||
hash_list: [ "AAA", "BBB" ]
|
||||
path: "some/path"
|
||||
period: 1m
|
||||
invert: false
|
||||
storage_ctx: APPROVED_HASH
|
||||
```
|
||||
|
||||
12
go.mod
12
go.mod
@@ -3,7 +3,7 @@ module github.com/sot-tech/mochi
|
||||
go 1.22
|
||||
|
||||
require (
|
||||
code.cloudfoundry.org/go-diodes v0.0.0-20240730232652-ce6331b0e7c0
|
||||
code.cloudfoundry.org/go-diodes v0.0.0-20240813203737-5032edb05ceb
|
||||
github.com/MicahParks/jwkset v0.5.18
|
||||
github.com/MicahParks/keyfunc/v3 v3.3.3
|
||||
github.com/PowerDNS/lmdb-go v1.9.2
|
||||
@@ -15,11 +15,12 @@ require (
|
||||
github.com/libp2p/go-reuseport v0.4.0
|
||||
github.com/minio/sha256-simd v1.0.1
|
||||
github.com/mitchellh/mapstructure v1.5.0
|
||||
github.com/prometheus/client_golang v1.19.1
|
||||
github.com/prometheus/client_golang v1.20.2
|
||||
github.com/redis/go-redis/v9 v9.6.1
|
||||
github.com/rs/zerolog v1.33.0
|
||||
github.com/stretchr/testify v1.9.0
|
||||
github.com/valyala/fasthttp v1.55.0
|
||||
github.com/zeebo/bencode v1.0.0
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
)
|
||||
|
||||
@@ -35,7 +36,6 @@ require (
|
||||
github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 // indirect
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
|
||||
github.com/fsnotify/fsnotify v1.7.0 // indirect
|
||||
github.com/huandu/xstrings v1.5.0 // indirect
|
||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
||||
@@ -50,16 +50,16 @@ require (
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/prometheus/client_model v0.6.1 // indirect
|
||||
github.com/prometheus/common v0.55.0 // indirect
|
||||
github.com/prometheus/common v0.57.0 // indirect
|
||||
github.com/prometheus/procfs v0.15.1 // indirect
|
||||
github.com/savsgio/gotils v0.0.0-20240704082632-aef3928b8a38 // indirect
|
||||
github.com/spaolacci/murmur3 v1.1.0 // indirect
|
||||
github.com/valyala/bytebufferpool v1.0.0 // indirect
|
||||
golang.org/x/crypto v0.26.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect
|
||||
golang.org/x/exp v0.0.0-20240823005443-9b4947da3948 // indirect
|
||||
golang.org/x/net v0.28.0 // indirect
|
||||
golang.org/x/sync v0.8.0 // indirect
|
||||
golang.org/x/sys v0.23.0 // indirect
|
||||
golang.org/x/sys v0.24.0 // indirect
|
||||
golang.org/x/text v0.17.0 // indirect
|
||||
golang.org/x/time v0.6.0 // indirect
|
||||
google.golang.org/protobuf v1.34.2 // indirect
|
||||
|
||||
38
go.sum
38
go.sum
@@ -1,7 +1,7 @@
|
||||
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||
code.cloudfoundry.org/go-diodes v0.0.0-20240730232652-ce6331b0e7c0 h1:8B0xmGIfbmq8zCjVI00vwEzqanSGcrxvccau4bdjGRk=
|
||||
code.cloudfoundry.org/go-diodes v0.0.0-20240730232652-ce6331b0e7c0/go.mod h1:QtvWqJPmrug53E4Vl1SNYn4jorWAjDJQr/DcAssb4TU=
|
||||
code.cloudfoundry.org/go-diodes v0.0.0-20240813203737-5032edb05ceb h1:Last39ehN2b866DrM0B5FPIbdBFouL9humzWTqVTEX4=
|
||||
code.cloudfoundry.org/go-diodes v0.0.0-20240813203737-5032edb05ceb/go.mod h1:PfkH7ePa4EhyC6VRisbZJOaIHhas7ofB0heANM//Z6Y=
|
||||
crawshaw.io/iox v0.0.0-20181124134642-c51c3df30797/go.mod h1:sXBiorCo8c46JlQV3oXPKINnZ8mcqnye1EkVkqsectk=
|
||||
crawshaw.io/sqlite v0.3.2/go.mod h1:igAO5JulrQ1DbdZdtVq48mnZUBAPOeFzer7VhDWNtW4=
|
||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
@@ -88,8 +88,6 @@ github.com/fasthttp/router v1.5.2/go.mod h1:C8EY53ozOwpONyevc/V7Gr8pqnEjwnkFFqPo
|
||||
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
|
||||
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
|
||||
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
|
||||
github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
|
||||
github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
|
||||
github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE=
|
||||
github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE=
|
||||
github.com/glycerine/go-unsnap-stream v0.0.0-20190901134440-81cf024a9e0a/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE=
|
||||
@@ -133,8 +131,8 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
|
||||
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
|
||||
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 h1:k7nVchz72niMH6YLQNvHSdIE7iqsQxK1P41mySCvssg=
|
||||
github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw=
|
||||
github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8 h1:FKHo8hFI3A+7w0aUQuYXQ+6EN5stWmeY/AZqtM8xk9k=
|
||||
github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20190309154008-847fc94819f9/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
|
||||
@@ -175,6 +173,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
|
||||
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
|
||||
github.com/libp2p/go-reuseport v0.4.0 h1:nR5KU7hD0WxXCJbmw7r2rhRYruNRl2koHw8fQscQm2s=
|
||||
github.com/libp2p/go-reuseport v0.4.0/go.mod h1:ZtI03j/wO5hZVDFo2jKywN6bYKWLOy8Se6DrI2E1cLU=
|
||||
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
|
||||
@@ -206,8 +206,8 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW
|
||||
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs=
|
||||
github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/ginkgo/v2 v2.19.1 h1:QXgq3Z8Crl5EL1WBAC98A5sEBHARrAJNzAmMxzLcRF0=
|
||||
github.com/onsi/ginkgo/v2 v2.19.1/go.mod h1:O3DtEWQkPa/F7fBMgmZQKKsluAy8pd3rEQdrjkPb9zA=
|
||||
github.com/onsi/ginkgo/v2 v2.20.0 h1:PE84V2mHqoT1sglvHc8ZdQtPcwmvvt29WLEEO3xmdZw=
|
||||
github.com/onsi/ginkgo/v2 v2.20.0/go.mod h1:lG9ey2Z29hR41WMVthyJBGUBcBhGOtoPF2VFMvBXFCI=
|
||||
github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
|
||||
github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k=
|
||||
github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY=
|
||||
@@ -223,8 +223,8 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP
|
||||
github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs=
|
||||
github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
|
||||
github.com/prometheus/client_golang v1.5.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU=
|
||||
github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE=
|
||||
github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho=
|
||||
github.com/prometheus/client_golang v1.20.2 h1:5ctymQzZlyOON1666svgwn3s6IKWgfbjsejTMiXIyjg=
|
||||
github.com/prometheus/client_golang v1.20.2/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE=
|
||||
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
|
||||
github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
|
||||
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
|
||||
@@ -234,8 +234,8 @@ github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQy
|
||||
github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
|
||||
github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
|
||||
github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4=
|
||||
github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc=
|
||||
github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8=
|
||||
github.com/prometheus/common v0.57.0 h1:Ro/rKjwdq9mZn1K5QPctzh+MA4Lp0BuYk5ZZEVhoNcY=
|
||||
github.com/prometheus/common v0.57.0/go.mod h1:7uRPFSUTbfZWsJ7MHY56sqt7hLQu3bxXHDnNhl8E9qI=
|
||||
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
|
||||
github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
|
||||
github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
|
||||
@@ -280,6 +280,8 @@ github.com/valyala/fasthttp v1.55.0 h1:Zkefzgt6a7+bVKHnu/YaYSOPfNYNisSVBo/unVCf8
|
||||
github.com/valyala/fasthttp v1.55.0/go.mod h1:NkY9JtkrpPKmgwV3HTaS2HWaJss9RSIsRVfcxxoHiOM=
|
||||
github.com/willf/bitset v1.1.9/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
|
||||
github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
|
||||
github.com/zeebo/bencode v1.0.0 h1:zgop0Wu1nu4IexAZeCZ5qbsjU4O1vMrfCrVgUjbHVuA=
|
||||
github.com/zeebo/bencode v1.0.0/go.mod h1:Ct7CkrWIQuLWAy9M3atFHYq4kG9Ao/SsY5cdtCXmp9Y=
|
||||
go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
|
||||
go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
|
||||
go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
|
||||
@@ -288,8 +290,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
|
||||
golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw=
|
||||
golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8=
|
||||
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY=
|
||||
golang.org/x/exp v0.0.0-20240823005443-9b4947da3948 h1:kx6Ds3MlpiUHKj7syVnbp57++8WpuKPcR5yjLBjvLEA=
|
||||
golang.org/x/exp v0.0.0-20240823005443-9b4947da3948/go.mod h1:akd2r19cwCdwSwWeIdzYQGa/EZZyqcOdwWiwj5L5eKQ=
|
||||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
|
||||
golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
@@ -330,8 +332,8 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM=
|
||||
golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg=
|
||||
golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc=
|
||||
@@ -344,8 +346,8 @@ golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGm
|
||||
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
|
||||
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.23.0 h1:SGsXPZ+2l4JsgaCKkx+FQ9YZ5XEtA1GZYuoDjenLjvg=
|
||||
golang.org/x/tools v0.23.0/go.mod h1:pnu6ufv6vQkll6szChhK3C3L/ruaIv5eBeztNG8wtsI=
|
||||
golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24=
|
||||
golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk=
|
||||
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||
|
||||
@@ -6,22 +6,34 @@ package directory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha1"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/anacrolix/torrent/metainfo"
|
||||
"github.com/anacrolix/torrent/util/dirwatch"
|
||||
"github.com/minio/sha256-simd"
|
||||
|
||||
"github.com/zeebo/bencode"
|
||||
|
||||
"github.com/sot-tech/mochi/bittorrent"
|
||||
"github.com/sot-tech/mochi/middleware/torrentapproval/container"
|
||||
"github.com/sot-tech/mochi/middleware/torrentapproval/container/list"
|
||||
"github.com/sot-tech/mochi/pkg/conf"
|
||||
"github.com/sot-tech/mochi/pkg/log"
|
||||
"github.com/sot-tech/mochi/pkg/str2bytes"
|
||||
"github.com/sot-tech/mochi/storage"
|
||||
)
|
||||
|
||||
var logger = log.NewLogger("middleware/torrent approval/directory")
|
||||
|
||||
const (
|
||||
defaultPeriod = time.Minute
|
||||
maxTorrentSize = 10 * 1024 * 1024
|
||||
)
|
||||
|
||||
func init() {
|
||||
container.Register("directory", build)
|
||||
}
|
||||
@@ -32,6 +44,8 @@ type Config struct {
|
||||
list.Config
|
||||
// Path in filesystem where torrent files stored and should be watched
|
||||
Path string
|
||||
// Period is time between two Path checks
|
||||
Period time.Duration
|
||||
}
|
||||
|
||||
func build(conf conf.MapConfig, st storage.DataStorage) (container.Container, error) {
|
||||
@@ -46,7 +60,7 @@ func build(conf conf.MapConfig, st storage.DataStorage) (container.Container, er
|
||||
Storage: st,
|
||||
StorageCtx: c.StorageCtx,
|
||||
},
|
||||
watcher: nil,
|
||||
closed: make(chan bool),
|
||||
}
|
||||
if len(d.StorageCtx) == 0 {
|
||||
logger.Warn().
|
||||
@@ -56,77 +70,132 @@ func build(conf conf.MapConfig, st storage.DataStorage) (container.Container, er
|
||||
Msg("falling back to default configuration")
|
||||
d.StorageCtx = container.DefaultStorageCtxName
|
||||
}
|
||||
var w *dirwatch.Instance
|
||||
if w, err = dirwatch.New(c.Path); err != nil {
|
||||
return nil, fmt.Errorf("unable to initialize directory watch: %w", err)
|
||||
if c.Period == 0 {
|
||||
logger.Warn().
|
||||
Str("name", "Period").
|
||||
Dur("provided", 0).
|
||||
Dur("default", defaultPeriod).
|
||||
Msg("falling back to default configuration")
|
||||
c.Period = defaultPeriod
|
||||
}
|
||||
d.watcher = w
|
||||
go func() {
|
||||
for event := range d.watcher.Events {
|
||||
var mi *metainfo.MetaInfo
|
||||
if mi, err = metainfo.LoadFromFile(event.TorrentFilePath); err == nil {
|
||||
s256 := sha256.New()
|
||||
s256.Write(mi.InfoBytes)
|
||||
v2hash, _ := bittorrent.NewInfoHash(s256.Sum(nil))
|
||||
switch event.Change {
|
||||
case dirwatch.Added:
|
||||
var name string
|
||||
if info, err := mi.UnmarshalInfo(); err == nil {
|
||||
name = info.Name
|
||||
} else {
|
||||
logger.Error().
|
||||
Err(err).
|
||||
Str("file", event.TorrentFilePath).
|
||||
Stringer("infoHash", event.InfoHash).
|
||||
Stringer("infoHashV2", v2hash).
|
||||
Msg("unable to unmarshal torrent info")
|
||||
go d.runScan(c.Path, c.Period)
|
||||
return d, err
|
||||
}
|
||||
|
||||
// BencodeRawBytes wrapper for byte slice to get raw 'info' section from
|
||||
// torrent file
|
||||
type BencodeRawBytes []byte
|
||||
|
||||
// UnmarshalBencode just appends raw byte slice to result
|
||||
func (ba *BencodeRawBytes) UnmarshalBencode(in []byte) error {
|
||||
*ba = append([]byte(nil), in...)
|
||||
return nil
|
||||
}
|
||||
|
||||
type torrentRawInfoStruct struct {
|
||||
Info BencodeRawBytes `bencode:"info"`
|
||||
}
|
||||
|
||||
type torrentNameInfoStruct struct {
|
||||
Name string `bencode:"name"`
|
||||
}
|
||||
|
||||
func (d *directory) runScan(path string, period time.Duration) {
|
||||
t := time.NewTicker(period)
|
||||
defer t.Stop()
|
||||
files := make(map[string][2]bittorrent.InfoHash)
|
||||
tmpFiles := make(map[string]bool)
|
||||
// nolint:gosec
|
||||
s1, s2 := sha1.New(), sha256.New()
|
||||
for {
|
||||
select {
|
||||
case <-d.closed:
|
||||
return
|
||||
case <-t.C:
|
||||
logger.Debug().Msg("starting directory scan")
|
||||
if entries, err := os.ReadDir(path); err == nil {
|
||||
for _, e := range entries {
|
||||
if !e.IsDir() && strings.ToLower(filepath.Ext(e.Name())) == ".torrent" {
|
||||
tmpFiles[filepath.Join(path, e.Name())] = true
|
||||
}
|
||||
if len(name) == 0 {
|
||||
name = list.DUMMY
|
||||
}
|
||||
bName := []byte(name)
|
||||
logger.Err(d.Storage.Put(context.Background(), d.StorageCtx, storage.Entry{
|
||||
Key: event.InfoHash.AsString(),
|
||||
Value: bName,
|
||||
}, storage.Entry{
|
||||
Key: v2hash.RawString(),
|
||||
Value: bName,
|
||||
}, storage.Entry{
|
||||
Key: v2hash.TruncateV1().RawString(),
|
||||
Value: bName,
|
||||
})).
|
||||
Str("action", "add").
|
||||
Str("file", event.TorrentFilePath).
|
||||
Stringer("infoHash", event.InfoHash).
|
||||
Stringer("infoHashV2", v2hash).
|
||||
Msg("approval torrent watcher event")
|
||||
case dirwatch.Removed:
|
||||
logger.Err(d.Storage.Delete(context.Background(), c.StorageCtx, event.InfoHash.AsString(), v2hash.RawString(), v2hash.TruncateV1().RawString())).
|
||||
Str("action", "delete").
|
||||
Str("file", event.TorrentFilePath).
|
||||
Stringer("infoHash", event.InfoHash).
|
||||
Stringer("infoHashV2", v2hash).
|
||||
Msg("approval torrent watcher event")
|
||||
}
|
||||
for p := range tmpFiles {
|
||||
if _, exists := files[p]; !exists {
|
||||
var f *os.File
|
||||
if f, err = os.Open(p); err == nil {
|
||||
var info torrentRawInfoStruct
|
||||
err = bencode.NewDecoder(io.LimitReader(f, maxTorrentSize)).Decode(&info)
|
||||
_ = f.Close()
|
||||
if err == nil {
|
||||
s1.Write(info.Info)
|
||||
h1, _ := bittorrent.NewInfoHash(s1.Sum(nil))
|
||||
s1.Reset()
|
||||
|
||||
s2.Write(info.Info)
|
||||
h2, _ := bittorrent.NewInfoHash(s2.Sum(nil))
|
||||
s2.Reset()
|
||||
|
||||
files[p] = [2]bittorrent.InfoHash{h1, h2}
|
||||
var name torrentNameInfoStruct
|
||||
if err := bencode.DecodeBytes(info.Info, &name); err != nil {
|
||||
logger.Warn().
|
||||
Err(err).
|
||||
Str("file", p).
|
||||
Msg("unable to unmarshal torrent info")
|
||||
}
|
||||
if len(name.Name) == 0 {
|
||||
name.Name = list.DUMMY
|
||||
}
|
||||
bName := str2bytes.StringToBytes(name.Name)
|
||||
logger.Err(d.Storage.Put(context.Background(), d.StorageCtx, storage.Entry{
|
||||
Key: h1.RawString(),
|
||||
Value: bName,
|
||||
}, storage.Entry{
|
||||
Key: h2.RawString(),
|
||||
Value: bName,
|
||||
}, storage.Entry{
|
||||
Key: h2.TruncateV1().RawString(),
|
||||
Value: bName,
|
||||
})).
|
||||
Str("file", p).
|
||||
Stringer("infoHash", h1).
|
||||
Stringer("infoHashV2", h2).
|
||||
Msg("added torrent to approval list")
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
logger.Warn().Err(err).Str("file", p).Msg("unable to read file")
|
||||
}
|
||||
}
|
||||
}
|
||||
for p, ih := range files {
|
||||
if _, isOk := tmpFiles[p]; !isOk {
|
||||
delete(files, p)
|
||||
logger.Err(d.Storage.Delete(context.Background(), d.StorageCtx, ih[0].RawString(),
|
||||
ih[1].RawString(), ih[1].TruncateV1().RawString())).
|
||||
Str("file", p).
|
||||
Stringer("infoHash", ih[1]).
|
||||
Stringer("infoHashV2", ih[1]).
|
||||
Msg("deleted torrent from approval list")
|
||||
}
|
||||
}
|
||||
clear(tmpFiles)
|
||||
} else {
|
||||
logger.Error().Err(err).
|
||||
Str("file", event.TorrentFilePath).
|
||||
Msg("unable to load torrent file")
|
||||
logger.Warn().Err(err).Msg("unable to get directory content")
|
||||
}
|
||||
}
|
||||
}()
|
||||
return d, err
|
||||
}
|
||||
}
|
||||
|
||||
type directory struct {
|
||||
list.List
|
||||
watcher *dirwatch.Instance
|
||||
closed chan bool
|
||||
}
|
||||
|
||||
// Close closes watching of torrent directory
|
||||
func (d *directory) Close() error {
|
||||
if d.watcher != nil {
|
||||
d.watcher.Close()
|
||||
if d.closed != nil {
|
||||
close(d.closed)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -95,6 +95,7 @@ func TestHandleAnnounce(t *testing.T) {
|
||||
} else {
|
||||
require.Equal(t, err, ErrTorrentUnapproved)
|
||||
}
|
||||
_ = h.(*hook).Close()
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user