change async torrent approval directory watch to periodic

.golangci.yaml

@@ -1,8 +1,6 @@
 ---
 run:
-    # mochi in not written with generics (a.t.m),
+    go: "1.22"
-    # so we can check with 1.17
-    go: "1.17"
     timeout: "5m"
 output:
     sort-results: true
@@ -15,6 +13,7 @@ linters-settings:
     gosec:
         excludes:
             - "G505"  # Allow SHA1 usage
+            - "G115" # FIXME: remove after https://github.com/securego/gosec/issues/1187 resolve
 linters:
     enable:
         - "bidichk"

dist/example_config.yaml

@@ -214,6 +214,10 @@ prehooks:
 #                configuration:
 #                    hash_list:
 #                        - "a1b2c3d4e5a1b2c3d4e5a1b2c3d4e5a1b2c3d4e5"
+# Path to watch new torrent files (only for initial_source: 'directory')
+#                    path: "some/path"
+# Time between two directory checks
+#                    period: 5m
 # true - whitelist mode, false - blacklist
 #                    invert: false
 # Name of storage context where store hash list

docs/middleware/torrent_approval.md

@@ -25,7 +25,7 @@ There are two sources of hashes: `list` and `directory`.
 
 * `directory` will watch for `*.torrent` files in specified path and
   append/delete records from storage. This source will parse all existing
-  files at start and then watch for new files to add, or for delete events
+  files at start and then periodically watch for new files to add, or for delete events
   to remove hash from storage.
 
 Note: if storage is not `memory`, and `preserve` option set to `true`, records
@@ -37,7 +37,8 @@ to storage) won't delete it.
 This middleware provides the following parameters for configuration:
 
 - `initial_source` - source type: `list` or `directory`
-- `preserve`: - save source provided data into storage
+- `storage` - storage configuration to store data, structure is same as global `storage` section.
+If `name` is empty or `internal` global storage will be used
 - `configuration` - options for specified source
 	- `list`:
 		- `hash_list` - list of HEX encoded hashes
@@ -46,6 +47,7 @@ This middleware provides the following parameters for configuration:
 		  It may be redis hash key, DB table name etc.
 	- `directory`:
 		- `path` - directory to watch
+        - `period` - time between two directory checks
 		- `invert` and `storage_ctx` has the same meanins as `list`'s options
 
 Configuration example:
@@ -58,9 +60,13 @@ mochi:
         -   name: torrent approval
                 options:
                     initial_source: list
-                    preserve: true
+					storage:
+						name: internal
+						config:
                     configuration:
                         hash_list: [ "AAA", "BBB" ]
+						path: "some/path"
+						period: 1m
                         invert: false
                         storage_ctx: APPROVED_HASH
 ```

go.mod

@@ -3,7 +3,7 @@ module github.com/sot-tech/mochi
 go 1.22
 
 require (
-	code.cloudfoundry.org/go-diodes v0.0.0-20240730232652-ce6331b0e7c0
+	code.cloudfoundry.org/go-diodes v0.0.0-20240813203737-5032edb05ceb
 	github.com/MicahParks/jwkset v0.5.18
 	github.com/MicahParks/keyfunc/v3 v3.3.3
 	github.com/PowerDNS/lmdb-go v1.9.2
@@ -15,11 +15,12 @@ require (
 	github.com/libp2p/go-reuseport v0.4.0
 	github.com/minio/sha256-simd v1.0.1
 	github.com/mitchellh/mapstructure v1.5.0
-	github.com/prometheus/client_golang v1.19.1
+	github.com/prometheus/client_golang v1.20.2
 	github.com/redis/go-redis/v9 v9.6.1
 	github.com/rs/zerolog v1.33.0
 	github.com/stretchr/testify v1.9.0
 	github.com/valyala/fasthttp v1.55.0
+	github.com/zeebo/bencode v1.0.0
 	gopkg.in/yaml.v3 v3.0.1
 )
 
@@ -35,7 +36,6 @@ require (
 	github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
-	github.com/fsnotify/fsnotify v1.7.0 // indirect
 	github.com/huandu/xstrings v1.5.0 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
 	github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
@@ -50,16 +50,16 @@ require (
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_model v0.6.1 // indirect
-	github.com/prometheus/common v0.55.0 // indirect
+	github.com/prometheus/common v0.57.0 // indirect
 	github.com/prometheus/procfs v0.15.1 // indirect
 	github.com/savsgio/gotils v0.0.0-20240704082632-aef3928b8a38 // indirect
 	github.com/spaolacci/murmur3 v1.1.0 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	golang.org/x/crypto v0.26.0 // indirect
-	golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect
+	golang.org/x/exp v0.0.0-20240823005443-9b4947da3948 // indirect
 	golang.org/x/net v0.28.0 // indirect
 	golang.org/x/sync v0.8.0 // indirect
-	golang.org/x/sys v0.23.0 // indirect
+	golang.org/x/sys v0.24.0 // indirect
 	golang.org/x/text v0.17.0 // indirect
 	golang.org/x/time v0.6.0 // indirect
 	google.golang.org/protobuf v1.34.2 // indirect

go.sum

@@ -1,7 +1,7 @@
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-code.cloudfoundry.org/go-diodes v0.0.0-20240730232652-ce6331b0e7c0 h1:8B0xmGIfbmq8zCjVI00vwEzqanSGcrxvccau4bdjGRk=
+code.cloudfoundry.org/go-diodes v0.0.0-20240813203737-5032edb05ceb h1:Last39ehN2b866DrM0B5FPIbdBFouL9humzWTqVTEX4=
-code.cloudfoundry.org/go-diodes v0.0.0-20240730232652-ce6331b0e7c0/go.mod h1:QtvWqJPmrug53E4Vl1SNYn4jorWAjDJQr/DcAssb4TU=
+code.cloudfoundry.org/go-diodes v0.0.0-20240813203737-5032edb05ceb/go.mod h1:PfkH7ePa4EhyC6VRisbZJOaIHhas7ofB0heANM//Z6Y=
 crawshaw.io/iox v0.0.0-20181124134642-c51c3df30797/go.mod h1:sXBiorCo8c46JlQV3oXPKINnZ8mcqnye1EkVkqsectk=
 crawshaw.io/sqlite v0.3.2/go.mod h1:igAO5JulrQ1DbdZdtVq48mnZUBAPOeFzer7VhDWNtW4=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
@@ -88,8 +88,6 @@ github.com/fasthttp/router v1.5.2/go.mod h1:C8EY53ozOwpONyevc/V7Gr8pqnEjwnkFFqPo
 github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
 github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
-github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
-github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
 github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE=
 github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE=
 github.com/glycerine/go-unsnap-stream v0.0.0-20190901134440-81cf024a9e0a/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE=
@@ -133,8 +131,8 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 h1:k7nVchz72niMH6YLQNvHSdIE7iqsQxK1P41mySCvssg=
+github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8 h1:FKHo8hFI3A+7w0aUQuYXQ+6EN5stWmeY/AZqtM8xk9k=
-github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw=
+github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gopherjs/gopherjs v0.0.0-20190309154008-847fc94819f9/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
@@ -175,6 +173,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
+github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/libp2p/go-reuseport v0.4.0 h1:nR5KU7hD0WxXCJbmw7r2rhRYruNRl2koHw8fQscQm2s=
 github.com/libp2p/go-reuseport v0.4.0/go.mod h1:ZtI03j/wO5hZVDFo2jKywN6bYKWLOy8Se6DrI2E1cLU=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
@@ -206,8 +206,8 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs=
 github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo/v2 v2.19.1 h1:QXgq3Z8Crl5EL1WBAC98A5sEBHARrAJNzAmMxzLcRF0=
+github.com/onsi/ginkgo/v2 v2.20.0 h1:PE84V2mHqoT1sglvHc8ZdQtPcwmvvt29WLEEO3xmdZw=
-github.com/onsi/ginkgo/v2 v2.19.1/go.mod h1:O3DtEWQkPa/F7fBMgmZQKKsluAy8pd3rEQdrjkPb9zA=
+github.com/onsi/ginkgo/v2 v2.20.0/go.mod h1:lG9ey2Z29hR41WMVthyJBGUBcBhGOtoPF2VFMvBXFCI=
 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k=
 github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY=
@@ -223,8 +223,8 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP
 github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs=
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
 github.com/prometheus/client_golang v1.5.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU=
-github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE=
+github.com/prometheus/client_golang v1.20.2 h1:5ctymQzZlyOON1666svgwn3s6IKWgfbjsejTMiXIyjg=
-github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho=
+github.com/prometheus/client_golang v1.20.2/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
@@ -234,8 +234,8 @@ github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQy
 github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4=
-github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc=
+github.com/prometheus/common v0.57.0 h1:Ro/rKjwdq9mZn1K5QPctzh+MA4Lp0BuYk5ZZEVhoNcY=
-github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8=
+github.com/prometheus/common v0.57.0/go.mod h1:7uRPFSUTbfZWsJ7MHY56sqt7hLQu3bxXHDnNhl8E9qI=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
@@ -280,6 +280,8 @@ github.com/valyala/fasthttp v1.55.0 h1:Zkefzgt6a7+bVKHnu/YaYSOPfNYNisSVBo/unVCf8
 github.com/valyala/fasthttp v1.55.0/go.mod h1:NkY9JtkrpPKmgwV3HTaS2HWaJss9RSIsRVfcxxoHiOM=
 github.com/willf/bitset v1.1.9/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
 github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
+github.com/zeebo/bencode v1.0.0 h1:zgop0Wu1nu4IexAZeCZ5qbsjU4O1vMrfCrVgUjbHVuA=
+github.com/zeebo/bencode v1.0.0/go.mod h1:Ct7CkrWIQuLWAy9M3atFHYq4kG9Ao/SsY5cdtCXmp9Y=
 go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
 go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
 go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
@@ -288,8 +290,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
 golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw=
 golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8=
+golang.org/x/exp v0.0.0-20240823005443-9b4947da3948 h1:kx6Ds3MlpiUHKj7syVnbp57++8WpuKPcR5yjLBjvLEA=
-golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY=
+golang.org/x/exp v0.0.0-20240823005443-9b4947da3948/go.mod h1:akd2r19cwCdwSwWeIdzYQGa/EZZyqcOdwWiwj5L5eKQ=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
 golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -330,8 +332,8 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM=
+golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg=
-golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc=
@@ -344,8 +346,8 @@ golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGm
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.23.0 h1:SGsXPZ+2l4JsgaCKkx+FQ9YZ5XEtA1GZYuoDjenLjvg=
+golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24=
-golang.org/x/tools v0.23.0/go.mod h1:pnu6ufv6vQkll6szChhK3C3L/ruaIv5eBeztNG8wtsI=
+golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=

middleware/torrentapproval/container/directory/directory.go

@@ -6,22 +6,34 @@ package directory
 
 import (
 	"context"
+	"crypto/sha1"
 	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
 
-	"github.com/anacrolix/torrent/metainfo"
-	"github.com/anacrolix/torrent/util/dirwatch"
 	"github.com/minio/sha256-simd"
 
+	"github.com/zeebo/bencode"
+
 	"github.com/sot-tech/mochi/bittorrent"
 	"github.com/sot-tech/mochi/middleware/torrentapproval/container"
 	"github.com/sot-tech/mochi/middleware/torrentapproval/container/list"
 	"github.com/sot-tech/mochi/pkg/conf"
 	"github.com/sot-tech/mochi/pkg/log"
+	"github.com/sot-tech/mochi/pkg/str2bytes"
 	"github.com/sot-tech/mochi/storage"
 )
 
 var logger = log.NewLogger("middleware/torrent approval/directory")
 
+const (
+	defaultPeriod  = time.Minute
+	maxTorrentSize = 10 * 1024 * 1024
+)
+
 func init() {
 	container.Register("directory", build)
 }
@@ -32,6 +44,8 @@ type Config struct {
 	list.Config
 	// Path in filesystem where torrent files stored and should be watched
 	Path string
+	// Period is time between two Path checks
+	Period time.Duration
 }
 
 func build(conf conf.MapConfig, st storage.DataStorage) (container.Container, error) {
@@ -46,7 +60,7 @@ func build(conf conf.MapConfig, st storage.DataStorage) (container.Container, er
 			Storage:    st,
 			StorageCtx: c.StorageCtx,
 		},
-		watcher: nil,
+		closed: make(chan bool),
 	}
 	if len(d.StorageCtx) == 0 {
 		logger.Warn().
@@ -56,77 +70,132 @@ func build(conf conf.MapConfig, st storage.DataStorage) (container.Container, er
 			Msg("falling back to default configuration")
 		d.StorageCtx = container.DefaultStorageCtxName
 	}
-	var w *dirwatch.Instance
+	if c.Period == 0 {
-	if w, err = dirwatch.New(c.Path); err != nil {
+		logger.Warn().
-		return nil, fmt.Errorf("unable to initialize directory watch: %w", err)
+			Str("name", "Period").
+			Dur("provided", 0).
+			Dur("default", defaultPeriod).
+			Msg("falling back to default configuration")
+		c.Period = defaultPeriod
 	}
-	d.watcher = w
+	go d.runScan(c.Path, c.Period)
-	go func() {
+	return d, err
-		for event := range d.watcher.Events {
+}
-			var mi *metainfo.MetaInfo
+
-			if mi, err = metainfo.LoadFromFile(event.TorrentFilePath); err == nil {
+// BencodeRawBytes wrapper for byte slice to get raw 'info' section from
-				s256 := sha256.New()
+// torrent file
-				s256.Write(mi.InfoBytes)
+type BencodeRawBytes []byte
-				v2hash, _ := bittorrent.NewInfoHash(s256.Sum(nil))
+
-				switch event.Change {
+// UnmarshalBencode just appends raw byte slice to result
-				case dirwatch.Added:
+func (ba *BencodeRawBytes) UnmarshalBencode(in []byte) error {
-					var name string
+	*ba = append([]byte(nil), in...)
-					if info, err := mi.UnmarshalInfo(); err == nil {
+	return nil
-						name = info.Name
+}
-					} else {
+
-						logger.Error().
+type torrentRawInfoStruct struct {
-							Err(err).
+	Info BencodeRawBytes `bencode:"info"`
-							Str("file", event.TorrentFilePath).
+}
-							Stringer("infoHash", event.InfoHash).
+
-							Stringer("infoHashV2", v2hash).
+type torrentNameInfoStruct struct {
-							Msg("unable to unmarshal torrent info")
+	Name string `bencode:"name"`
+}
+
+func (d *directory) runScan(path string, period time.Duration) {
+	t := time.NewTicker(period)
+	defer t.Stop()
+	files := make(map[string][2]bittorrent.InfoHash)
+	tmpFiles := make(map[string]bool)
+	// nolint:gosec
+	s1, s2 := sha1.New(), sha256.New()
+	for {
+		select {
+		case <-d.closed:
+			return
+		case <-t.C:
+			logger.Debug().Msg("starting directory scan")
+			if entries, err := os.ReadDir(path); err == nil {
+				for _, e := range entries {
+					if !e.IsDir() && strings.ToLower(filepath.Ext(e.Name())) == ".torrent" {
+						tmpFiles[filepath.Join(path, e.Name())] = true
 					}
-					if len(name) == 0 {
-						name = list.DUMMY
-					}
-					bName := []byte(name)
-					logger.Err(d.Storage.Put(context.Background(), d.StorageCtx, storage.Entry{
-						Key:   event.InfoHash.AsString(),
-						Value: bName,
-					}, storage.Entry{
-						Key:   v2hash.RawString(),
-						Value: bName,
-					}, storage.Entry{
-						Key:   v2hash.TruncateV1().RawString(),
-						Value: bName,
-					})).
-						Str("action", "add").
-						Str("file", event.TorrentFilePath).
-						Stringer("infoHash", event.InfoHash).
-						Stringer("infoHashV2", v2hash).
-						Msg("approval torrent watcher event")
-				case dirwatch.Removed:
-					logger.Err(d.Storage.Delete(context.Background(), c.StorageCtx, event.InfoHash.AsString(), v2hash.RawString(), v2hash.TruncateV1().RawString())).
-						Str("action", "delete").
-						Str("file", event.TorrentFilePath).
-						Stringer("infoHash", event.InfoHash).
-						Stringer("infoHashV2", v2hash).
-						Msg("approval torrent watcher event")
 				}
+				for p := range tmpFiles {
+					if _, exists := files[p]; !exists {
+						var f *os.File
+						if f, err = os.Open(p); err == nil {
+							var info torrentRawInfoStruct
+							err = bencode.NewDecoder(io.LimitReader(f, maxTorrentSize)).Decode(&info)
+							_ = f.Close()
+							if err == nil {
+								s1.Write(info.Info)
+								h1, _ := bittorrent.NewInfoHash(s1.Sum(nil))
+								s1.Reset()
+
+								s2.Write(info.Info)
+								h2, _ := bittorrent.NewInfoHash(s2.Sum(nil))
+								s2.Reset()
+
+								files[p] = [2]bittorrent.InfoHash{h1, h2}
+								var name torrentNameInfoStruct
+								if err := bencode.DecodeBytes(info.Info, &name); err != nil {
+									logger.Warn().
+										Err(err).
+										Str("file", p).
+										Msg("unable to unmarshal torrent info")
+								}
+								if len(name.Name) == 0 {
+									name.Name = list.DUMMY
+								}
+								bName := str2bytes.StringToBytes(name.Name)
+								logger.Err(d.Storage.Put(context.Background(), d.StorageCtx, storage.Entry{
+									Key:   h1.RawString(),
+									Value: bName,
+								}, storage.Entry{
+									Key:   h2.RawString(),
+									Value: bName,
+								}, storage.Entry{
+									Key:   h2.TruncateV1().RawString(),
+									Value: bName,
+								})).
+									Str("file", p).
+									Stringer("infoHash", h1).
+									Stringer("infoHashV2", h2).
+									Msg("added torrent to approval list")
+							}
+						}
+						if err != nil {
+							logger.Warn().Err(err).Str("file", p).Msg("unable to read file")
+						}
+					}
+				}
+				for p, ih := range files {
+					if _, isOk := tmpFiles[p]; !isOk {
+						delete(files, p)
+						logger.Err(d.Storage.Delete(context.Background(), d.StorageCtx, ih[0].RawString(),
+							ih[1].RawString(), ih[1].TruncateV1().RawString())).
+							Str("file", p).
+							Stringer("infoHash", ih[1]).
+							Stringer("infoHashV2", ih[1]).
+							Msg("deleted torrent from approval list")
+					}
+				}
+				clear(tmpFiles)
 			} else {
-				logger.Error().Err(err).
+				logger.Warn().Err(err).Msg("unable to get directory content")
-					Str("file", event.TorrentFilePath).
-					Msg("unable to load torrent file")
 			}
 		}
-	}()
+	}
-	return d, err
 }
 
 type directory struct {
 	list.List
-	watcher *dirwatch.Instance
+	closed chan bool
 }
 
 // Close closes watching of torrent directory
 func (d *directory) Close() error {
-	if d.watcher != nil {
+	if d.closed != nil {
-		d.watcher.Close()
+		close(d.closed)
 	}
 	return nil
 }

middleware/torrentapproval/torrentapproval_test.go

@@ -95,6 +95,7 @@ func TestHandleAnnounce(t *testing.T) {
 			} else {
 				require.Equal(t, err, ErrTorrentUnapproved)
 			}
+			_ = h.(*hook).Close()
 		})
 	}
 }