From 048ace4d428fb0e1a4b55047bff77b6d5166f5d7 Mon Sep 17 00:00:00 2001 From: "Lawrence, Rendall" Date: Wed, 17 Aug 2022 15:10:23 +0300 Subject: [PATCH 1/7] (untested) reimplement `jwt` middleware * change github.com/SermoDigital/jose to github.com/golang-jwt/jwt * remove pkg/errors dependency * update dependencies TODO: * add test * flag if announce/scrape handle needed --- bittorrent/bittorrent.go | 2 +- go.mod | 27 +++--- go.sum | 60 ++++++------ middleware/jwt/jwt.go | 198 ++++++++++++++------------------------- 4 files changed, 117 insertions(+), 170 deletions(-) diff --git a/bittorrent/bittorrent.go b/bittorrent/bittorrent.go index 8fc840e..8a62a29 100644 --- a/bittorrent/bittorrent.go +++ b/bittorrent/bittorrent.go @@ -8,11 +8,11 @@ import ( "crypto/sha256" "encoding/binary" "encoding/hex" + "errors" "fmt" "net" "net/netip" - "github.com/pkg/errors" "github.com/rs/zerolog" ) diff --git a/go.mod b/go.mod index 27a2a7a..d86caa2 100644 --- a/go.mod +++ b/go.mod @@ -3,18 +3,17 @@ module github.com/sot-tech/mochi go 1.18 require ( - code.cloudfoundry.org/go-diodes v0.0.0-20220601181242-ac2da19efd60 - github.com/SermoDigital/jose v0.9.2-0.20180104203859-803625baeddc + code.cloudfoundry.org/go-diodes v0.0.0-20220725190411-383eb6634c40 + github.com/MicahParks/keyfunc v1.2.2 github.com/anacrolix/torrent v1.46.0 github.com/go-redis/redis/v8 v8.11.5 - github.com/jackc/pgx/v4 v4.16.1 + github.com/golang-jwt/jwt/v4 v4.4.2 + github.com/jackc/pgx/v4 v4.17.0 github.com/julienschmidt/httprouter v1.3.0 github.com/libp2p/go-reuseport v0.2.0 - github.com/mendsley/gojwk v0.0.0-20141217222730-4d5ec6e58103 github.com/minio/sha256-simd v1.0.0 github.com/mitchellh/mapstructure v1.5.0 - github.com/pkg/errors v0.9.1 - github.com/prometheus/client_golang v1.12.2 + github.com/prometheus/client_golang v1.13.0 github.com/rs/zerolog v1.27.0 github.com/stretchr/testify v1.8.0 gopkg.in/yaml.v3 v3.0.1 @@ -34,23 +33,23 @@ require ( github.com/golang/protobuf v1.5.2 // indirect github.com/huandu/xstrings v1.3.2 // indirect github.com/jackc/chunkreader/v2 v2.0.1 // indirect - github.com/jackc/pgconn v1.12.1 // indirect + github.com/jackc/pgconn v1.13.0 // indirect github.com/jackc/pgio v1.0.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect - github.com/jackc/pgproto3/v2 v2.3.0 // indirect + github.com/jackc/pgproto3/v2 v2.3.1 // indirect github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect - github.com/jackc/pgtype v1.11.0 // indirect + github.com/jackc/pgtype v1.12.0 // indirect github.com/jackc/puddle v1.2.1 // indirect github.com/klauspost/cpuid/v2 v2.1.0 // indirect - github.com/mattn/go-colorable v0.1.12 // indirect - github.com/mattn/go-isatty v0.0.14 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.16 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.2.0 // indirect github.com/prometheus/common v0.37.0 // indirect - github.com/prometheus/procfs v0.7.3 // indirect + github.com/prometheus/procfs v0.8.0 // indirect golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa // indirect - golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f // indirect + golang.org/x/sys v0.0.0-20220817070843-5a390386f1f2 // indirect golang.org/x/text v0.3.7 // indirect - google.golang.org/protobuf v1.28.0 // indirect + google.golang.org/protobuf v1.28.1 // indirect ) diff --git a/go.sum b/go.sum index 669b55d..5dc7073 100644 --- a/go.sum +++ b/go.sum @@ -30,19 +30,19 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -code.cloudfoundry.org/go-diodes v0.0.0-20220601181242-ac2da19efd60 h1:hBycJRDauXKMEAOl90iFjTqrQ4VNGb918x3lLLDXBBU= -code.cloudfoundry.org/go-diodes v0.0.0-20220601181242-ac2da19efd60/go.mod h1:HLP7HKUU1eqMAGMk247yT91tDDi4xxnehkyXh6hGcr0= +code.cloudfoundry.org/go-diodes v0.0.0-20220725190411-383eb6634c40 h1:wzkYwwcf4uMGcDpn48WAbq8GtoqDny49tdQ4zJVAsmo= +code.cloudfoundry.org/go-diodes v0.0.0-20220725190411-383eb6634c40/go.mod h1:Nx9ASXN4nIlRDEXv+qXE3dpuhnTnO28Lxl/bMUd6BMc= crawshaw.io/iox v0.0.0-20181124134642-c51c3df30797/go.mod h1:sXBiorCo8c46JlQV3oXPKINnZ8mcqnye1EkVkqsectk= crawshaw.io/sqlite v0.3.2/go.mod h1:igAO5JulrQ1DbdZdtVq48mnZUBAPOeFzer7VhDWNtW4= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= +github.com/MicahParks/keyfunc v1.2.2 h1:MtA/6bnmhcyDMarVF9QRQmfKg1ssKxH1C+k3Gw8DY1g= +github.com/MicahParks/keyfunc v1.2.2/go.mod h1:GWZYIBflWXRPShQPMFRrUg+8buvyD0IWeg3Fi2rcrME= github.com/RoaringBitmap/roaring v0.4.7/go.mod h1:8khRDP4HmeXns4xIj9oGrKSz7XTQiJx2zgh7AcNke4w= github.com/RoaringBitmap/roaring v0.4.17/go.mod h1:D3qVegWTmfCaX4Bl5CrBE9hfrSrrXIr8KVNvRsDi1NI= github.com/RoaringBitmap/roaring v0.4.23/go.mod h1:D0gp8kJQgE1A4LQ5wFLggQEyvDi06Mq5mKs52e1TwOo= -github.com/SermoDigital/jose v0.9.2-0.20180104203859-803625baeddc h1:MhBvG7RLaLqlyjxMR6of35vt6MVQ+eXMcgn9X/sy0FE= -github.com/SermoDigital/jose v0.9.2-0.20180104203859-803625baeddc/go.mod h1:ARgCUhI1MHQH+ONky/PAtmVHQrP5JlGY0F3poXOp/fA= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -146,6 +146,9 @@ github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPh github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/golang-jwt/jwt/v4 v4.4.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-jwt/jwt/v4 v4.4.2 h1:rcc4lwaZgFMCZ5jxF9ABolDcIHdBytAFgqFPbSJQAYs= +github.com/golang-jwt/jwt/v4 v4.4.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -188,7 +191,7 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o= +github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -227,8 +230,8 @@ github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsU github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o= github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8/2JY= github.com/jackc/pgconn v1.9.1-0.20210724152538-d89c8390a530/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= -github.com/jackc/pgconn v1.12.1 h1:rsDFzIpRk7xT4B8FufgpCCeyjdNpKyghZeSefViE5W8= -github.com/jackc/pgconn v1.12.1/go.mod h1:ZkhRC59Llhrq3oSfrikvwQ5NaxYExr6twkdkMLaKono= +github.com/jackc/pgconn v1.13.0 h1:3L1XMNV2Zvca/8BYhzcRFS70Lr0WlDg16Di6SFGAbys= +github.com/jackc/pgconn v1.13.0/go.mod h1:AnowpAqO4CMIIJNZl2VJp+KrkAZciAkhEl0W0JIobpI= github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= @@ -244,22 +247,22 @@ github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvW github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= -github.com/jackc/pgproto3/v2 v2.3.0 h1:brH0pCGBDkBW07HWlN/oSBXrmo3WB0UvZd1pIuDcL8Y= -github.com/jackc/pgproto3/v2 v2.3.0/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.3.1 h1:nwj7qwf0S+Q7ISFfBndqeLwSwxs+4DPsbRFjECT1Y4Y= +github.com/jackc/pgproto3/v2 v2.3.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg= github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= github.com/jackc/pgtype v1.8.1-0.20210724151600-32e20a603178/go.mod h1:C516IlIV9NKqfsMCXTdChteoXmwgUceqaLfjg2e3NlM= -github.com/jackc/pgtype v1.11.0 h1:u4uiGPz/1hryuXzyaBhSk6dnIyyG2683olG2OV+UUgs= -github.com/jackc/pgtype v1.11.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= +github.com/jackc/pgtype v1.12.0 h1:Dlq8Qvcch7kiehm8wPGIW0W3KsCCHJnRacKW0UM8n5w= +github.com/jackc/pgtype v1.12.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs= -github.com/jackc/pgx/v4 v4.16.1 h1:JzTglcal01DrghUqt+PmzWsZx/Yh7SC/CTQmSBMTd0Y= -github.com/jackc/pgx/v4 v4.16.1/go.mod h1:SIhx0D5hoADaiXZVyv+3gSm3LCIIINTVO0PficsvWGQ= +github.com/jackc/pgx/v4 v4.17.0 h1:Hsx+baY8/zU2WtPLQyZi8WbecgcsWEeyoK1jvg/WgIo= +github.com/jackc/pgx/v4 v4.17.0/go.mod h1:Gd6RmOhtFLTu8cp/Fhq4kP195KrshxYJH3oW8AWJ1pw= github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= @@ -303,17 +306,17 @@ github.com/libp2p/go-reuseport v0.2.0 h1:18PRvIMlpY6ZK85nIAicSBuXXvrYoSw3dsBAR7z github.com/libp2p/go-reuseport v0.2.0/go.mod h1:bvVho6eLMm6Bz5hmU0LYN3ixd3nPPvtIlaURZZgOY4k= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/mendsley/gojwk v0.0.0-20141217222730-4d5ec6e58103 h1:Z/i1e+gTZrmcGeZyWckaLfucYG6KYOXLWo4co8pZYNY= -github.com/mendsley/gojwk v0.0.0-20141217222730-4d5ec6e58103/go.mod h1:o9YPB5aGP8ob35Vy6+vyq3P3bWe7NQWzf+JLiXCiMaE= github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= @@ -332,7 +335,7 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.19.0 h1:4ieX6qQjPP/BfC3mpsAtIGGlxTWPeA3Inl/7DtXw1tw= +github.com/onsi/gomega v1.20.0 h1:8W0cWlwFkflGPLltQvLRB7ZVD5HuP6ng320w2IS245Q= github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= @@ -350,8 +353,8 @@ github.com/prometheus/client_golang v1.5.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3O github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= -github.com/prometheus/client_golang v1.12.2 h1:51L9cDoUHVrXx4zWYlcLQIZ+d+VXHgqnYKkIuq4g/34= -github.com/prometheus/client_golang v1.12.2/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= +github.com/prometheus/client_golang v1.13.0 h1:b71QUfeo5M8gq2+evJdTPfZhYMAU0uKPkyPJ7TPsloU= +github.com/prometheus/client_golang v1.13.0/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -373,8 +376,9 @@ github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+Gx github.com/prometheus/procfs v0.0.11/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= +github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= @@ -515,9 +519,10 @@ golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220225172249-27dd8689420f h1:oA4XRj0qtSt8Yo1Zms0CUlsT3KG69V2UGQWPBxujDmc= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 h1:HVyaeDAYux4pnY+D/SiwmLOR36ewZ4iGQIIrtnuCjFA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -584,8 +589,9 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f h1:v4INt8xihDGvnrfjMDVXGxw9wrfxYyCjk0KbXjhR55s= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220817070843-5a390386f1f2 h1:fqTvyMIIj+HRzMmnzr9NtpHP6uVpvB5fkHcgPDC4nu8= +golang.org/x/sys v0.0.0-20220817070843-5a390386f1f2/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -652,7 +658,6 @@ golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= @@ -731,8 +736,8 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= -google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -748,7 +753,6 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/middleware/jwt/jwt.go b/middleware/jwt/jwt.go index 6875968..4abb5db 100644 --- a/middleware/jwt/jwt.go +++ b/middleware/jwt/jwt.go @@ -3,23 +3,18 @@ // // JWTs are validated against the standard claims in RFC7519 along with an // extra "infohash" claim that verifies the client has access to the Swarm. -// RS256 keys are asychronously rotated from a provided JWK Set HTTP endpoint. +// RS256 keys are asynchronously rotated from a provided JWK Set HTTP endpoint. package jwt import ( "context" - "crypto" - "encoding/hex" - "encoding/json" + "crypto/subtle" "errors" "fmt" - "net/http" "time" - jc "github.com/SermoDigital/jose/crypto" - "github.com/SermoDigital/jose/jws" - "github.com/SermoDigital/jose/jwt" - "github.com/mendsley/gojwk" + "github.com/MicahParks/keyfunc" + "github.com/golang-jwt/jwt/v4" "github.com/sot-tech/mochi/bittorrent" "github.com/sot-tech/mochi/middleware" @@ -44,11 +39,11 @@ var ( // ErrInvalidJWT is returned when a JWT fails to verify. ErrInvalidJWT = bittorrent.ClientError("unapproved request: invalid jwt") - errInvalidInfoHashClaim = errors.New("claim \"infohash\" is invalid") + errInvalidInfoHashClaim = errors.New("token has invalid \"infohash\" claim") - errInvalidKid = errors.New("invalid kid") + errJWKsNotSet = errors.New("required parameters not provided: Issuer, Audience and/or JWKSetURL") - errUnknownKidSigner = errors.New("signed by unknown kid") + hmacAlgorithms = jwt.WithValidMethods([]string{jwt.SigningMethodHS256.Name, jwt.SigningMethodHS384.Name, jwt.SigningMethodHS512.Name}) ) // Config represents all the values required by this middleware to fetch JWKs @@ -61,86 +56,53 @@ type Config struct { } type hook struct { - cfg Config - publicKeys map[string]crypto.PublicKey - closing chan struct{} + cfg Config + jwks *keyfunc.JWKS } -func build(options conf.MapConfig, _ storage.PeerStorage) (middleware.Hook, error) { +type claims struct { + jwt.RegisteredClaims + InfoHash string `json:"infohash,omitempty"` +} + +func build(options conf.MapConfig, _ storage.PeerStorage) (h middleware.Hook, err error) { var cfg Config - if err := options.Unmarshal(&cfg); err != nil { + if err = options.Unmarshal(&cfg); err != nil { return nil, fmt.Errorf("middleware %s: %w", Name, err) } logger.Debug().Object("options", options).Msg("creating new JWT middleware") - h := &hook{ - cfg: cfg, - publicKeys: map[string]crypto.PublicKey{}, - closing: make(chan struct{}), - } - logger.Debug().Msg("performing initial fetch of JWKs") - if err := h.updateKeys(); err != nil { - return nil, fmt.Errorf("failed to fetch initial JWK Set: %w", err) - } - - go func() { - for { - select { - case <-h.closing: - return - case <-time.After(cfg.JWKUpdateInterval): - logger.Debug().Msg("performing fetch of JWKs") - _ = h.updateKeys() + if len(cfg.JWKSetURL) > 0 { + var jwks *keyfunc.JWKS + jwks, err = keyfunc.Get(cfg.JWKSetURL, keyfunc.Options{ + Ctx: context.Background(), + RefreshErrorHandler: func(err error) { + logger.Error().Err(err).Msg("error occurred while updating JWKs") + }, + RefreshInterval: cfg.JWKUpdateInterval, + RefreshUnknownKID: true, + }) + if err == nil { + h = &hook{ + cfg: cfg, + jwks: jwks, } } - }() - - return h, nil -} - -func (h *hook) updateKeys() error { - resp, err := http.Get(h.cfg.JWKSetURL) - if err != nil { - logger.Error().Err(err).Msg("failed to fetch JWK Set") - return err - } - defer resp.Body.Close() - var parsedJWKs gojwk.Key - err = json.NewDecoder(resp.Body).Decode(&parsedJWKs) - if err != nil { - logger.Error().Err(err).Msg("failed to decode JWK JSON") - return err + } else { + err = errJWKsNotSet } - keys := map[string]crypto.PublicKey{} - for _, parsedJWK := range parsedJWKs.Keys { - publicKey, err := parsedJWK.DecodePublicKey() - if err != nil { - logger.Error().Err(err).Msg("failed to decode JWK into public key") - return err - } - keys[parsedJWK.Kid] = publicKey - } - h.publicKeys = keys - - logger.Debug().Msg("successfully fetched JWK Set") - return nil + return } func (h *hook) Stop() stop.Result { logger.Debug().Msg("attempting to shutdown JWT middleware") - select { - case <-h.closing: - return stop.AlreadyStopped - default: - } c := make(stop.Channel) - go func() { - close(h.closing) - c.Done() - }() + if h.jwks != nil { + go h.jwks.EndBackground() + } return c.Result() } @@ -154,7 +116,11 @@ func (h *hook) HandleAnnounce(ctx context.Context, req *bittorrent.AnnounceReque return ctx, ErrMissingJWT } - if err := validateJWT(req.InfoHash, []byte(jwtParam), h.cfg.Issuer, h.cfg.Audience, h.publicKeys); err != nil { + if errs := h.validateJWT(req.InfoHash, jwtParam); len(errs) > 0 { + logger.Info(). + Errs("errors", errs). + Object("source", req.RequestPeer). + Msg("JWT validation failed") return ctx, ErrInvalidJWT } @@ -166,70 +132,48 @@ func (h *hook) HandleScrape(ctx context.Context, _ *bittorrent.ScrapeRequest, _ return ctx, nil } -func validateJWT(ih bittorrent.InfoHash, jwtBytes []byte, cfgIss, cfgAud string, publicKeys map[string]crypto.PublicKey) error { - parsedJWT, err := jws.ParseJWT(jwtBytes) +func (h *hook) validateJWT(ih bittorrent.InfoHash, rawJwt string) []error { + // KeyFunc will check KID, Parse will check ALG and signature + errs := make([]error, 0, 4) + token, err := jwt.ParseWithClaims(rawJwt, claims{}, h.jwks.Keyfunc, hmacAlgorithms) if err != nil { - return err + return []error{err} } - claims := parsedJWT.Claims() - if iss, ok := claims.Issuer(); !ok || iss != cfgIss { + if err = token.Claims.Valid(); err != nil { + errs = append(errs, err) + } + + claims := token.Claims.(claims) + + if !claims.VerifyIssuer(h.cfg.Issuer, true) { logger.Debug(). - Bool("exists", ok). - Str("claim", iss). - Str("config", cfgIss). + Str("provided", claims.Issuer). + Str("required", h.cfg.Issuer). Msg("unequal or missing issuer when validating JWT") - return jwt.ErrInvalidISSClaim + errs = append(errs, jwt.ErrTokenInvalidIssuer) } - if auds, ok := claims.Audience(); !ok || !in(cfgAud, auds) { + if !claims.VerifyAudience(h.cfg.Audience, true) { logger.Debug(). - Bool("exists", ok). - Strs("claim", auds). - Str("config", cfgAud). + Strs("provided", claims.Audience). + Str("required", h.cfg.Audience). Msg("unequal or missing audience when validating JWT") - return jwt.ErrInvalidAUDClaim + errs = append(errs, jwt.ErrTokenInvalidAudience) } - ihHex := hex.EncodeToString([]byte(ih)) - if ihClaim, ok := claims.Get("infohash").(string); !ok || ihClaim != ihHex { - logger.Debug(). - Bool("exists", ok). - Str("claim", ihClaim). - Str("request", ihHex). - Msg("unequal or missing infohash when validating JWT") - return errInvalidInfoHashClaim - } - - parsedJWS := parsedJWT.(jws.JWS) - kid, ok := parsedJWS.Protected().Get("kid").(string) - if !ok { - logger.Debug(). - Bool("exists", ok). - Str("claim", kid). - Msg("missing kid when validating JWT") - return errInvalidKid - } - publicKey, ok := publicKeys[kid] - if !ok { - logger.Debug().Str("claim", kid).Msg("missing public key forkid when validating JWT") - return errUnknownKidSigner - } - - err = parsedJWS.Verify(publicKey, jc.SigningMethodRS256) + providedIh, err := bittorrent.NewInfoHash(claims.InfoHash) if err != nil { - logger.Debug().Err(err).Msg("failed to verify signature of JWT") - return err + errs = append(errs, err) + } + if subtle.ConstantTimeCompare([]byte(providedIh), []byte(ih)) != 0 { + logger.Error(). + Err(err). + Stringer("provided", providedIh). + Stringer("required", ih). + Msg("invalid or unequal info hash when validating JWT") + errs = append(errs, errInvalidInfoHashClaim) } - return nil -} - -func in(x string, xs []string) bool { - for _, y := range xs { - if x == y { - return true - } - } - return false + return errs } From f5a58630db9e2a3d3ce6300141cad672e1b0f806 Mon Sep 17 00:00:00 2001 From: "Lawrence, Rendall" Date: Thu, 18 Aug 2022 17:06:22 +0300 Subject: [PATCH 2/7] (tested) add simple jwt middleware tests --- middleware/jwt/jwt.go | 18 ++-- middleware/jwt/jwt_test.go | 180 +++++++++++++++++++++++++++++++++++++ 2 files changed, 191 insertions(+), 7 deletions(-) create mode 100644 middleware/jwt/jwt_test.go diff --git a/middleware/jwt/jwt.go b/middleware/jwt/jwt.go index 4abb5db..8a3ace4 100644 --- a/middleware/jwt/jwt.go +++ b/middleware/jwt/jwt.go @@ -8,7 +8,6 @@ package jwt import ( "context" - "crypto/subtle" "errors" "fmt" "time" @@ -43,7 +42,13 @@ var ( errJWKsNotSet = errors.New("required parameters not provided: Issuer, Audience and/or JWKSetURL") - hmacAlgorithms = jwt.WithValidMethods([]string{jwt.SigningMethodHS256.Name, jwt.SigningMethodHS384.Name, jwt.SigningMethodHS512.Name}) + hmacAlgorithms = jwt.WithValidMethods([]string{ + jwt.SigningMethodHS256.Alg(), jwt.SigningMethodHS384.Alg(), jwt.SigningMethodHS512.Alg(), + jwt.SigningMethodRS256.Alg(), jwt.SigningMethodRS384.Alg(), jwt.SigningMethodRS512.Alg(), + jwt.SigningMethodPS256.Alg(), jwt.SigningMethodPS384.Alg(), jwt.SigningMethodPS512.Alg(), + jwt.SigningMethodES256.Alg(), jwt.SigningMethodES384.Alg(), jwt.SigningMethodES512.Alg(), + jwt.SigningMethodEdDSA.Alg(), + }) ) // Config represents all the values required by this middleware to fetch JWKs @@ -135,7 +140,8 @@ func (h *hook) HandleScrape(ctx context.Context, _ *bittorrent.ScrapeRequest, _ func (h *hook) validateJWT(ih bittorrent.InfoHash, rawJwt string) []error { // KeyFunc will check KID, Parse will check ALG and signature errs := make([]error, 0, 4) - token, err := jwt.ParseWithClaims(rawJwt, claims{}, h.jwks.Keyfunc, hmacAlgorithms) + claims := new(claims) + token, err := jwt.ParseWithClaims(rawJwt, claims, h.jwks.Keyfunc, hmacAlgorithms) if err != nil { return []error{err} } @@ -144,8 +150,6 @@ func (h *hook) validateJWT(ih bittorrent.InfoHash, rawJwt string) []error { errs = append(errs, err) } - claims := token.Claims.(claims) - if !claims.VerifyIssuer(h.cfg.Issuer, true) { logger.Debug(). Str("provided", claims.Issuer). @@ -166,8 +170,8 @@ func (h *hook) validateJWT(ih bittorrent.InfoHash, rawJwt string) []error { if err != nil { errs = append(errs, err) } - if subtle.ConstantTimeCompare([]byte(providedIh), []byte(ih)) != 0 { - logger.Error(). + if providedIh != ih { + logger.Debug(). Err(err). Stringer("provided", providedIh). Stringer("required", ih). diff --git a/middleware/jwt/jwt_test.go b/middleware/jwt/jwt_test.go new file mode 100644 index 0000000..4e42130 --- /dev/null +++ b/middleware/jwt/jwt_test.go @@ -0,0 +1,180 @@ +package jwt + +import ( + "context" + "crypto/ecdsa" + "crypto/elliptic" + "encoding/base64" + "encoding/json" + "math/rand" + "net/http" + "net/http/httptest" + "strconv" + "testing" + "time" + + "github.com/golang-jwt/jwt/v4" + "github.com/minio/sha256-simd" + "github.com/rs/zerolog" + "github.com/stretchr/testify/require" + + "github.com/sot-tech/mochi/bittorrent" + "github.com/sot-tech/mochi/pkg/conf" + "github.com/sot-tech/mochi/pkg/log" + _ "github.com/sot-tech/mochi/pkg/randseed" +) + +const ( + privKeyPEM = ` +-----BEGIN PRIVATE KEY----- +MEECAQAwEwYHKoZIzj0CAQYIKoZIzj0DAQcEJzAlAgEBBCCI7Zc2IUKazCBCK5VY +WxxE6lVhGR+exaWgrh0Yq9t4gQ== +-----END PRIVATE KEY----- +` +) + +var ( + privKey *ecdsa.PrivateKey + infoHash bittorrent.InfoHash + jwksData JWKSKeys +) + +type JWKSKey struct { + KeyType string `json:"kty"` + Usage string `json:"use"` + KeyID string `json:"kid"` + Algorithm string `json:"alg"` + Curve string `json:"crv"` + X string `json:"x"` + Y string `json:"y"` +} + +type JWKSKeys struct { + Keys []JWKSKey `json:"keys"` +} + +type params struct { + data map[string]string +} + +func (p params) String(key string) (out string, found bool) { + out, found = p.data[key] + return +} + +func (params) RawPath() (s string) { + return +} + +func (params) RawQuery() (s string) { + return +} + +func (params) MarshalZerologObject(*zerolog.Event) { + return +} + +func init() { + _ = log.ConfigureLogger("", "info", false, false) + privKey, _ = jwt.ParseECPrivateKeyFromPEM([]byte(privKeyPEM)) + ihBytes := make([]byte, bittorrent.InfoHashV1Len) + rand.Read(ihBytes) + infoHash, _ = bittorrent.NewInfoHash(ihBytes) + s2 := sha256.New() + s2.Write(elliptic.Marshal(privKey.PublicKey.Curve, privKey.PublicKey.X, privKey.PublicKey.Y)) + jwksData = JWKSKeys{Keys: []JWKSKey{ + { + KeyType: "EC", + Usage: "sig", + KeyID: base64.RawURLEncoding.EncodeToString(s2.Sum(nil)), + Algorithm: jwt.SigningMethodES256.Name, + Curve: privKey.Curve.Params().Name, + X: base64.RawURLEncoding.EncodeToString(privKey.PublicKey.X.Bytes()), + Y: base64.RawURLEncoding.EncodeToString(privKey.PublicKey.Y.Bytes()), + }, + }} +} + +func TestHook_HandleAnnounceValid(t *testing.T) { + s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _ = json.NewEncoder(w).Encode(jwksData) + })) + defer s.Close() + + token := jwt.NewWithClaims(jwt.SigningMethodES256, claims{ + RegisteredClaims: jwt.RegisteredClaims{ + Issuer: "CN=test", + Subject: "CN=test", + Audience: []string{"test"}, + ExpiresAt: &jwt.NumericDate{Time: time.Now().Add(time.Hour)}, + NotBefore: &jwt.NumericDate{Time: time.Now().Add(-time.Hour)}, + ID: strconv.FormatInt(rand.Int63(), 16), + }, + InfoHash: infoHash.String(), + }) + + token.Header["kid"] = jwksData.Keys[0].KeyID + tokenString, err := token.SignedString(privKey) + require.Nil(t, err) + //goland:noinspection HttpUrlsUsage + cfg := conf.MapConfig{ + "issuer": "CN=test", + "audience": "test", + "jwk_set_url": "http://" + s.Listener.Addr().String(), + "jwk_set_update_interval": time.Minute, + } + h, err := build(cfg, nil) + require.Nil(t, err) + data := make(map[string]string) + data["jwt"] = tokenString + _, err = h.HandleAnnounce(context.Background(), &bittorrent.AnnounceRequest{ + InfoHash: infoHash, + RequestPeer: bittorrent.RequestPeer{}, + Params: ¶ms{data: data}, + }, nil) + require.Nil(t, err) +} + +func TestHook_HandleAnnounceInvalid(t *testing.T) { + s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _ = json.NewEncoder(w).Encode(jwksData) + })) + defer s.Close() + + // now we wll use HMAC-SHA256 with invalid random key + // all errors should be nil except announce request + token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims{ + RegisteredClaims: jwt.RegisteredClaims{ + Issuer: "CN=test", + Subject: "CN=test", + Audience: []string{"test"}, + ExpiresAt: &jwt.NumericDate{Time: time.Now().Add(time.Hour)}, + NotBefore: &jwt.NumericDate{Time: time.Now().Add(-time.Hour)}, + ID: strconv.FormatInt(rand.Int63(), 16), + }, + InfoHash: infoHash.String(), + }) + + token.Header["kid"] = jwksData.Keys[0].KeyID + k := make([]byte, 20) + rand.Read(k) + tokenString, err := token.SignedString(k) + require.Nil(t, err) + //goland:noinspection HttpUrlsUsage + cfg := conf.MapConfig{ + "issuer": "CN=test", + "audience": "test", + "jwk_set_url": "http://" + s.Listener.Addr().String(), + "jwk_set_update_interval": time.Minute, + } + h, err := build(cfg, nil) + require.Nil(t, err) + data := make(map[string]string) + data["jwt"] = tokenString + _, err = h.HandleAnnounce(context.Background(), &bittorrent.AnnounceRequest{ + InfoHash: infoHash, + RequestPeer: bittorrent.RequestPeer{}, + Params: ¶ms{data: data}, + }, nil) + require.NotNil(t, err) +} From 498779aeaf1ae86dfbc7d1408844dfb7798b4618 Mon Sep 17 00:00:00 2001 From: "Lawrence, Rendall" Date: Thu, 25 Aug 2022 18:31:48 +0300 Subject: [PATCH 3/7] (partially tested) add scrape JWT validation * remove `xorshift` package, add internal function in `varinterval` * change `bittorrent.QueryParams` getters to search case insensitive keys --- bittorrent/params.go | 14 +- dist/example_config.yaml | 3 + middleware/jwt/jwt.go | 244 +++++++++++++++------ middleware/jwt/jwt_test.go | 50 ++++- middleware/pkg/random/entropy.go | 19 -- middleware/pkg/random/xorshift.go | 28 --- middleware/pkg/random/xorshift_test.go | 38 ---- middleware/varinterval/varinterval.go | 36 ++- middleware/varinterval/varinterval_test.go | 13 ++ 9 files changed, 276 insertions(+), 169 deletions(-) delete mode 100644 middleware/pkg/random/entropy.go delete mode 100644 middleware/pkg/random/xorshift.go delete mode 100644 middleware/pkg/random/xorshift_test.go diff --git a/bittorrent/params.go b/bittorrent/params.go index f7aaf02..3a08b64 100644 --- a/bittorrent/params.go +++ b/bittorrent/params.go @@ -180,15 +180,15 @@ func parseQuery(query string) (q *QueryParams, err error) { // String returns a string parsed from a query. Every key can be returned as a // string because they are encoded in the URL as strings. -func (qp *QueryParams) String(key string) (string, bool) { - value, ok := qp.params[key] +func (qp QueryParams) String(key string) (string, bool) { + value, ok := qp.params[strings.ToLower(key)] return value, ok } // Uint returns an uint parsed from a query. After being called, it is safe to // cast the uint64 to your desired length. -func (qp *QueryParams) Uint(key string, bitSize int) (uint64, error) { - str, exists := qp.params[key] +func (qp QueryParams) Uint(key string, bitSize int) (uint64, error) { + str, exists := qp.params[strings.ToLower(key)] if !exists { return 0, ErrKeyNotFound } @@ -197,17 +197,17 @@ func (qp *QueryParams) Uint(key string, bitSize int) (uint64, error) { } // InfoHashes returns a list of requested infohashes. -func (qp *QueryParams) InfoHashes() []InfoHash { +func (qp QueryParams) InfoHashes() []InfoHash { return qp.infoHashes } // RawPath returns the raw path from the parsed URL. -func (qp *QueryParams) RawPath() string { +func (qp QueryParams) RawPath() string { return qp.path } // RawQuery returns the raw query from the parsed URL. -func (qp *QueryParams) RawQuery() string { +func (qp QueryParams) RawQuery() string { return qp.query } diff --git a/dist/example_config.yaml b/dist/example_config.yaml index b8bfad2..250bfc5 100644 --- a/dist/example_config.yaml +++ b/dist/example_config.yaml @@ -264,10 +264,13 @@ mochi: prehooks: # - name: jwt # options: + # header: "authorization" # issuer: "https://issuer.com" # audience: "https://some.issuer.com" # jwk_set_url: "https://issuer.com/keys" # jwk_set_update_interval: 5m + # handle_announce: true + # handle_scrape: false # # - name: client approval # options: diff --git a/middleware/jwt/jwt.go b/middleware/jwt/jwt.go index 8a3ace4..8e700a2 100644 --- a/middleware/jwt/jwt.go +++ b/middleware/jwt/jwt.go @@ -1,15 +1,15 @@ -// Package jwt implements a Hook that fails an Announce if the client's request +// Package jwt implements a Hook that fails on Announce or Scrape if the client's request // is missing a valid JSON Web Token. // // JWTs are validated against the standard claims in RFC7519 along with an -// extra "infohash" claim that verifies the client has access to the Swarm. -// RS256 keys are asynchronously rotated from a provided JWK Set HTTP endpoint. +// extra "infohash(es)" claim that verifies the client has access to the Swarm. package jwt import ( "context" "errors" "fmt" + "strings" "time" "github.com/MicahParks/keyfunc" @@ -24,7 +24,11 @@ import ( ) // Name is the name by which this middleware is registered with Conf. -const Name = "jwt" +const ( + Name = "jwt" + authorizationHeader = "authorization" + bearerAuthPrefix = "bearer " +) func init() { middleware.RegisterBuilder(Name, build) @@ -38,8 +42,6 @@ var ( // ErrInvalidJWT is returned when a JWT fails to verify. ErrInvalidJWT = bittorrent.ClientError("unapproved request: invalid jwt") - errInvalidInfoHashClaim = errors.New("token has invalid \"infohash\" claim") - errJWKsNotSet = errors.New("required parameters not provided: Issuer, Audience and/or JWKSetURL") hmacAlgorithms = jwt.WithValidMethods([]string{ @@ -54,10 +56,13 @@ var ( // Config represents all the values required by this middleware to fetch JWKs // and verify JWTs. type Config struct { + Header string Issuer string Audience string JWKSetURL string `cfg:"jwk_set_url"` JWKUpdateInterval time.Duration `cfg:"jwk_set_update_interval"` + HandleAnnounce bool `cfg:"handle_announce"` + HandleScrape bool `cfg:"handle_scrape"` } type hook struct { @@ -65,30 +70,37 @@ type hook struct { jwks *keyfunc.JWKS } -type claims struct { - jwt.RegisteredClaims - InfoHash string `json:"infohash,omitempty"` -} - func build(options conf.MapConfig, _ storage.PeerStorage) (h middleware.Hook, err error) { var cfg Config + logger.Debug().Object("options", options).Msg("creating new JWT middleware") + if err = options.Unmarshal(&cfg); err != nil { return nil, fmt.Errorf("middleware %s: %w", Name, err) } - logger.Debug().Object("options", options).Msg("creating new JWT middleware") + if len(cfg.JWKSetURL) > 0 && len(cfg.Issuer) > 0 && len(cfg.Audience) > 0 { + if len(cfg.Header) == 0 { + cfg.Header = authorizationHeader + logger.Warn(). + Str("name", "Header"). + Str("default", cfg.Header). + Msg("falling back to default configuration") + } - if len(cfg.JWKSetURL) > 0 { var jwks *keyfunc.JWKS - jwks, err = keyfunc.Get(cfg.JWKSetURL, keyfunc.Options{ - Ctx: context.Background(), - RefreshErrorHandler: func(err error) { - logger.Error().Err(err).Msg("error occurred while updating JWKs") - }, - RefreshInterval: cfg.JWKUpdateInterval, - RefreshUnknownKID: true, - }) + if cfg.HandleAnnounce || cfg.HandleScrape { + jwks, err = keyfunc.Get(cfg.JWKSetURL, keyfunc.Options{ + Ctx: context.Background(), + RefreshErrorHandler: func(err error) { + logger.Error().Err(err).Msg("error occurred while updating JWKs") + }, + RefreshInterval: cfg.JWKUpdateInterval, + RefreshUnknownKID: true, + }) + } else { + logger.Warn().Msg("both announce and scrape handle disabled") + } if err == nil { h = &hook{ cfg: cfg, @@ -111,73 +123,171 @@ func (h *hook) Stop() stop.Result { return c.Result() } +type compatibleClaims interface { + Valid() error + ToRegisteredClaims() jwt.RegisteredClaims +} + +type announceClaims struct { + jwt.RegisteredClaims + InfoHash string `json:"infohash,omitempty"` +} + +func (ac announceClaims) ToRegisteredClaims() jwt.RegisteredClaims { + return ac.RegisteredClaims +} + func (h *hook) HandleAnnounce(ctx context.Context, req *bittorrent.AnnounceRequest, _ *bittorrent.AnnounceResponse) (context.Context, error) { - if req.Params == nil { - return ctx, ErrMissingJWT + if !h.cfg.HandleAnnounce { + return ctx, nil + } + var err error + + if jwtParam := h.getJWT(req.Params); len(jwtParam) == 0 { + err = ErrMissingJWT + } else { + claims := new(announceClaims) + if errs := h.validateBaseJWT(jwtParam, claims); len(errs) > 0 { + logger.Info(). + Errs("errors", errs). + Object("source", req.RequestPeer). + Msg("JWT validation failed") + err = ErrInvalidJWT + } else { + var claimIH bittorrent.InfoHash + if claimIH, err = bittorrent.NewInfoHash(claims.InfoHash); err != nil { + logger.Info(). + Err(err). + Object("source", req.RequestPeer). + Msg("InfoHash claim parse failed") + err = ErrInvalidJWT + } + if req.InfoHash != claimIH { + logger.Info(). + Stringer("provided", claimIH). + Stringer("required", req.InfoHash). + Object("source", req.RequestPeer). + Msg("InfoHash claim not equals to request InfoHash") + err = ErrInvalidJWT + } + } } - jwtParam, ok := req.Params.String("jwt") - if !ok { - return ctx, ErrMissingJWT - } - - if errs := h.validateJWT(req.InfoHash, jwtParam); len(errs) > 0 { - logger.Info(). - Errs("errors", errs). - Object("source", req.RequestPeer). - Msg("JWT validation failed") - return ctx, ErrInvalidJWT - } - - return ctx, nil + return ctx, err } -func (h *hook) HandleScrape(ctx context.Context, _ *bittorrent.ScrapeRequest, _ *bittorrent.ScrapeResponse) (context.Context, error) { - // Scrapes don't require any protection. - return ctx, nil +type scrapeClaims struct { + jwt.RegisteredClaims + InfoHashes []string `json:"infohashes,omitempty"` } -func (h *hook) validateJWT(ih bittorrent.InfoHash, rawJwt string) []error { - // KeyFunc will check KID, Parse will check ALG and signature - errs := make([]error, 0, 4) - claims := new(claims) - token, err := jwt.ParseWithClaims(rawJwt, claims, h.jwks.Keyfunc, hmacAlgorithms) - if err != nil { - return []error{err} +func (sc scrapeClaims) ToRegisteredClaims() jwt.RegisteredClaims { + return sc.RegisteredClaims +} + +func (h *hook) HandleScrape(ctx context.Context, req *bittorrent.ScrapeRequest, _ *bittorrent.ScrapeResponse) (context.Context, error) { + if !h.cfg.HandleScrape { + return ctx, nil } - if err = token.Claims.Valid(); err != nil { + var err error + + if jwtParam := h.getJWT(req.Params); len(jwtParam) == 0 { + err = ErrMissingJWT + } else { + claims := new(scrapeClaims) + if errs := h.validateBaseJWT(jwtParam, claims); len(errs) > 0 { + logger.Info(). + Errs("errors", errs). + Array("source", req.RequestAddresses). + Msg("JWT validation failed") + err = ErrInvalidJWT + } else { + var claimIHs bittorrent.InfoHashes + for _, s := range claims.InfoHashes { + if providedIh, err := bittorrent.NewInfoHash(s); err == nil { + claimIHs = append(claimIHs, providedIh) + } else { + logger.Info(). + Err(err). + Array("source", req.RequestAddresses). + Msg("InfoHash claim parse failed") + } + } + eq := len(req.InfoHashes) == len(claimIHs) + if eq { + for _, rIH := range req.InfoHashes { + found := false + for _, cIH := range claimIHs { + if rIH == cIH { + found = true + break + } + } + if !found { + eq = false + break + } + } + } + if !eq { + logger.Info(). + Array("provided", claimIHs). + Array("required", req.InfoHashes). + Array("source", req.RequestAddresses). + Msg("InfoHashes claim not equals to request InfoHashes") + err = ErrInvalidJWT + } + } + } + + return ctx, err +} + +func (h *hook) getJWT(params bittorrent.Params) (jwt string) { + if params != nil { + var found bool + if jwt, found = params.String(h.cfg.Header); found { + if strings.HasPrefix(strings.ToLower(jwt), bearerAuthPrefix) { + jwt = jwt[len(bearerAuthPrefix):] + } + } + } + return +} + +func (h *hook) validateBaseJWT(jwtParam string, claims compatibleClaims) (errs []error) { + if strings.HasPrefix(strings.ToLower(jwtParam), bearerAuthPrefix) { + jwtParam = jwtParam[len(bearerAuthPrefix):] + } + if _, err := jwt.ParseWithClaims(jwtParam, claims, h.jwks.Keyfunc, hmacAlgorithms); err != nil { errs = append(errs, err) } + if err := claims.Valid(); err != nil { + errs = append(errs, err) + } + if errs0 := h.validateRegisteredClaims(claims); len(errs0) > 0 { + errs = append(errs, errs0...) + } + return +} - if !claims.VerifyIssuer(h.cfg.Issuer, true) { +func (h *hook) validateRegisteredClaims(cl compatibleClaims) (errs []error) { + rc := cl.ToRegisteredClaims() + if !rc.VerifyIssuer(h.cfg.Issuer, true) { logger.Debug(). - Str("provided", claims.Issuer). + Str("provided", rc.Issuer). Str("required", h.cfg.Issuer). Msg("unequal or missing issuer when validating JWT") errs = append(errs, jwt.ErrTokenInvalidIssuer) } - if !claims.VerifyAudience(h.cfg.Audience, true) { + if !rc.VerifyAudience(h.cfg.Audience, true) { logger.Debug(). - Strs("provided", claims.Audience). + Strs("provided", rc.Audience). Str("required", h.cfg.Audience). Msg("unequal or missing audience when validating JWT") errs = append(errs, jwt.ErrTokenInvalidAudience) } - - providedIh, err := bittorrent.NewInfoHash(claims.InfoHash) - if err != nil { - errs = append(errs, err) - } - if providedIh != ih { - logger.Debug(). - Err(err). - Stringer("provided", providedIh). - Stringer("required", ih). - Msg("invalid or unequal info hash when validating JWT") - errs = append(errs, errInvalidInfoHashClaim) - } - - return errs + return } diff --git a/middleware/jwt/jwt_test.go b/middleware/jwt/jwt_test.go index 4e42130..046b83b 100644 --- a/middleware/jwt/jwt_test.go +++ b/middleware/jwt/jwt_test.go @@ -101,7 +101,7 @@ func TestHook_HandleAnnounceValid(t *testing.T) { })) defer s.Close() - token := jwt.NewWithClaims(jwt.SigningMethodES256, claims{ + token := jwt.NewWithClaims(jwt.SigningMethodES256, announceClaims{ RegisteredClaims: jwt.RegisteredClaims{ Issuer: "CN=test", Subject: "CN=test", @@ -118,6 +118,7 @@ func TestHook_HandleAnnounceValid(t *testing.T) { require.Nil(t, err) //goland:noinspection HttpUrlsUsage cfg := conf.MapConfig{ + "handle_announce": true, "issuer": "CN=test", "audience": "test", "jwk_set_url": "http://" + s.Listener.Addr().String(), @@ -126,7 +127,7 @@ func TestHook_HandleAnnounceValid(t *testing.T) { h, err := build(cfg, nil) require.Nil(t, err) data := make(map[string]string) - data["jwt"] = tokenString + data[authorizationHeader] = bearerAuthPrefix + tokenString _, err = h.HandleAnnounce(context.Background(), &bittorrent.AnnounceRequest{ InfoHash: infoHash, RequestPeer: bittorrent.RequestPeer{}, @@ -143,7 +144,7 @@ func TestHook_HandleAnnounceInvalid(t *testing.T) { // now we wll use HMAC-SHA256 with invalid random key // all errors should be nil except announce request - token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims{ + token := jwt.NewWithClaims(jwt.SigningMethodHS256, announceClaims{ RegisteredClaims: jwt.RegisteredClaims{ Issuer: "CN=test", Subject: "CN=test", @@ -162,6 +163,8 @@ func TestHook_HandleAnnounceInvalid(t *testing.T) { require.Nil(t, err) //goland:noinspection HttpUrlsUsage cfg := conf.MapConfig{ + "handle_announce": true, + "header": "jwt", "issuer": "CN=test", "audience": "test", "jwk_set_url": "http://" + s.Listener.Addr().String(), @@ -178,3 +181,44 @@ func TestHook_HandleAnnounceInvalid(t *testing.T) { }, nil) require.NotNil(t, err) } + +func TestHook_HandleScrapeValid(t *testing.T) { + s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _ = json.NewEncoder(w).Encode(jwksData) + })) + defer s.Close() + + token := jwt.NewWithClaims(jwt.SigningMethodES256, scrapeClaims{ + RegisteredClaims: jwt.RegisteredClaims{ + Issuer: "CN=test", + Subject: "CN=test", + Audience: []string{"test"}, + ExpiresAt: &jwt.NumericDate{Time: time.Now().Add(time.Hour)}, + NotBefore: &jwt.NumericDate{Time: time.Now().Add(-time.Hour)}, + ID: strconv.FormatInt(rand.Int63(), 16), + }, + InfoHashes: []string{infoHash.String()}, + }) + + token.Header["kid"] = jwksData.Keys[0].KeyID + tokenString, err := token.SignedString(privKey) + require.Nil(t, err) + //goland:noinspection HttpUrlsUsage + cfg := conf.MapConfig{ + "handle_scrape": true, + "issuer": "CN=test", + "audience": "test", + "jwk_set_url": "http://" + s.Listener.Addr().String(), + "jwk_set_update_interval": time.Minute, + } + h, err := build(cfg, nil) + require.Nil(t, err) + data := make(map[string]string) + data[authorizationHeader] = bearerAuthPrefix + tokenString + _, err = h.HandleScrape(context.Background(), &bittorrent.ScrapeRequest{ + InfoHashes: bittorrent.InfoHashes{infoHash}, + RequestAddresses: bittorrent.RequestAddresses{}, + Params: ¶ms{data: data}, + }, nil) + require.Nil(t, err) +} diff --git a/middleware/pkg/random/entropy.go b/middleware/pkg/random/entropy.go deleted file mode 100644 index 6fd01bf..0000000 --- a/middleware/pkg/random/entropy.go +++ /dev/null @@ -1,19 +0,0 @@ -package random - -import ( - "encoding/binary" - - "github.com/sot-tech/mochi/bittorrent" -) - -// DeriveEntropyFromRequest generates 2*64 bits of pseudo random state from an -// AnnounceRequest. -// -// Calling DeriveEntropyFromRequest multiple times yields the same values. -func DeriveEntropyFromRequest(req *bittorrent.AnnounceRequest) (v0 uint64, v1 uint64) { - if len(req.InfoHash) >= bittorrent.InfoHashV1Len { - v0 = binary.BigEndian.Uint64([]byte(req.InfoHash[:8])) + binary.BigEndian.Uint64([]byte(req.InfoHash[8:16])) - } - v1 = binary.BigEndian.Uint64(req.ID[:8]) + binary.BigEndian.Uint64(req.ID[8:16]) - return -} diff --git a/middleware/pkg/random/xorshift.go b/middleware/pkg/random/xorshift.go deleted file mode 100644 index 78d5f03..0000000 --- a/middleware/pkg/random/xorshift.go +++ /dev/null @@ -1,28 +0,0 @@ -// Package random implements the XORShift PRNG and a way to derive random state -// from an AnnounceRequest. -package random - -// GenerateAndAdvance applies XORShift128Plus on s0 and s1, returning -// the new states newS0, newS1 and a pseudo-random number v. -func GenerateAndAdvance(s0, s1 uint64) (v, newS0, newS1 uint64) { - v = s0 + s1 - newS0 = s1 - s0 ^= s0 << 23 - newS1 = s0 ^ s1 ^ (s0 >> 18) ^ (s1 >> 5) - return -} - -// Intn generates an int k that satisfies k >= 0 && k < n. -// n must be > 0. -// It returns the generated k and the new state of the generator. -func Intn(s0, s1 uint64, n int) (int, uint64, uint64) { - if n <= 0 { - panic("invalid n <= 0") - } - v, newS0, newS1 := GenerateAndAdvance(s0, s1) - k := int(v) - if k < 0 { - k = -k - } - return k % n, newS0, newS1 -} diff --git a/middleware/pkg/random/xorshift_test.go b/middleware/pkg/random/xorshift_test.go deleted file mode 100644 index 6df98b1..0000000 --- a/middleware/pkg/random/xorshift_test.go +++ /dev/null @@ -1,38 +0,0 @@ -package random - -import ( - "math/rand" - "testing" - - "github.com/stretchr/testify/require" - - _ "github.com/sot-tech/mochi/pkg/randseed" -) - -func TestIntn(t *testing.T) { - s0, s1 := rand.Uint64(), rand.Uint64() - var k int - for i := 0; i < 10000; i++ { - k, s0, s1 = Intn(s0, s1, 10) - require.True(t, k >= 0, "Intn() must be >= 0") - require.True(t, k < 10, "Intn(k) must be < k") - } -} - -func BenchmarkAdvanceXORShift128Plus(b *testing.B) { - s0, s1 := rand.Uint64(), rand.Uint64() - var v uint64 - for i := 0; i < b.N; i++ { - v, s0, s1 = GenerateAndAdvance(s0, s1) - } - _, _, _ = v, s0, s1 -} - -func BenchmarkIntn(b *testing.B) { - s0, s1 := rand.Uint64(), rand.Uint64() - var v int - for i := 0; i < b.N; i++ { - v, s0, s1 = Intn(s0, s1, 1000) - } - _, _, _ = v, s0, s1 -} diff --git a/middleware/varinterval/varinterval.go b/middleware/varinterval/varinterval.go index f9540ac..c677c76 100644 --- a/middleware/varinterval/varinterval.go +++ b/middleware/varinterval/varinterval.go @@ -3,14 +3,15 @@ package varinterval import ( "context" + "encoding/binary" "errors" "fmt" + "math" "sync" "time" "github.com/sot-tech/mochi/bittorrent" "github.com/sot-tech/mochi/middleware" - "github.com/sot-tech/mochi/middleware/pkg/random" "github.com/sot-tech/mochi/pkg/conf" "github.com/sot-tech/mochi/storage" ) @@ -79,14 +80,12 @@ type hook struct { } func (h *hook) HandleAnnounce(ctx context.Context, req *bittorrent.AnnounceRequest, resp *bittorrent.AnnounceResponse) (context.Context, error) { - s0, s1 := random.DeriveEntropyFromRequest(req) // Generate a probability p < 1.0. - v, s0, s1 := random.Intn(s0, s1, 1<<24) - p := float32(v) / (1 << 24) - if h.cfg.ModifyResponseProbability == 1 || p < h.cfg.ModifyResponseProbability { + p, s0, s1 := xoroshiro128p(deriveEntropyFromRequest(req)) + if float32(float64(p)/math.MaxUint64) < h.cfg.ModifyResponseProbability { // Generate the increase delta. - v, _, _ = random.Intn(s0, s1, h.cfg.MaxIncreaseDelta) - add := time.Duration(v+1) * time.Second + v, _, _ := xoroshiro128p(s0, s1) + add := time.Duration(v%uint64(h.cfg.MaxIncreaseDelta)+1) * time.Second resp.Interval += add @@ -102,3 +101,26 @@ func (h *hook) HandleScrape(ctx context.Context, _ *bittorrent.ScrapeRequest, _ // Scrapes are not altered. return ctx, nil } + +// deriveEntropyFromRequest generates 2*64 bits of pseudo random state from an +// bittorrent.AnnounceRequest. +// +// Calling deriveEntropyFromRequest multiple times yields the same values. +func deriveEntropyFromRequest(req *bittorrent.AnnounceRequest) (v0 uint64, v1 uint64) { + if len(req.InfoHash) >= bittorrent.InfoHashV1Len { + v0 = binary.BigEndian.Uint64([]byte(req.InfoHash[:8])) + binary.BigEndian.Uint64([]byte(req.InfoHash[8:16])) + } + v1 = binary.BigEndian.Uint64(req.ID[:8]) + binary.BigEndian.Uint64(req.ID[8:16]) + return +} + +// xoroshiro128p calculates predictable pseudorandom number +// with XOR/rotate/shift/rotate 128+ algorithm. +// see https://prng.di.unimi.it/xoroshiro128plus.c +func xoroshiro128p(s0, s1 uint64) (result, ns0, ns1 uint64) { + result = s0 + s1 + s1 ^= s0 + ns0 = ((s0 << 24) | (s0 >> 40)) ^ s1 ^ (s1 << 16) // rotl(s0, 24) ^ s1 ^ (s1 << 16) + ns1 = (s1 << 37) | (s1 >> 27) // rotl(s1, 37) + return +} diff --git a/middleware/varinterval/varinterval_test.go b/middleware/varinterval/varinterval_test.go index f8c298e..8ff8ee6 100644 --- a/middleware/varinterval/varinterval_test.go +++ b/middleware/varinterval/varinterval_test.go @@ -3,12 +3,15 @@ package varinterval import ( "context" "fmt" + "math/rand" "testing" "github.com/stretchr/testify/require" "github.com/sot-tech/mochi/bittorrent" "github.com/sot-tech/mochi/pkg/conf" + + _ "github.com/sot-tech/mochi/pkg/randseed" ) var configTests = []struct { @@ -61,3 +64,13 @@ func TestHandleAnnounce(t *testing.T) { require.True(t, resp.Interval > 0, "interval should have been increased") require.True(t, resp.MinInterval > 0, "min_interval should have been increased") } + +func BenchmarkXORoShiRo128Plus(b *testing.B) { + s0, s1 := rand.Uint64(), rand.Uint64() + var v uint64 + b.ResetTimer() + for i := 0; i < b.N; i++ { + v, s0, s1 = xoroshiro128p(s0, s1) + } + _, _, _ = v, s0, s1 +} From d7777ca583e20a77ac87d330dac8c9b3efaae3a9 Mon Sep 17 00:00:00 2001 From: "Lawrence, Rendall" Date: Fri, 26 Aug 2022 11:13:18 +0300 Subject: [PATCH 4/7] (minor) refactor `jwt` --- .golangci.yaml | 5 ++- middleware/jwt/jwt.go | 79 ++++++++++++++++++------------------ middleware/jwt/jwt_test.go | 83 +++++++++++++++++++++----------------- 3 files changed, 90 insertions(+), 77 deletions(-) diff --git a/.golangci.yaml b/.golangci.yaml index 31cbeb7..b315bdb 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -1,5 +1,8 @@ --- run: + # mochi in not written with generics (a.t.m), + # so we can check with 1.17 + go: "1.17" timeout: "5m" output: sort-results: true @@ -51,4 +54,4 @@ issues: - "EXC0012" # Exported should have comment - "EXC0013" # Package comment should be of form - "EXC0014" # Comment on exported should be of form - - "EXC0015" # Should have a package comment \ No newline at end of file + - "EXC0015" # Should have a package comment diff --git a/middleware/jwt/jwt.go b/middleware/jwt/jwt.go index 8e700a2..82aae1b 100644 --- a/middleware/jwt/jwt.go +++ b/middleware/jwt/jwt.go @@ -123,27 +123,38 @@ func (h *hook) Stop() stop.Result { return c.Result() } -type compatibleClaims interface { - Valid() error - ToRegisteredClaims() jwt.RegisteredClaims +type verifiableClaims interface { + jwt.Claims + VerifyIssuer(iss string, req bool) bool + GetIssuer() string + VerifyAudience(aud string, req bool) bool + GetAudience() []string +} + +type registeredClaimsWrapper struct { + jwt.RegisteredClaims +} + +func (rc registeredClaimsWrapper) GetIssuer() string { + return rc.Issuer +} + +func (rc registeredClaimsWrapper) GetAudience() []string { + return rc.Audience } type announceClaims struct { - jwt.RegisteredClaims + registeredClaimsWrapper InfoHash string `json:"infohash,omitempty"` } -func (ac announceClaims) ToRegisteredClaims() jwt.RegisteredClaims { - return ac.RegisteredClaims -} - func (h *hook) HandleAnnounce(ctx context.Context, req *bittorrent.AnnounceRequest, _ *bittorrent.AnnounceResponse) (context.Context, error) { if !h.cfg.HandleAnnounce { return ctx, nil } var err error - if jwtParam := h.getJWT(req.Params); len(jwtParam) == 0 { + if jwtParam := h.getJWTString(req.Params); len(jwtParam) == 0 { err = ErrMissingJWT } else { claims := new(announceClaims) @@ -159,15 +170,15 @@ func (h *hook) HandleAnnounce(ctx context.Context, req *bittorrent.AnnounceReque logger.Info(). Err(err). Object("source", req.RequestPeer). - Msg("InfoHash claim parse failed") + Msg("'infohash' claim parse failed") err = ErrInvalidJWT } if req.InfoHash != claimIH { logger.Info(). - Stringer("provided", claimIH). - Stringer("required", req.InfoHash). - Object("source", req.RequestPeer). - Msg("InfoHash claim not equals to request InfoHash") + Stringer("claimInfoHash", claimIH). + Stringer("requestInfoHash", req.InfoHash). + Object("peer", req.RequestPeer). + Msg("unequal 'infohash' claim when validating JWT") err = ErrInvalidJWT } } @@ -177,14 +188,10 @@ func (h *hook) HandleAnnounce(ctx context.Context, req *bittorrent.AnnounceReque } type scrapeClaims struct { - jwt.RegisteredClaims + registeredClaimsWrapper InfoHashes []string `json:"infohashes,omitempty"` } -func (sc scrapeClaims) ToRegisteredClaims() jwt.RegisteredClaims { - return sc.RegisteredClaims -} - func (h *hook) HandleScrape(ctx context.Context, req *bittorrent.ScrapeRequest, _ *bittorrent.ScrapeResponse) (context.Context, error) { if !h.cfg.HandleScrape { return ctx, nil @@ -192,7 +199,7 @@ func (h *hook) HandleScrape(ctx context.Context, req *bittorrent.ScrapeRequest, var err error - if jwtParam := h.getJWT(req.Params); len(jwtParam) == 0 { + if jwtParam := h.getJWTString(req.Params); len(jwtParam) == 0 { err = ErrMissingJWT } else { claims := new(scrapeClaims) @@ -210,8 +217,8 @@ func (h *hook) HandleScrape(ctx context.Context, req *bittorrent.ScrapeRequest, } else { logger.Info(). Err(err). - Array("source", req.RequestAddresses). - Msg("InfoHash claim parse failed") + Array("addresses", req.RequestAddresses). + Msg("'infohashes' claim parse failed") } } eq := len(req.InfoHashes) == len(claimIHs) @@ -232,10 +239,10 @@ func (h *hook) HandleScrape(ctx context.Context, req *bittorrent.ScrapeRequest, } if !eq { logger.Info(). - Array("provided", claimIHs). - Array("required", req.InfoHashes). - Array("source", req.RequestAddresses). - Msg("InfoHashes claim not equals to request InfoHashes") + Array("claimInfoHashes", claimIHs). + Array("requestInfoHashes", req.InfoHashes). + Array("addresses", req.RequestAddresses). + Msg("unequal 'infohashes' claim when validating JWT") err = ErrInvalidJWT } } @@ -244,7 +251,7 @@ func (h *hook) HandleScrape(ctx context.Context, req *bittorrent.ScrapeRequest, return ctx, err } -func (h *hook) getJWT(params bittorrent.Params) (jwt string) { +func (h *hook) getJWTString(params bittorrent.Params) (jwt string) { if params != nil { var found bool if jwt, found = params.String(h.cfg.Header); found { @@ -256,7 +263,7 @@ func (h *hook) getJWT(params bittorrent.Params) (jwt string) { return } -func (h *hook) validateBaseJWT(jwtParam string, claims compatibleClaims) (errs []error) { +func (h *hook) validateBaseJWT(jwtParam string, claims verifiableClaims) (errs []error) { if strings.HasPrefix(strings.ToLower(jwtParam), bearerAuthPrefix) { jwtParam = jwtParam[len(bearerAuthPrefix):] } @@ -266,25 +273,17 @@ func (h *hook) validateBaseJWT(jwtParam string, claims compatibleClaims) (errs [ if err := claims.Valid(); err != nil { errs = append(errs, err) } - if errs0 := h.validateRegisteredClaims(claims); len(errs0) > 0 { - errs = append(errs, errs0...) - } - return -} -func (h *hook) validateRegisteredClaims(cl compatibleClaims) (errs []error) { - rc := cl.ToRegisteredClaims() - if !rc.VerifyIssuer(h.cfg.Issuer, true) { + if !claims.VerifyIssuer(h.cfg.Issuer, true) { logger.Debug(). - Str("provided", rc.Issuer). + Str("provided", claims.GetIssuer()). Str("required", h.cfg.Issuer). Msg("unequal or missing issuer when validating JWT") errs = append(errs, jwt.ErrTokenInvalidIssuer) } - - if !rc.VerifyAudience(h.cfg.Audience, true) { + if !claims.VerifyAudience(h.cfg.Audience, true) { logger.Debug(). - Strs("provided", rc.Audience). + Strs("provided", claims.GetAudience()). Str("required", h.cfg.Audience). Msg("unequal or missing audience when validating JWT") errs = append(errs, jwt.ErrTokenInvalidAudience) diff --git a/middleware/jwt/jwt_test.go b/middleware/jwt/jwt_test.go index 046b83b..04a4983 100644 --- a/middleware/jwt/jwt_test.go +++ b/middleware/jwt/jwt_test.go @@ -53,12 +53,10 @@ type JWKSKeys struct { Keys []JWKSKey `json:"keys"` } -type params struct { - data map[string]string -} +type params map[string]string func (p params) String(key string) (out string, found bool) { - out, found = p.data[key] + out, found = p[key] return } @@ -70,9 +68,7 @@ func (params) RawQuery() (s string) { return } -func (params) MarshalZerologObject(*zerolog.Event) { - return -} +func (params) MarshalZerologObject(*zerolog.Event) {} func init() { _ = log.ConfigureLogger("", "info", false, false) @@ -102,13 +98,15 @@ func TestHook_HandleAnnounceValid(t *testing.T) { defer s.Close() token := jwt.NewWithClaims(jwt.SigningMethodES256, announceClaims{ - RegisteredClaims: jwt.RegisteredClaims{ - Issuer: "CN=test", - Subject: "CN=test", - Audience: []string{"test"}, - ExpiresAt: &jwt.NumericDate{Time: time.Now().Add(time.Hour)}, - NotBefore: &jwt.NumericDate{Time: time.Now().Add(-time.Hour)}, - ID: strconv.FormatInt(rand.Int63(), 16), + registeredClaimsWrapper: registeredClaimsWrapper{ + RegisteredClaims: jwt.RegisteredClaims{ + Issuer: "CN=test", + Subject: "CN=test", + Audience: []string{"test"}, + ExpiresAt: &jwt.NumericDate{Time: time.Now().Add(time.Hour)}, + NotBefore: &jwt.NumericDate{Time: time.Now().Add(-time.Hour)}, + ID: strconv.FormatInt(rand.Int63(), 16), + }, }, InfoHash: infoHash.String(), }) @@ -126,12 +124,12 @@ func TestHook_HandleAnnounceValid(t *testing.T) { } h, err := build(cfg, nil) require.Nil(t, err) - data := make(map[string]string) + data := make(params) data[authorizationHeader] = bearerAuthPrefix + tokenString _, err = h.HandleAnnounce(context.Background(), &bittorrent.AnnounceRequest{ InfoHash: infoHash, RequestPeer: bittorrent.RequestPeer{}, - Params: ¶ms{data: data}, + Params: data, }, nil) require.Nil(t, err) } @@ -145,13 +143,15 @@ func TestHook_HandleAnnounceInvalid(t *testing.T) { // now we wll use HMAC-SHA256 with invalid random key // all errors should be nil except announce request token := jwt.NewWithClaims(jwt.SigningMethodHS256, announceClaims{ - RegisteredClaims: jwt.RegisteredClaims{ - Issuer: "CN=test", - Subject: "CN=test", - Audience: []string{"test"}, - ExpiresAt: &jwt.NumericDate{Time: time.Now().Add(time.Hour)}, - NotBefore: &jwt.NumericDate{Time: time.Now().Add(-time.Hour)}, - ID: strconv.FormatInt(rand.Int63(), 16), + registeredClaimsWrapper: registeredClaimsWrapper{ + RegisteredClaims: jwt.RegisteredClaims{ + Issuer: "CN=test", + Subject: "CN=test", + Audience: []string{"test"}, + ExpiresAt: &jwt.NumericDate{Time: time.Now().Add(time.Hour)}, + NotBefore: &jwt.NumericDate{Time: time.Now().Add(-time.Hour)}, + ID: strconv.FormatInt(rand.Int63(), 16), + }, }, InfoHash: infoHash.String(), }) @@ -172,12 +172,12 @@ func TestHook_HandleAnnounceInvalid(t *testing.T) { } h, err := build(cfg, nil) require.Nil(t, err) - data := make(map[string]string) + data := make(params) data["jwt"] = tokenString _, err = h.HandleAnnounce(context.Background(), &bittorrent.AnnounceRequest{ InfoHash: infoHash, RequestPeer: bittorrent.RequestPeer{}, - Params: ¶ms{data: data}, + Params: data, }, nil) require.NotNil(t, err) } @@ -188,16 +188,27 @@ func TestHook_HandleScrapeValid(t *testing.T) { })) defer s.Close() + ihs := make(bittorrent.InfoHashes, rand.Intn(10)+1) + ihss := make([]string, len(ihs)) + for i := range ihs { + bb := []byte(infoHash) + bb[i] = byte(i) + ihs[i] = bittorrent.InfoHash(bb) + ihss[i] = ihs[i].String() + } + token := jwt.NewWithClaims(jwt.SigningMethodES256, scrapeClaims{ - RegisteredClaims: jwt.RegisteredClaims{ - Issuer: "CN=test", - Subject: "CN=test", - Audience: []string{"test"}, - ExpiresAt: &jwt.NumericDate{Time: time.Now().Add(time.Hour)}, - NotBefore: &jwt.NumericDate{Time: time.Now().Add(-time.Hour)}, - ID: strconv.FormatInt(rand.Int63(), 16), + registeredClaimsWrapper: registeredClaimsWrapper{ + RegisteredClaims: jwt.RegisteredClaims{ + Issuer: "CN=test", + Subject: "CN=test", + Audience: []string{"test"}, + ExpiresAt: &jwt.NumericDate{Time: time.Now().Add(time.Hour)}, + NotBefore: &jwt.NumericDate{Time: time.Now().Add(-time.Hour)}, + ID: strconv.FormatInt(rand.Int63(), 16), + }, }, - InfoHashes: []string{infoHash.String()}, + InfoHashes: ihss, }) token.Header["kid"] = jwksData.Keys[0].KeyID @@ -213,12 +224,12 @@ func TestHook_HandleScrapeValid(t *testing.T) { } h, err := build(cfg, nil) require.Nil(t, err) - data := make(map[string]string) + data := make(params) data[authorizationHeader] = bearerAuthPrefix + tokenString _, err = h.HandleScrape(context.Background(), &bittorrent.ScrapeRequest{ - InfoHashes: bittorrent.InfoHashes{infoHash}, + InfoHashes: ihs, RequestAddresses: bittorrent.RequestAddresses{}, - Params: ¶ms{data: data}, + Params: data, }, nil) require.Nil(t, err) } From 170639eb3e4700021c5495233adc35e015acfcf6 Mon Sep 17 00:00:00 2001 From: "Lawrence, Rendall" Date: Fri, 26 Aug 2022 11:58:22 +0300 Subject: [PATCH 5/7] (minor) remove double bearer trim --- middleware/jwt/jwt.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/middleware/jwt/jwt.go b/middleware/jwt/jwt.go index 82aae1b..fe57633 100644 --- a/middleware/jwt/jwt.go +++ b/middleware/jwt/jwt.go @@ -42,7 +42,7 @@ var ( // ErrInvalidJWT is returned when a JWT fails to verify. ErrInvalidJWT = bittorrent.ClientError("unapproved request: invalid jwt") - errJWKsNotSet = errors.New("required parameters not provided: Issuer, Audience and/or JWKSetURL") + errJWKsNotSet = errors.New("required parameters not provided: Issuer/Audience/JWKSetURL") hmacAlgorithms = jwt.WithValidMethods([]string{ jwt.SigningMethodHS256.Alg(), jwt.SigningMethodHS384.Alg(), jwt.SigningMethodHS512.Alg(), @@ -264,9 +264,6 @@ func (h *hook) getJWTString(params bittorrent.Params) (jwt string) { } func (h *hook) validateBaseJWT(jwtParam string, claims verifiableClaims) (errs []error) { - if strings.HasPrefix(strings.ToLower(jwtParam), bearerAuthPrefix) { - jwtParam = jwtParam[len(bearerAuthPrefix):] - } if _, err := jwt.ParseWithClaims(jwtParam, claims, h.jwks.Keyfunc, hmacAlgorithms); err != nil { errs = append(errs, err) } From 694592f881c51893329ad53080a3c5130def01a2 Mon Sep 17 00:00:00 2001 From: "Lawrence, Rendall" Date: Tue, 6 Sep 2022 13:34:57 +0300 Subject: [PATCH 6/7] fix list warnings * remove deprecated lint checks * reformat files * update dependencies --- .golangci.yaml | 4 -- cmd/mochi-e2e/e2e.go | 2 + cmd/mochi/main.go | 1 + dist/example_config.yaml | 80 ++++++++++++++++++------------------ docs/storage/postgres.md | 2 +- go.mod | 16 ++++---- go.sum | 35 ++++++++-------- pkg/log/log.go | 6 ++- storage/pg/storage.go | 3 ++ storage/redis/storage.go | 88 ++++++++++++++++++++-------------------- storage/storage.go | 4 +- 11 files changed, 123 insertions(+), 118 deletions(-) diff --git a/.golangci.yaml b/.golangci.yaml index b315bdb..7eb8775 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -20,7 +20,6 @@ linters: enable: - "bidichk" - "bodyclose" - - "deadcode" - "errcheck" - "errname" - "errorlint" @@ -30,7 +29,6 @@ linters: - "gosec" - "gosimple" - "govet" - - "ifshort" - "importas" - "ineffassign" - "makezero" @@ -39,13 +37,11 @@ linters: - "revive" - "rowserrcheck" - "staticcheck" - - "structcheck" - "stylecheck" - "tenv" - "typecheck" - "unconvert" - "unused" - - "varcheck" - "wastedassign" - "whitespace" issues: diff --git a/cmd/mochi-e2e/e2e.go b/cmd/mochi-e2e/e2e.go index 7abc45a..0d68274 100644 --- a/cmd/mochi-e2e/e2e.go +++ b/cmd/mochi-e2e/e2e.go @@ -1,3 +1,5 @@ +// Package main contains End-to-End MoChi check implementation. +// not used in production package main import ( diff --git a/cmd/mochi/main.go b/cmd/mochi/main.go index 12399dd..6b26686 100644 --- a/cmd/mochi/main.go +++ b/cmd/mochi/main.go @@ -1,3 +1,4 @@ +// Package main contains entry point logic of MoChi server package main import ( diff --git a/dist/example_config.yaml b/dist/example_config.yaml index 250bfc5..692ab2a 100644 --- a/dist/example_config.yaml +++ b/dist/example_config.yaml @@ -154,58 +154,58 @@ mochi: # are collected and posted to Prometheus. prometheus_reporting_interval: 1s - # This block defines configuration used for redis storage. - #storage: - #name: redis - #config: - # The frequency which stale peers are removed. - # This balances between - # - collecting garbage more often, potentially using more CPU time, but potentially using less memory (lower value) - # - collecting garbage less frequently, saving CPU time, but keeping old peers long, thus using more memory (higher value). - #gc_interval: 3m + # This block defines configuration used for redis storage. + #storage: + #name: redis + #config: + # The frequency which stale peers are removed. + # This balances between + # - collecting garbage more often, potentially using more CPU time, but potentially using less memory (lower value) + # - collecting garbage less frequently, saving CPU time, but keeping old peers long, thus using more memory (higher value). + #gc_interval: 3m - # The interval at which metrics about the number of infohashes and peers - # are collected and posted to Prometheus. - #prometheus_reporting_interval: 1s + # The interval at which metrics about the number of infohashes and peers + # are collected and posted to Prometheus. + #prometheus_reporting_interval: 1s - # The amount of time until a peer is considered stale. - # To avoid churn, keep this slightly larger than `announce_interval` - #peer_lifetime: 31m + # The amount of time until a peer is considered stale. + # To avoid churn, keep this slightly larger than `announce_interval` + #peer_lifetime: 31m - # The addresses of redis storage. - # If neither sentinel not cluster switched, - # only first address used - #addresses: ["127.0.0.1:6379"] + # The addresses of redis storage. + # If neither sentinel not cluster switched, + # only first address used + #addresses: ["127.0.0.1:6379"] - # Database to be selected after connecting to the server. - #db: 0 + # Database to be selected after connecting to the server. + #db: 0 - # Maximum number of socket connections, default is 10 per CPU - #pool_size: 10 + # Maximum number of socket connections, default is 10 per CPU + #pool_size: 10 - # Use the specified login/username to authenticate the current connection - #login: "" + # Use the specified login/username to authenticate the current connection + #login: "" - # Optional password - #password: "" + # Optional password + #password: "" - # Connect to sentinel nodes - #sentinel: false + # Connect to sentinel nodes + #sentinel: false - # The master name - #sentinel_master: "" + # The master name + #sentinel_master: "" - # Connect to the redis cluster - #cluster: false + # Connect to the redis cluster + #cluster: false - # The timeout for reading a command reply from redis. - #read_timeout: 15s + # The timeout for reading a command reply from redis. + #read_timeout: 15s - # The timeout for writing a command to redis. - #write_timeout: 15s + # The timeout for writing a command to redis. + #write_timeout: 15s - # Dial timeout for establishing new connections. - #connect_timeout: 15s + # Dial timeout for establishing new connections. + #connect_timeout: 15s # This block defines configuration used for PostgreSQL storage. # example `mo_peers` table structure: @@ -242,7 +242,7 @@ mochi: #data: # expected parameters: 1 - context (varchar), 2 - name (bytea), 3 - value (bytea) #add_query: INSERT INTO mo_kv VALUES($1, $2, $3) ON CONFLICT (context, name) DO NOTHING - #del_query: DELETE FROM mo_kv WHERE context=$1 AND name=$2 + #del_query: DELETE FROM mo_kv WHERE context=$1 AND name=$2 #get_query: SELECT value FROM mo_kv WHERE context=$1 AND name=$2 # query for check if database is alive #ping_query: SELECT 1 diff --git a/docs/storage/postgres.md b/docs/storage/postgres.md index b52e219..143baec 100644 --- a/docs/storage/postgres.md +++ b/docs/storage/postgres.md @@ -47,7 +47,7 @@ Implementation expects next data types: * value - byte array (`bytea`) (*) in KV table `name` present as byte array because of possibility -to place hash as _raw_ string, which is not supported by PostgreSQL. +to place hash as _raw_ string, which is not supported by PostgreSQL. Sample script to create tables: diff --git a/go.mod b/go.mod index d86caa2..e058e28 100644 --- a/go.mod +++ b/go.mod @@ -3,24 +3,24 @@ module github.com/sot-tech/mochi go 1.18 require ( - code.cloudfoundry.org/go-diodes v0.0.0-20220725190411-383eb6634c40 + code.cloudfoundry.org/go-diodes v0.0.0-20220830191835-c1b067f33eca github.com/MicahParks/keyfunc v1.2.2 github.com/anacrolix/torrent v1.46.0 github.com/go-redis/redis/v8 v8.11.5 github.com/golang-jwt/jwt/v4 v4.4.2 - github.com/jackc/pgx/v4 v4.17.0 + github.com/jackc/pgx/v4 v4.17.2 github.com/julienschmidt/httprouter v1.3.0 github.com/libp2p/go-reuseport v0.2.0 github.com/minio/sha256-simd v1.0.0 github.com/mitchellh/mapstructure v1.5.0 github.com/prometheus/client_golang v1.13.0 - github.com/rs/zerolog v1.27.0 + github.com/rs/zerolog v1.28.0 github.com/stretchr/testify v1.8.0 gopkg.in/yaml.v3 v3.0.1 ) require ( - github.com/anacrolix/dht/v2 v2.18.0 // indirect + github.com/anacrolix/dht/v2 v2.18.1 // indirect github.com/anacrolix/log v0.13.2-0.20220426014722-7b7d13a55d55 // indirect github.com/anacrolix/missinggo v1.3.0 // indirect github.com/anacrolix/missinggo/v2 v2.7.0 // indirect @@ -39,8 +39,8 @@ require ( github.com/jackc/pgproto3/v2 v2.3.1 // indirect github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect github.com/jackc/pgtype v1.12.0 // indirect - github.com/jackc/puddle v1.2.1 // indirect - github.com/klauspost/cpuid/v2 v2.1.0 // indirect + github.com/jackc/puddle v1.3.0 // indirect + github.com/klauspost/cpuid/v2 v2.1.1 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.16 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect @@ -48,8 +48,8 @@ require ( github.com/prometheus/client_model v0.2.0 // indirect github.com/prometheus/common v0.37.0 // indirect github.com/prometheus/procfs v0.8.0 // indirect - golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa // indirect - golang.org/x/sys v0.0.0-20220817070843-5a390386f1f2 // indirect + golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90 // indirect + golang.org/x/sys v0.0.0-20220829200755-d48e67d00261 // indirect golang.org/x/text v0.3.7 // indirect google.golang.org/protobuf v1.28.1 // indirect ) diff --git a/go.sum b/go.sum index 5dc7073..4415f52 100644 --- a/go.sum +++ b/go.sum @@ -30,8 +30,8 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -code.cloudfoundry.org/go-diodes v0.0.0-20220725190411-383eb6634c40 h1:wzkYwwcf4uMGcDpn48WAbq8GtoqDny49tdQ4zJVAsmo= -code.cloudfoundry.org/go-diodes v0.0.0-20220725190411-383eb6634c40/go.mod h1:Nx9ASXN4nIlRDEXv+qXE3dpuhnTnO28Lxl/bMUd6BMc= +code.cloudfoundry.org/go-diodes v0.0.0-20220830191835-c1b067f33eca h1:snlCH49dbxeL78CFFgSWhqeQ8UY3mCp9/utxkOGhKPM= +code.cloudfoundry.org/go-diodes v0.0.0-20220830191835-c1b067f33eca/go.mod h1:02VWjyNigcAIscOkao4SSaMv9siy/gQMwsLHuoaKVIQ= crawshaw.io/iox v0.0.0-20181124134642-c51c3df30797/go.mod h1:sXBiorCo8c46JlQV3oXPKINnZ8mcqnye1EkVkqsectk= crawshaw.io/sqlite v0.3.2/go.mod h1:igAO5JulrQ1DbdZdtVq48mnZUBAPOeFzer7VhDWNtW4= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= @@ -50,8 +50,8 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= -github.com/anacrolix/dht/v2 v2.18.0 h1:btjVjzjKqO5nKGbJHJ2UmuwiRx+EgX3e+OCHC9+WRz8= -github.com/anacrolix/dht/v2 v2.18.0/go.mod h1:mxrSeP/LIY429SgWMO9o6UdjBjB8ZjBh6HHCmd8Ly1g= +github.com/anacrolix/dht/v2 v2.18.1 h1:0y/2HoD1VzhJ4o/XDtcTwKGDc5rotFmpV+bEC48ZAd8= +github.com/anacrolix/dht/v2 v2.18.1/go.mod h1:mxrSeP/LIY429SgWMO9o6UdjBjB8ZjBh6HHCmd8Ly1g= github.com/anacrolix/envpprof v0.0.0-20180404065416-323002cec2fa/go.mod h1:KgHhUaQMc8cC0+cEflSgCFNFbKwi5h54gqtVn8yhP7c= github.com/anacrolix/envpprof v1.0.0/go.mod h1:KgHhUaQMc8cC0+cEflSgCFNFbKwi5h54gqtVn8yhP7c= github.com/anacrolix/envpprof v1.1.0/go.mod h1:My7T5oSqVfEn4MD4Meczkw/f5lSIndGAKu/0SM/rkf4= @@ -261,13 +261,13 @@ github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08 github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs= -github.com/jackc/pgx/v4 v4.17.0 h1:Hsx+baY8/zU2WtPLQyZi8WbecgcsWEeyoK1jvg/WgIo= -github.com/jackc/pgx/v4 v4.17.0/go.mod h1:Gd6RmOhtFLTu8cp/Fhq4kP195KrshxYJH3oW8AWJ1pw= +github.com/jackc/pgx/v4 v4.17.2 h1:0Ut0rpeKwvIVbMQ1KbMBU4h6wxehBI535LK6Flheh8E= +github.com/jackc/pgx/v4 v4.17.2/go.mod h1:lcxIZN44yMIrWI78a5CpucdD14hX0SBDbNRvjDBItsw= github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= -github.com/jackc/puddle v1.2.1 h1:gI8os0wpRXFd4FiAY2dWiqRK037tjj3t7rKFeO4X5iw= -github.com/jackc/puddle v1.2.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.3.0 h1:eHK/5clGOatcjX3oWGBO/MpxpbHzSwud5EWTSCI+MX0= +github.com/jackc/puddle v1.3.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -283,8 +283,8 @@ github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4d github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.1.0 h1:eyi1Ad2aNJMW95zcSbmGg7Cg6cq3ADwLpMAP96d8rF0= -github.com/klauspost/cpuid/v2 v2.1.0/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= +github.com/klauspost/cpuid/v2 v2.1.1 h1:t0wUqjowdm8ezddV5k0tLWVklVuvLJpoHeb4WBdydm0= +github.com/klauspost/cpuid/v2 v2.1.1/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -335,7 +335,7 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.20.0 h1:8W0cWlwFkflGPLltQvLRB7ZVD5HuP6ng320w2IS245Q= +github.com/onsi/gomega v1.20.1 h1:PA/3qinGoukvymdIDV8pii6tiZgC8kbmJO6Z5+b002Q= github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= @@ -385,11 +385,11 @@ github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTE github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8= github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= -github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= -github.com/rs/zerolog v1.27.0 h1:1T7qCieN22GVc8S4Q2yuexzBb1EqjbgjSH9RohbMjKs= -github.com/rs/zerolog v1.27.0/go.mod h1:7frBqO0oezxmnO7GF86FY++uy8I0Tk/If5ni1G9Qc0U= +github.com/rs/zerolog v1.28.0 h1:MirSo27VyNi7RJYP3078AA1+Cyzd2GB66qy3aUHvsWY= +github.com/rs/zerolog v1.28.0/go.mod h1:NILgTygv/Uej1ra5XxGf82ZFSLk58MFGAUS2o6usyD0= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= @@ -454,8 +454,9 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa h1:zuSxTR4o9y82ebqCUJYNGJbGPo6sKVl54f/TVDObg1c= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90 h1:Y/gsMcFOcR+6S6f3YeMKl5g+dZMEWqcz5Czj/GWYbkM= +golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -590,8 +591,8 @@ golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220817070843-5a390386f1f2 h1:fqTvyMIIj+HRzMmnzr9NtpHP6uVpvB5fkHcgPDC4nu8= -golang.org/x/sys v0.0.0-20220817070843-5a390386f1f2/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220829200755-d48e67d00261 h1:v6hYoSR9T5oet+pMXwUWkbiVqx/63mlHjefrHmxwfeY= +golang.org/x/sys v0.0.0-20220829200755-d48e67d00261/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= diff --git a/pkg/log/log.go b/pkg/log/log.go index 798798c..798e07e 100644 --- a/pkg/log/log.go +++ b/pkg/log/log.go @@ -32,7 +32,8 @@ func init() { // ConfigureLogger initializes root and all child loggers. // NOTE: this function MUST be called before any child log call -// otherwise any goroutine, which uses logger will wait logger initialization +// +// otherwise any goroutine, which uses logger will wait logger initialization func ConfigureLogger(output, level string, formatted, colored bool) (err error) { lvl := zerolog.WarnLevel output = strings.ToLower(output) @@ -298,7 +299,8 @@ func Close() { // NewLogger creates child logger with specified component name // NOTE: root logger MUST be initialized with ConfigureLogger -// before any logger call +// +// before any logger call func NewLogger(component string) *Logger { return &Logger{comp: component} } diff --git a/storage/pg/storage.go b/storage/pg/storage.go index e1a889a..b0f91cb 100644 --- a/storage/pg/storage.go +++ b/storage/pg/storage.go @@ -1,3 +1,6 @@ +// Package pg implements PostgreSQL-like storage interface. +// This implementation does not use ORM and relies on database structure +// and queries provided in configuration. package pg import ( diff --git a/storage/redis/storage.go b/storage/redis/storage.go index 1cd231a..78d25a0 100644 --- a/storage/redis/storage.go +++ b/storage/redis/storage.go @@ -2,21 +2,21 @@ // BitTorrent tracker keeping peer data in redis with hash. // There two categories of hash: // -// - CHI_{L,S}{4,6}_ (hash type) -// To save peers that hold the infohash, used for fast searching, -// deleting, and timeout handling +// - CHI_{L,S}{4,6}_ (hash type) +// To save peers that hold the infohash, used for fast searching, +// deleting, and timeout handling // -// - CHI_I (set type) -// To save all the infohashes, used for garbage collection, -// metrics aggregation and leecher graduation +// - CHI_I (set type) +// To save all the infohashes, used for garbage collection, +// metrics aggregation and leecher graduation // // Two keys are used to record the count of seeders and leechers. // -// - CHI_C_S (key type) -// To record the number of seeders. +// - CHI_C_S (key type) +// To record the number of seeders. // -// - CHI_C_L (key type) -// To record the number of leechers. +// - CHI_C_L (key type) +// To record the number of leechers. package redis import ( @@ -644,45 +644,45 @@ func (ps *Connection) Ping() error { // This function must be able to execute while other methods on this interface // are being executed in parallel. // -// - The Delete(Seeder|Leecher) and GraduateLeecher methods never delete an -// infohash key from an addressFamily hash. They also never decrement the -// infohash counter. -// - The Put(Seeder|Leecher) and GraduateLeecher methods only ever add infohash -// keys to addressFamily hashes and increment the infohash counter. -// - The only method that deletes from the addressFamily hashes is -// gc, which also decrements the counters. That means that, -// even if a Delete(Seeder|Leecher) call removes the last peer from a swarm, -// the infohash counter is not changed and the infohash is left in the -// addressFamily hash until it will be cleaned up by gc. -// - gc must run regularly. -// - A WATCH ... MULTI ... EXEC block fails, if between the WATCH and the 'EXEC' -// any of the watched keys have changed. The location of the 'MULTI' doesn't -// matter. +// - The Delete(Seeder|Leecher) and GraduateLeecher methods never delete an +// infohash key from an addressFamily hash. They also never decrement the +// infohash counter. +// - The Put(Seeder|Leecher) and GraduateLeecher methods only ever add infohash +// keys to addressFamily hashes and increment the infohash counter. +// - The only method that deletes from the addressFamily hashes is +// gc, which also decrements the counters. That means that, +// even if a Delete(Seeder|Leecher) call removes the last peer from a swarm, +// the infohash counter is not changed and the infohash is left in the +// addressFamily hash until it will be cleaned up by gc. +// - gc must run regularly. +// - A WATCH ... MULTI ... EXEC block fails, if between the WATCH and the 'EXEC' +// any of the watched keys have changed. The location of the 'MULTI' doesn't +// matter. // // We have to analyze four cases to prove our algorithm works. I'll characterize // them by a tuple (number of peers in a swarm before WATCH, number of peers in // the swarm during the transaction). // -// 1. (0,0), the easy case: The swarm is empty, we watch the key, we execute -// HLEN and find it empty. We remove it and decrement the counter. It stays -// empty the entire time, the transaction goes through. -// 2. (1,n > 0): The swarm is not empty, we watch the key, we find it non-empty, -// we unwatch the key. All good. No transaction is made, no transaction fails. -// 3. (0,1): We have to analyze this in two ways. -// - If the change happens before the HLEN call, we will see that the swarm is -// not empty and start no transaction. -// - If the change happens after the HLEN, we will attempt a transaction and it -// will fail. This is okay, the swarm is not empty, we will try cleaning it up -// next time gc runs. -// 4. (1,0): Again, two ways: -// - If the change happens before the HLEN, we will see an empty swarm. This -// situation happens if a call to Delete(Seeder|Leecher) removed the last -// peer asynchronously. We will attempt a transaction, but the transaction -// will fail. This is okay, the infohash key will remain in the addressFamily -// hash, we will attempt to clean it up the next time 'gc` runs. -// - If the change happens after the HLEN, we will not even attempt to make the -// transaction. The infohash key will remain in the addressFamil hash and -// we'll attempt to clean it up the next time gc runs. +// 1. (0,0), the easy case: The swarm is empty, we watch the key, we execute +// HLEN and find it empty. We remove it and decrement the counter. It stays +// empty the entire time, the transaction goes through. +// 2. (1,n > 0): The swarm is not empty, we watch the key, we find it non-empty, +// we unwatch the key. All good. No transaction is made, no transaction fails. +// 3. (0,1): We have to analyze this in two ways. +// - If the change happens before the HLEN call, we will see that the swarm is +// not empty and start no transaction. +// - If the change happens after the HLEN, we will attempt a transaction and it +// will fail. This is okay, the swarm is not empty, we will try cleaning it up +// next time gc runs. +// 4. (1,0): Again, two ways: +// - If the change happens before the HLEN, we will see an empty swarm. This +// situation happens if a call to Delete(Seeder|Leecher) removed the last +// peer asynchronously. We will attempt a transaction, but the transaction +// will fail. This is okay, the infohash key will remain in the addressFamily +// hash, we will attempt to clean it up the next time 'gc` runs. +// - If the change happens after the HLEN, we will not even attempt to make the +// transaction. The infohash key will remain in the addressFamil hash and +// we'll attempt to clean it up the next time gc runs. func (ps *store) gc(cutoff time.Time) { cutoffNanos := cutoff.UnixNano() // list all infoHashKeys in the group diff --git a/storage/storage.go b/storage/storage.go index b8f46df..3fca68b 100644 --- a/storage/storage.go +++ b/storage/storage.go @@ -124,12 +124,12 @@ type DataStorage interface { // Implementations of the PeerStorage interface must do the following in addition // to implementing the methods of the interface in the way documented: // -// - Implement a garbage-collection strategy that ensures stale data is removed. +// - Implement a garbage-collection strategy that ensures stale data is removed. // For example, a timestamp on each InfoHash/Peer combination can be used // to track the last activity for that Peer. The entire database can then // be scanned periodically and too old Peers removed. The intervals and // durations involved should be configurable. -// - IPv4 and IPv6 swarms may be isolated from each other. +// - IPv4 and IPv6 swarms may be isolated from each other. // // Implementations can be tested against this interface using the tests in // storage_test.go and the benchmarks in storage_bench.go. From d843c42931ae5d3ba27afa099f316fa1067d83a8 Mon Sep 17 00:00:00 2001 From: "Lawrence, Rendall" Date: Tue, 6 Sep 2022 14:59:01 +0300 Subject: [PATCH 7/7] restore example config commented blocks formatting --- dist/example_config.yaml | 153 ++++++++++++++++++++------------------- 1 file changed, 77 insertions(+), 76 deletions(-) diff --git a/dist/example_config.yaml b/dist/example_config.yaml index 692ab2a..ed059b6 100644 --- a/dist/example_config.yaml +++ b/dist/example_config.yaml @@ -1,3 +1,4 @@ +# @formatter:off mochi: # The interval communicated with BitTorrent clients informing them how # frequently they should announce in between client events. @@ -154,96 +155,96 @@ mochi: # are collected and posted to Prometheus. prometheus_reporting_interval: 1s - # This block defines configuration used for redis storage. - #storage: - #name: redis - #config: - # The frequency which stale peers are removed. - # This balances between - # - collecting garbage more often, potentially using more CPU time, but potentially using less memory (lower value) - # - collecting garbage less frequently, saving CPU time, but keeping old peers long, thus using more memory (higher value). - #gc_interval: 3m + # This block defines configuration used for redis storage. + #storage: + #name: redis + #config: + # The frequency which stale peers are removed. + # This balances between + # - collecting garbage more often, potentially using more CPU time, but potentially using less memory (lower value) + # - collecting garbage less frequently, saving CPU time, but keeping old peers long, thus using more memory (higher value). + #gc_interval: 3m - # The interval at which metrics about the number of infohashes and peers - # are collected and posted to Prometheus. - #prometheus_reporting_interval: 1s + # The interval at which metrics about the number of infohashes and peers + # are collected and posted to Prometheus. + #prometheus_reporting_interval: 1s - # The amount of time until a peer is considered stale. - # To avoid churn, keep this slightly larger than `announce_interval` - #peer_lifetime: 31m + # The amount of time until a peer is considered stale. + # To avoid churn, keep this slightly larger than `announce_interval` + #peer_lifetime: 31m - # The addresses of redis storage. - # If neither sentinel not cluster switched, - # only first address used - #addresses: ["127.0.0.1:6379"] + # The addresses of redis storage. + # If neither sentinel not cluster switched, + # only first address used + #addresses: ["127.0.0.1:6379"] - # Database to be selected after connecting to the server. - #db: 0 + # Database to be selected after connecting to the server. + #db: 0 - # Maximum number of socket connections, default is 10 per CPU - #pool_size: 10 + # Maximum number of socket connections, default is 10 per CPU + #pool_size: 10 - # Use the specified login/username to authenticate the current connection - #login: "" + # Use the specified login/username to authenticate the current connection + #login: "" - # Optional password - #password: "" + # Optional password + #password: "" - # Connect to sentinel nodes - #sentinel: false + # Connect to sentinel nodes + #sentinel: false - # The master name - #sentinel_master: "" + # The master name + #sentinel_master: "" - # Connect to the redis cluster - #cluster: false + # Connect to the redis cluster + #cluster: false - # The timeout for reading a command reply from redis. - #read_timeout: 15s + # The timeout for reading a command reply from redis. + #read_timeout: 15s - # The timeout for writing a command to redis. - #write_timeout: 15s + # The timeout for writing a command to redis. + #write_timeout: 15s - # Dial timeout for establishing new connections. - #connect_timeout: 15s + # Dial timeout for establishing new connections. + #connect_timeout: 15s - # This block defines configuration used for PostgreSQL storage. - # example `mo_peers` table structure: - # - info_hash bytea - # - peer_id bytea - # - address inet or bytea - # - port int4 - # - is_seeder bool - # - is_v6 bool - # - created timestamp - #storage: - #name: pg - #config: - # connection string to pg storage. may be URL (postgres://...) or DSN (host=... port=...) - #connection_string: host=127.0.0.1 database=test user=postgres pool_max_conns=50 - # query and parameters for announce operation - #announce: - #query: SELECT peer_id, address, port FROM mo_peers WHERE info_hash=$1 AND is_seeder=$2 AND is_v6=$3 LIMIT $4 - #peer_id_column: peer_id - #address_column: address - #port_column: port - #peer: - # expected parameters: 1 - info hash (bytea), 2 - peer id (bytea), 3 - ip address (bytea/inet) - # 4 - port (int), 5 - is seeder (bool), 6 - is IPv6 (bool), 7 - create date and time (timestamp) - #add_query: INSERT INTO mo_peers VALUES($1, $2, $3, $4, $5, $6, $7) ON CONFLICT (info_hash, peer_id, address, port) DO UPDATE SET created = EXCLUDED.created, is_seeder = EXCLUDED.is_seeder - #del_query: DELETE FROM mo_peers WHERE info_hash=$1 AND peer_id=$2 AND address=$3 AND port=$4 AND is_seeder=$5 - #graduate_query: UPDATE mo_peers SET is_seeder=TRUE WHERE info_hash=$1 AND peer_id=$2 AND address=$3 AND port=$4 AND NOT is_seeder - #count_query: SELECT COUNT(1) FILTER (WHERE is_seeder) AS seeders, COUNT(1) FILTER (WHERE NOT is_seeder) AS leechers FROM mo_peers - # predicate part of `count_query` for get count of peers by info hash - #by_info_hash_clause: WHERE info_hash = $1 - #count_seeders_column: seeders - #count_leechers_column: leechers - # queries for KV-store - #data: - # expected parameters: 1 - context (varchar), 2 - name (bytea), 3 - value (bytea) - #add_query: INSERT INTO mo_kv VALUES($1, $2, $3) ON CONFLICT (context, name) DO NOTHING - #del_query: DELETE FROM mo_kv WHERE context=$1 AND name=$2 - #get_query: SELECT value FROM mo_kv WHERE context=$1 AND name=$2 + # This block defines configuration used for PostgreSQL storage. + # example `mo_peers` table structure: + # - info_hash bytea + # - peer_id bytea + # - address inet or bytea + # - port int4 + # - is_seeder bool + # - is_v6 bool + # - created timestamp + #storage: + #name: pg + #config: + # connection string to pg storage. may be URL (postgres://...) or DSN (host=... port=...) + #connection_string: host=127.0.0.1 database=test user=postgres pool_max_conns=50 + # query and parameters for announce operation + #announce: + #query: SELECT peer_id, address, port FROM mo_peers WHERE info_hash=$1 AND is_seeder=$2 AND is_v6=$3 LIMIT $4 + #peer_id_column: peer_id + #address_column: address + #port_column: port + #peer: + # expected parameters: 1 - info hash (bytea), 2 - peer id (bytea), 3 - ip address (bytea/inet) + # 4 - port (int), 5 - is seeder (bool), 6 - is IPv6 (bool), 7 - create date and time (timestamp) + #add_query: INSERT INTO mo_peers VALUES($1, $2, $3, $4, $5, $6, $7) ON CONFLICT (info_hash, peer_id, address, port) DO UPDATE SET created = EXCLUDED.created, is_seeder = EXCLUDED.is_seeder + #del_query: DELETE FROM mo_peers WHERE info_hash=$1 AND peer_id=$2 AND address=$3 AND port=$4 AND is_seeder=$5 + #graduate_query: UPDATE mo_peers SET is_seeder=TRUE WHERE info_hash=$1 AND peer_id=$2 AND address=$3 AND port=$4 AND NOT is_seeder + #count_query: SELECT COUNT(1) FILTER (WHERE is_seeder) AS seeders, COUNT(1) FILTER (WHERE NOT is_seeder) AS leechers FROM mo_peers + # predicate part of `count_query` for get count of peers by info hash + #by_info_hash_clause: WHERE info_hash = $1 + #count_seeders_column: seeders + #count_leechers_column: leechers + # queries for KV-store + #data: + # expected parameters: 1 - context (varchar), 2 - name (bytea), 3 - value (bytea) + #add_query: INSERT INTO mo_kv VALUES($1, $2, $3) ON CONFLICT (context, name) DO NOTHING + #del_query: DELETE FROM mo_kv WHERE context=$1 AND name=$2 + #get_query: SELECT value FROM mo_kv WHERE context=$1 AND name=$2 # query for check if database is alive #ping_query: SELECT 1 # query for garbage collection, expected parameter is timestamp