diff --git a/go.mod b/go.mod index e92a837..4ed15df 100644 --- a/go.mod +++ b/go.mod @@ -3,59 +3,46 @@ module git.ifooth.com/common/pkg go 1.25 require ( - github.com/dustin/go-humanize v1.0.1 - github.com/ggicci/httpin v0.19.0 - github.com/go-chi/chi/v5 v5.0.11 - github.com/go-chi/render v1.0.2 github.com/go-playground/locales v0.14.1 github.com/go-playground/universal-translator v0.18.1 github.com/go-playground/validator/v10 v10.23.0 - github.com/go-resty/resty/v2 v2.7.0 github.com/google/uuid v1.6.0 github.com/patrickmn/go-cache v2.1.0+incompatible - github.com/prometheus/client_golang v1.14.0 - github.com/redis/go-redis/v9 v9.0.3 + github.com/prometheus/client_golang v1.20.5 + github.com/redis/go-redis/v9 v9.12.1 github.com/samber/lo v1.47.0 github.com/spf13/cobra v1.10.2 - github.com/stretchr/testify v1.9.0 + github.com/stretchr/testify v1.10.0 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.52.0 ) require ( - github.com/ajg/form v1.5.1 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/gabriel-vasile/mimetype v1.4.3 // indirect - github.com/ggicci/owl v0.8.2 // indirect - github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/golang/protobuf v1.5.4 // indirect + github.com/google/go-cmp v0.7.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/kr/pretty v0.3.1 // indirect - github.com/labstack/echo/v4 v4.12.0 // indirect - github.com/labstack/gommon v0.4.2 // indirect github.com/leodido/go-urn v1.4.0 // indirect - github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.20 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_model v0.3.0 // indirect - github.com/prometheus/common v0.42.0 // indirect - github.com/prometheus/procfs v0.9.0 // indirect - github.com/rogpeppe/go-internal v1.12.0 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.62.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect + github.com/rogpeppe/go-internal v1.14.1 // indirect github.com/spf13/pflag v1.0.9 // indirect - github.com/valyala/bytebufferpool v1.0.0 // indirect - github.com/valyala/fasttemplate v1.2.2 // indirect - go.opentelemetry.io/otel v1.27.0 // indirect - go.opentelemetry.io/otel/metric v1.27.0 // indirect - go.opentelemetry.io/otel/trace v1.27.0 // indirect - golang.org/x/crypto v0.23.0 // indirect - golang.org/x/net v0.25.0 // indirect - golang.org/x/sys v0.20.0 // indirect - golang.org/x/text v0.16.0 // indirect - google.golang.org/protobuf v1.34.1 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/otel v1.34.0 // indirect + go.opentelemetry.io/otel/metric v1.34.0 // indirect + go.opentelemetry.io/otel/trace v1.34.0 // indirect + golang.org/x/crypto v0.36.0 // indirect + golang.org/x/net v0.38.0 // indirect + golang.org/x/sys v0.31.0 // indirect + golang.org/x/text v0.23.0 // indirect + google.golang.org/protobuf v1.36.5 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 4cc0f35..3db6c57 100644 --- a/go.sum +++ b/go.sum @@ -1,36 +1,23 @@ -github.com/ajg/form v1.5.1 h1:t9c7v8JUKu/XxOGBU0yjNpaMloxGEJhUkqFRq0ibGeU= -github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bsm/ginkgo/v2 v2.7.0 h1:ItPMPH90RbmZJt5GtkcNvIRuGEdwlBItdNVoyzaNQao= -github.com/bsm/ginkgo/v2 v2.7.0/go.mod h1:AiKlXPm7ItEHNc/2+OkrNG4E0ITzojb9/xWzvQ9XZ9w= -github.com/bsm/gomega v1.26.0 h1:LhQm+AFcgV2M0WyKroMASzAzCAJVpAxQXv4SaI9a69Y= -github.com/bsm/gomega v1.26.0/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= -github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= -github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= -github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= -github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= -github.com/ggicci/httpin v0.19.0 h1:p0B3SWLVgg770VirYiHB14M5wdRx3zR8mCTzM/TkTQ8= -github.com/ggicci/httpin v0.19.0/go.mod h1:hzsQHcbqLabmGOycf7WNw6AAzcVbsMeoOp46bWAbIWc= -github.com/ggicci/owl v0.8.2 h1:og+lhqpzSMPDdEB+NJfzoAJARP7qCG3f8uUC3xvGukA= -github.com/ggicci/owl v0.8.2/go.mod h1:PHRD57u41vFN5UtFz2SF79yTVoM3HlWpjMiE+ZU2dj4= -github.com/go-chi/chi/v5 v5.0.11 h1:BnpYbFZ3T3S1WMpD79r7R5ThWX40TaFB7L31Y8xqSwA= -github.com/go-chi/chi/v5 v5.0.11/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= -github.com/go-chi/render v1.0.2 h1:4ER/udB0+fMWB2Jlf15RV3F4A2FDuYi/9f+lFttR/Lg= -github.com/go-chi/render v1.0.2/go.mod h1:/gr3hVkmYR0YlEy3LxCuVRFzEu9Ruok+gFqbIofjao0= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= -github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= @@ -41,57 +28,38 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= github.com/go-playground/validator/v10 v10.23.0 h1:/PwmTwZhS0dPkav3cdK9kV1FsAmrL8sThn8IHr/sO+o= github.com/go-playground/validator/v10 v10.23.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= -github.com/go-resty/resty/v2 v2.7.0 h1:me+K9p3uhSmXtrBZ4k9jcEAfJmuC8IivWHwaLZwPrFY= -github.com/go-resty/resty/v2 v2.7.0/go.mod h1:9PWDzw47qPphMRFfhsyk0NnSgvluHcljSMVIq3w7q0I= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= -github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= -github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= -github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/justinas/alice v1.2.0 h1:+MHSA/vccVCF4Uq37S42jwlkvI2Xzl7zTPCN5BnZNVo= -github.com/justinas/alice v1.2.0/go.mod h1:fN5HRH/reO/zrUflLfTN43t3vXvKzvZIENsNEe7i7qA= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/labstack/echo/v4 v4.12.0 h1:IKpw49IMryVB2p1a4dzwlhP1O2Tf2E0Ir/450lH+kI0= -github.com/labstack/echo/v4 v4.12.0/go.mod h1:UP9Cr2DJXbOK3Kr9ONYzNowSh7HP0aG0ShAyycHSJvM= -github.com/labstack/gommon v0.4.2 h1:F8qTUNXgG1+6WQmqoUWnz8WiEU60mXVVw0P4ht1WRA0= -github.com/labstack/gommon v0.4.2/go.mod h1:QlUFxVM+SNXhDL/Z7YhocGIBYOiwB0mXm1+1bAPHPyU= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= -github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= -github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= -github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= -github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= -github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= -github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw= -github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y= -github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= -github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= -github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= -github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= -github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= -github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= -github.com/redis/go-redis/v9 v9.0.3 h1:+7mmR26M0IvyLxGZUHxu4GiBkJkVDid0Un+j4ScYu4k= -github.com/redis/go-redis/v9 v9.0.3/go.mod h1:WqMKv5vnQbRuZstUwxQI195wHy+t4PuXDOjzMvcuQHk= -github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= -github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= -github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= +github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io= +github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/redis/go-redis/v9 v9.12.1 h1:k5iquqv27aBtnTm2tIkROUDp8JBXhXZIVu1InSgvovg= +github.com/redis/go-redis/v9 v9.12.1/go.mod h1:huWgSWd8mW6+m0VPhJjSSQ+d6Nh1VICQ6Q5lHuCH/Iw= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/samber/lo v1.47.0 h1:z7RynLwP5nbyRscyvcD043DWYoOcYRv3mV8lBeqOCLc= github.com/samber/lo v1.47.0/go.mod h1:RmDH9Ct32Qy3gduHQuKJ3gW1fMHAnE/fAzQuf6He5cU= @@ -99,40 +67,29 @@ github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= -github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo= -github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.52.0 h1:9l89oX4ba9kHbBol3Xin3leYJ+252h0zszDtBwyKe2A= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.52.0/go.mod h1:XLZfZboOJWHNKUv7eH0inh0E9VV6eWDFB/9yJyTLPp0= -go.opentelemetry.io/otel v1.27.0 h1:9BZoF3yMK/O1AafMiQTVu0YDj5Ea4hPhxCs7sGva+cg= -go.opentelemetry.io/otel v1.27.0/go.mod h1:DMpAK8fzYRzs+bi3rS5REupisuqTheUlSZJ1WnZaPAQ= -go.opentelemetry.io/otel/metric v1.27.0 h1:hvj3vdEKyeCi4YaYfNjv2NUje8FqKqUY8IlF0FxV/ik= -go.opentelemetry.io/otel/metric v1.27.0/go.mod h1:mVFgmRlhljgBiuk/MP/oKylr4hs85GZAylncepAX/ak= -go.opentelemetry.io/otel/trace v1.27.0 h1:IqYb813p7cmbHk0a5y6pD5JPakbVfftRXABGt5/Rscw= -go.opentelemetry.io/otel/trace v1.27.0/go.mod h1:6RiD1hkAprV4/q+yd2ln1HG9GoPx39SuvvstaLBl+l4= +go.opentelemetry.io/otel v1.34.0 h1:zRLXxLCgL1WyKsPVrgbSdMN4c0FMkDAskSTQP+0hdUY= +go.opentelemetry.io/otel v1.34.0/go.mod h1:OWFPOQ+h4G8xpyjgqo4SxJYdDQ/qmRH+wivy7zzx9oI= +go.opentelemetry.io/otel/metric v1.34.0 h1:+eTR3U0MyfWjRDhmFMxe2SsW64QrZ84AOhvqS7Y+PoQ= +go.opentelemetry.io/otel/metric v1.34.0/go.mod h1:CEDrp0fy2D0MvkXE+dPV7cMi8tWZwX3dmaIhwPOaqHE= +go.opentelemetry.io/otel/trace v1.34.0 h1:+ouXS2V8Rd4hp4580a8q23bg0azF2nI8cqLYnC8mh/k= +go.opentelemetry.io/otel/trace v1.34.0/go.mod h1:Svm7lSjQD7kG7KJ/MUHPVXSDGz2OX4h0M2jHBhmSfRE= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= -golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= -golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= -golang.org/x/net v0.0.0-20211029224645-99673261e6eb/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= -golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= -golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= -golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= -google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= +golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= +golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= +golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= +golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= +golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= +golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= +google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= +google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/http/rest/generic.go b/http/rest/generic.go deleted file mode 100644 index 21ba5ac..0000000 --- a/http/rest/generic.go +++ /dev/null @@ -1,179 +0,0 @@ -package rest - -import ( - "context" - "encoding/json" - "fmt" - "io" - "log/slog" - "net/http" - "strings" - "time" - - "github.com/ggicci/httpin" - httpin_integration "github.com/ggicci/httpin/integration" - "github.com/go-chi/chi/v5" - "github.com/go-chi/render" - - "git.ifooth.com/common/pkg/i18n" -) - -// UnaryFunc Unary or ClientStreaming handle function -type UnaryFunc[In, Out any] func(context.Context, *In) (*Out, error) - -// StreamingServer server or bidi streaming server -type StreamingServer interface { - http.ResponseWriter - Context() context.Context -} - -// StreamFunc ServerStreaming or BidiStreaming handle function -type StreamFunc[In any] func(*In, StreamingServer) error - -// Handle Composable HTTP Handlers using generics -func Handle[In any, Out any](fn UnaryFunc[In, Out]) func(w http.ResponseWriter, r *http.Request) { - handleName := getHandleName(fn) - - f := func(w http.ResponseWriter, r *http.Request) { - st := time.Now() - var err error - defer func() { - collectHandleMetrics(handleName, r.Method, st, err) - }() - - in, err := decodeReq[In](r) - if err != nil { - slog.Error("handle decode request failed", "err", err) - _ = render.Render(w, r, APIError(err)) - return - } - - // 设置语言 - ctx := i18n.SetLang(r.Context(), r.Header.Get("Accept-Language")) - ctx = context.WithValue(ctx, reqCtxKey, r) - - out, err := fn(ctx, in) - if err != nil { - _ = render.Render(w, r, APIError(err)) - return - } - _ = render.Render(w, r, APIOK(out)) - } - - f = tracingHandler(handleName, f) - - return f -} - -type streamingServer struct { - http.ResponseWriter - *http.ResponseController - ctx context.Context -} - -// Context return svr's context -func (s *streamingServer) Context() context.Context { - return s.ctx -} - -// Stream Composable HTTP Handlers using generics -func Stream[In any](fn StreamFunc[In]) func(w http.ResponseWriter, r *http.Request) { - handleName := getHandleName(fn) - - f := func(w http.ResponseWriter, r *http.Request) { - st := time.Now() - var err error - defer func() { - collectHandleMetrics(handleName, r.Method, st, err) - }() - - in, err := decodeReq[In](r) - if err != nil { - slog.Error("handle decode request failed ", "err", err) - _ = render.Render(w, r, APIError(err)) - return - } - - // 设置语言 - ctx := i18n.SetLang(r.Context(), r.Header.Get("Accept-Language")) - ctx = context.WithValue(ctx, reqCtxKey, r) - - svr := &streamingServer{ - ResponseWriter: w, - ResponseController: http.NewResponseController(w), - ctx: ctx, - } - - err = fn(in, svr) - if err != nil { - _ = render.Render(w, r, APIError(err)) - } - } - - f = tracingHandler(handleName, f) - return f -} - -// decodeReq ... -func decodeReq[T any](r *http.Request) (*T, error) { - in := new(T) - var err error - - // http.Request 直接返回 - if _, ok := any(in).(*http.Request); ok { - return any(r).(*T), nil - } - - // 空值不需要反序列化 - if _, ok := any(in).(*EmptyReq); ok { - return in, nil - } - - in, err = httpin.Decode[T](r) - if err != nil { - return nil, err - } - - // Get/Delete 请求, 请求参数从url中获取 - if r.Method == http.MethodGet || r.Method == http.MethodDelete { - return in, nil - } - - // 处理表单请求 - if strings.Contains(r.Header.Get("Content-Type"), "application/x-www-form-urlencoded") { - return in, nil - } - - // Post 请求等, 从body中获取 - body, err := io.ReadAll(r.Body) - if err != nil { - return nil, err - } - - if err = json.Unmarshal(body, in); err != nil { - return nil, fmt.Errorf("unmarshal json body: %s", err) - } - return in, nil -} - -// EmptyReq 空的请求 -type EmptyReq struct{} - -// EmptyResp 空的返回 -type EmptyResp struct{} - -// PaginationReq 分页接口通用请求 -type PaginationReq struct { - Offset int `json:"offset" in:"query=offset" validate:"gte=0"` - Limit int `json:"limit" in:"query=limit" validate:"gte=0"` -} - -// PaginationResp 分页接口通用返回 -type PaginationResp[T any] struct { - Count int64 `json:"count"` - Items []*T `json:"items"` -} - -func init() { - httpin_integration.UseGochiURLParam("path", chi.URLParam) -} diff --git a/http/rest/generic_test.go b/http/rest/generic_test.go deleted file mode 100644 index ddbd803..0000000 --- a/http/rest/generic_test.go +++ /dev/null @@ -1,103 +0,0 @@ -package rest - -import ( - "bytes" - "io" - "net/http" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// NewMockRequest creates a new mock request. -func NewMockRequest(t *testing.T, method string, body io.ReadCloser) *http.Request { - req, err := http.NewRequest(method, "/vm/xxx?name=alice", body) - require.NoError(t, err) - return req -} - -// TestDecodeReq tests the decodeReq function. -func TestDecodeReq(t *testing.T) { - // Define a test struct - type TestStruct struct { - Field string `json:"field"` - EnvID string `json:"env_id" in:"path=env_id" validate:"required"` - Name string `json:"name" in:"query=name"` - } - - // Test case 1: GET request with no body - t.Run("GET request with no body", func(t *testing.T) { - req := NewMockRequest(t, http.MethodGet, nil) - result, err := decodeReq[TestStruct](req) - assert.NoError(t, err) - assert.Equal(t, "alice", result.Name) - }) - - // Test case 2: POST request with valid JSON body - t.Run("POST request with valid JSON body", func(t *testing.T) { - jsonBody := `{"field": "value", "env_id": "test", "name": "alice1"}` - body := io.NopCloser(bytes.NewBufferString(jsonBody)) - req := NewMockRequest(t, http.MethodPost, body) - result, err := decodeReq[TestStruct](req) - assert.NoError(t, err) - assert.Equal(t, "value", result.Field) - assert.Equal(t, "test", result.EnvID) - assert.Equal(t, "alice1", result.Name) - }) - - // Test case 3: POST request with invalid JSON body - t.Run("POST request with invalid JSON body", func(t *testing.T) { - jsonBody := `{"field": "value"` // Invalid JSON - body := io.NopCloser(bytes.NewBufferString(jsonBody)) - req := NewMockRequest(t, http.MethodPost, body) - result, err := decodeReq[TestStruct](req) - assert.Error(t, err) - assert.Nil(t, result) - }) - - // Test case 4: Invalid request header - t.Run("invalid request header", func(t *testing.T) { - jsonBody := `{"field": "value", "env_id": "test", "name": "alice1"}` - body := io.NopCloser(bytes.NewBufferString(jsonBody)) - req := NewMockRequest(t, http.MethodPost, body) - req.Header.Set("Content-Type", "application/x-www-form-urlencoded") - result, err := decodeReq[TestStruct](req) - assert.Error(t, err) - assert.Nil(t, result) - }) - - // Test case 5: EmptyReq type check - t.Run("EmptyReq type check", func(t *testing.T) { - req := NewMockRequest(t, http.MethodPost, nil) - result, err := decodeReq[EmptyReq](req) - assert.NoError(t, err) - assert.NotNil(t, result) - }) - - // Test case 5: EmptyReq type check - t.Run("HttpRequest type check", func(t *testing.T) { - req := &http.Request{ - Method: http.MethodPost, - } - result, err := decodeReq[http.Request](req) - assert.NoError(t, err) - assert.Equal(t, req.Method, result.Method) - }) -} - -func BenchmarkDecodeReq(b *testing.B) { - r := &http.Request{ - Method: http.MethodPost, - } - - for i := 0; i < b.N; i++ { - result, err := decodeReq[http.Request](r) - if err != nil { - b.Error(err) - } - if result.Method != http.MethodPost { - b.Error("invalid result") - } - } -} diff --git a/http/rest/handler.go b/http/rest/handler.go deleted file mode 100644 index 816ab64..0000000 --- a/http/rest/handler.go +++ /dev/null @@ -1,37 +0,0 @@ -package rest - -import ( - "net/http" - "net/http/pprof" -) - -// ProfilerHandler -func ProfilerHandler() http.Handler { - mux := http.NewServeMux() - mux.HandleFunc("/debug/pprof/", pprof.Index) - mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) - mux.HandleFunc("/debug/pprof/profile", pprof.Profile) - mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol) - mux.HandleFunc("/debug/pprof/trace", pprof.Trace) - - // 性能监控 - // https://github.com/thanos-io/thanos/blob/main/pkg/server/http/http.go#L118 - // mux.Handle("/debug/fgprof", fgprof.Handler()) - - return mux -} - -// HealthyHandler Healthz 接口 -func HealthzHandler(w http.ResponseWriter, r *http.Request) { - w.Write([]byte("OK")) -} - -// HealthyHandler 健康检查 -func HealthyHandler(w http.ResponseWriter, r *http.Request) { - w.Write([]byte("OK")) -} - -// ReadyHandler 健康检查 -func ReadyHandler(w http.ResponseWriter, r *http.Request) { - w.Write([]byte("OK")) -} diff --git a/http/rest/metrics.go b/http/rest/metrics.go deleted file mode 100644 index bd46dfc..0000000 --- a/http/rest/metrics.go +++ /dev/null @@ -1,67 +0,0 @@ -package rest - -import ( - "net/http" - "reflect" - "runtime" - "strconv" - "strings" - "time" - - "github.com/prometheus/client_golang/prometheus" - "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" -) - -var ( - requestCounter = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "http_requests_total", - Help: "Number of get requests.", - }, - []string{"handler", "method", "code"}, - ) - responseTimeDuration = prometheus.NewHistogramVec( - prometheus.HistogramOpts{ - Name: "http_request_duration_seconds", - Help: "Histogram of response time for HTTP requests.", - Buckets: []float64{0.1, 0.2, 0.5, 1, 2, 5, 10, 30, 60}, - }, - []string{"handler", "method", "code"}, - ) -) - -// getHandleName 获取FuncHandle/StreamHandle函数名 -func getHandleName(fn any) string { - fullName := runtime.FuncForPC(reflect.ValueOf(fn).Pointer()).Name() - if fullName == "" { - panic("get func name is empty") - } - - parts := strings.Split(fullName, ".") - lastPart := parts[len(parts)-1] - name := strings.TrimSuffix(lastPart, "-fm") - return name -} - -// collectHandleMetrics api指标数据 -func collectHandleMetrics(funcName, method string, st time.Time, err error) { - code := 200 - if err != nil { - code = APIError(err).(*APIResponse).HTTPStatusCode - } - - codeStr := strconv.Itoa(code) - requestCounter.WithLabelValues(funcName, method, codeStr).Inc() - duration := time.Since(st).Seconds() - responseTimeDuration.WithLabelValues(funcName, method, codeStr).Observe(duration) -} - -// tracingHandler -func tracingHandler(operation string, fn func(w http.ResponseWriter, r *http.Request)) func(w http.ResponseWriter, r *http.Request) { - return otelhttp.NewHandler(http.HandlerFunc(fn), operation).ServeHTTP -} - -func init() { - prometheus.MustRegister(requestCounter) - prometheus.MustRegister(responseTimeDuration) -} diff --git a/http/rest/middleware.go b/http/rest/middleware.go deleted file mode 100644 index 163db50..0000000 --- a/http/rest/middleware.go +++ /dev/null @@ -1,69 +0,0 @@ -// Package apis for http -package rest - -import ( - "context" - "fmt" - "log/slog" - "net/http" - "time" - - "github.com/go-chi/chi/v5/middleware" - - "git.ifooth.com/common/pkg/http/httpserver" - "git.ifooth.com/common/pkg/http/restyclient" -) - -// RequestID reuqest_id -func RequestID(next http.Handler) http.Handler { - fn := func(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() - requestID := r.Header.Get(requestIDHeaderKey) - if requestID == "" { - requestID = GenRequestID() - } - - ctx = WithRequestID(ctx, requestID) - ctx = context.WithValue(ctx, middleware.RequestIDKey, requestID) - - w.Header().Set(requestIDHeaderKey, requestID) - next.ServeHTTP(w, r.WithContext(ctx)) - } - return http.HandlerFunc(fn) -} - -// HandleLogger 记录请求日志 -func HandleLogger(next http.Handler) http.Handler { - fn := func(w http.ResponseWriter, r *http.Request) { - st := time.Now() - - // 优先使用蓝鲸网关的request_id - reqId := r.Header.Get("X-Bkapi-Request-ID") - if reqId == "" { - reqId = r.Header.Get("X-Request-Id") - } - - ctx := restyclient.WithRequestID(r.Context(), reqId) - r = r.WithContext(ctx) - - limit := 2048 - reqBuf := httpserver.NewLimitBuffer(limit) - r.Body = httpserver.TeeReadCloser(r.Body, reqBuf) - - ww := middleware.NewWrapResponseWriter(w, r.ProtoMajor) - respBuf := httpserver.NewLimitBuffer(limit) - ww.Tee(respBuf) - - next.ServeHTTP(ww, r) - - // 保证能读取前1K字符 - // if reqBuf.Remain() > 0 { - // io.Copy(io.Discard, io.LimitReader(r.Body, int64(reqBuf.Remain()))) - // } - - msg := fmt.Sprintf("Handle %s %s From %s", r.Method, r.RequestURI, r.RemoteAddr) - slog.Info(msg, "req_id", reqId, "status", ww.Status(), "duration", time.Since(st), "req", reqBuf.String(), "resp", respBuf.String()) - } - - return http.HandlerFunc(fn) -} diff --git a/http/rest/request.go b/http/rest/request.go deleted file mode 100644 index b3f0a9f..0000000 --- a/http/rest/request.go +++ /dev/null @@ -1,54 +0,0 @@ -package rest - -import ( - "context" - "net/http" - - "github.com/google/uuid" -) - -type contextKey struct { - name string -} - -var ( - reqCtxKey = &contextKey{"HTTPRequest"} -) - -type ctxKey int - -const ( - requestIDCtxKey = ctxKey(1) - requestIDHeaderKey = "X-Request-Id" -) - -// HTTPRequest return svr's request -func HTTPRequest(ctx context.Context) *http.Request { // nolint - val, ok := ctx.Value(reqCtxKey).(*http.Request) - if !ok { - panic("missing request in context") - } - return val -} - -// GenRequestID 生产 request_id -func GenRequestID() string { - id, _ := uuid.NewRandom() - return id.String() -} - -// RequestIDValue 获取 RequestId 值 -func RequestIDValue(ctx context.Context) string { - v, ok := ctx.Value(requestIDCtxKey).(string) - if !ok { - return "" - } - - return v -} - -// WithRequestID 设置 request_id -func WithRequestID(ctx context.Context, id string) context.Context { - newCtx := context.WithValue(ctx, requestIDCtxKey, id) - return newCtx -} diff --git a/http/rest/response.go b/http/rest/response.go deleted file mode 100644 index 1af1b75..0000000 --- a/http/rest/response.go +++ /dev/null @@ -1,79 +0,0 @@ -package rest - -import ( - "errors" - "net/http" - - "github.com/go-chi/render" -) - -var ( - // UnauthorizedError 未登入 - UnauthorizedError = errors.New("用户未登入") -) - -// APIResponse 返回的标准结构 -type APIResponse struct { - Err error `json:"-"` // low-level runtime error - HTTPStatusCode int `json:"-"` // http response status code - Code int `json:"code"` - Message string `json:"message"` - RequestId string `json:"request_id"` - Data any `json:"data"` -} - -// Render chi Render 实现 -func (res *APIResponse) Render(w http.ResponseWriter, r *http.Request) error { - statusCode := res.HTTPStatusCode - if statusCode == 0 { - statusCode = http.StatusOK - } - - res.RequestId = RequestIDValue(r.Context()) - render.Status(r, statusCode) - - return nil -} - -// UnauthorizedErrRender 未登入返回 -func AbortWithUnauthorizedError(err error) render.Renderer { - return &APIResponse{ - Code: 1401, - Message: err.Error(), - HTTPStatusCode: http.StatusUnauthorized, - } -} - -// AbortWithBadRequestError 错误 -func AbortWithBadRequestError(err error) render.Renderer { - return &APIResponse{ - Code: 1400, - Message: err.Error(), - HTTPStatusCode: http.StatusBadRequest, - } - -} - -// AbortWithBadRequestError 错误 -func APIError(err error) render.Renderer { - return &APIResponse{ - Code: 1400, - Message: err.Error(), - HTTPStatusCode: http.StatusBadRequest, - } - -} - -// AbortWithWithForbiddenError 没有权限 -func AbortWithWithForbiddenError(err error) render.Renderer { - return &APIResponse{ - Code: 1403, - Message: err.Error(), - HTTPStatusCode: http.StatusForbidden, - } -} - -// OKRender 正常返回 -func APIOK(data interface{}) render.Renderer { - return &APIResponse{Data: data} -} diff --git a/http/restyclient/client.go b/http/restyclient/client.go deleted file mode 100644 index 5593948..0000000 --- a/http/restyclient/client.go +++ /dev/null @@ -1,101 +0,0 @@ -// Package restyclient for http client -package restyclient - -import ( - "context" - "crypto/tls" - "net" - "net/http" - "sync" - "time" - - "github.com/go-resty/resty/v2" - "github.com/google/uuid" - "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" -) - -const ( - timeout = time.Second * 30 -) - -var ( - clientOnce sync.Once - silentClientOnce sync.Once - globalClient *resty.Client - globalSilentClient *resty.Client -) - -var dialer = &net.Dialer{ - Timeout: 30 * time.Second, - KeepAlive: 30 * time.Second, -} - -// defaultTransport default transport -var defaultTransport http.RoundTripper = &http.Transport{ - DialContext: dialer.DialContext, - ForceAttemptHTTP2: true, - MaxIdleConns: 100, - IdleConnTimeout: 90 * time.Second, - TLSHandshakeTimeout: 10 * time.Second, - ExpectContinueTimeout: 1 * time.Second, - // NOCC:gas/tls(设计如此) - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, // nolint -} - -// New : 新建 Client, 设置公共参数，tracing 等; 每次新建，cookies不复用 -func New() *resty.Client { - if globalClient == nil { - clientOnce.Do(func() { - globalClient = resty.New(). - SetTransport(otelhttp.NewTransport(defaultTransport)). - SetTimeout(timeout). - SetCookieJar(nil). - SetDebugBodyLimit(1024). - OnAfterResponse(restyAfterResponseHook). - SetPreRequestHook(restyBeforeRequestHook). - OnError(restyErrHook). - SetHeader("User-Agent", "restyclient") - }) - } - return globalClient -} - -// silentNew 安静模式，只打印错误日志 -func silentNew() *resty.Client { - if globalSilentClient == nil { - silentClientOnce.Do(func() { - globalSilentClient = resty.New(). - SetTransport(otelhttp.NewTransport(defaultTransport)). - SetTimeout(timeout). - SetCookieJar(nil). - SetDebugBodyLimit(1024). - // OnAfterResponse(restyAfterResponseHook). - // SetPreRequestHook(restyBeforeRequestHook). - OnError(restyErrHook). - SetHeader("User-Agent", "envmgr-restyclient") - }) - } - return globalSilentClient -} - -// R : New().R() 快捷方式, 已设置公共参数，tracing 等 -func R() *resty.Request { - return New().R() -} - -// SilentR : 安静模式，只打印错误日志 已设置公共参数，tracing 等, 只打印错误日志 -func SilentR() *resty.Request { - return silentNew().R() -} - -// GenRequestID 生产 request_id -func GenRequestID() string { - id, _ := uuid.NewRandom() - return id.String() -} - -// WithRequestID 设置 request_id -func WithRequestID(ctx context.Context, id string) context.Context { - newCtx := context.WithValue(ctx, requestIDCtxKey, id) - return newCtx -} diff --git a/http/restyclient/helper.go b/http/restyclient/helper.go deleted file mode 100644 index 108f009..0000000 --- a/http/restyclient/helper.go +++ /dev/null @@ -1,74 +0,0 @@ -package restyclient - -import ( - "encoding/json" - "errors" - "fmt" - "strconv" - - resty "github.com/go-resty/resty/v2" -) - -// CodeNotZeroErr ... -var CodeNotZeroErr = errors.New("resp code != 0") - -// BKResult 蓝鲸返回规范的结构体 -type BKResult[T any] struct { - Result bool `json:"result"` // 部分蓝鲸接口有, 按需校验 - Code any `json:"code"` - Message string `json:"message"` - Data *T `json:"data"` -} - -// NewBKResult create NewBKResult by resp -func NewBKResult[T any](resp *resty.Response) (*BKResult[T], error) { - if !resp.IsSuccess() { - return nil, fmt.Errorf("request failed, status: %s, message: %s", resp.Status(), resp.Body()) - } - - bkResult := new(BKResult[T]) - if err := json.Unmarshal(resp.Body(), bkResult); err != nil { - return nil, err - } - - if err := bkResult.ValidateCode(); err != nil { - return nil, err - } - - return bkResult, nil -} - -// NewBKData only create data by resp -func NewBKData[T any](resp *resty.Response) (*T, error) { - bkResult, err := NewBKResult[T](resp) - if err != nil { - return nil, err - } - - return bkResult.Data, nil -} - -// ValidateCode 返回结果是否OK -func (r *BKResult[T]) ValidateCode() error { - var resultCode int - - switch code := r.Code.(type) { - case int: - resultCode = code - case float64: - resultCode = int(code) - case string: - c, err := strconv.Atoi(code) - if err != nil { - return err - } - resultCode = c - default: - return fmt.Errorf("conversion to int from %T not supported", code) - } - - if resultCode != 0 { - return fmt.Errorf("%w, code=%d, message=%s", CodeNotZeroErr, resultCode, r.Message) - } - return nil -} diff --git a/http/restyclient/hook.go b/http/restyclient/hook.go deleted file mode 100644 index fce5202..0000000 --- a/http/restyclient/hook.go +++ /dev/null @@ -1,42 +0,0 @@ -package restyclient - -import ( - "fmt" - "log/slog" - "net/http" - - "github.com/dustin/go-humanize" - "github.com/go-resty/resty/v2" -) - -// restyBeforeRequestHook 请求hook -func restyBeforeRequestHook(c *resty.Client, r *http.Request) error { - rid := getRequestID(r) - r.Header.Set(requestIDHeaderKey, rid) - - rbody, err := reqToCurl(r) - if err != nil { - return err - } - - slog.With("req_id", rid).Info("restyclient REQ", "body", rbody) - return nil -} - -// restyAfterResponseHook 正常返回hook -func restyAfterResponseHook(c *resty.Client, resp *resty.Response) error { - // 最大打印 1024 个字符 - body := string(resp.Body()) - if len(body) > 1024 { - body = fmt.Sprintf("%s...(Total %s)", body[:1024], humanize.Bytes(uint64(len(body)))) - } - - slog.With("req_id", getRequestID(resp.RawResponse.Request)).Info("restyclient RESP", "status", resp.Status(), "duration", resp.Time(), "body", body) - - return nil -} - -// restyErrHook 错误hook -func restyErrHook(r *resty.Request, err error) { - slog.With("req_id", getRequestID(r.RawRequest)).Error("restyclient RESP", "err", err) -} diff --git a/http/restyclient/transport.go b/http/restyclient/transport.go deleted file mode 100644 index a34de95..0000000 --- a/http/restyclient/transport.go +++ /dev/null @@ -1,162 +0,0 @@ -package restyclient - -import ( - "bytes" - "fmt" - "io" - "log/slog" - "net/http" - "time" - - "github.com/dustin/go-humanize" -) - -type ctxKey int - -const ( - requestIDCtxKey = ctxKey(1) - requestIDHeaderKey = "X-Request-Id" -) - -var ( - // maskKeys 敏感参数和头部key - maskKeys = map[string]struct{}{ - "bk_app_secret": {}, - "bk_token": {}, - "Authorization": {}, - "X-Bkapi-Authorization": {}, - } -) - -func getRequestID(r *http.Request) string { - v, ok := r.Context().Value(requestIDCtxKey).(string) - if ok && v != "" { - return v - } - - rid := r.Header.Get(requestIDHeaderKey) - if rid != "" { - return rid - } - - return GenRequestID() -} - -// reqToCurl curl 格式的请求日志 -func reqToCurl(r *http.Request) (string, error) { - // 过滤掉敏感信息, header 和 query - headers := "" - for key, values := range r.Header { - for _, value := range values { - if _, ok := maskKeys[key]; ok { - value = "***" - } - headers += fmt.Sprintf(" -H %q", fmt.Sprintf("%s: %s", key, value)) - } - } - - rawURL := *r.URL - queryValue := rawURL.Query() - for key := range queryValue { - if _, ok := maskKeys[key]; ok { - queryValue.Set(key, "") - } - } - rawURL.RawQuery = queryValue.Encode() - - reqMsg := fmt.Sprintf("curl -X %s '%s'%s", r.Method, rawURL.String(), headers) - if r.Body != nil { - bodyBytes, err := io.ReadAll(r.Body) - if err != nil { - return "", err - } - r.Body.Close() // nolint - - r.Body = io.NopCloser(bytes.NewBuffer(bodyBytes)) - if len(bodyBytes) > 1024 { - reqMsg += fmt.Sprintf(" -d '%s...(Total %s)'", bodyBytes[:1024], humanize.Bytes(uint64(len(bodyBytes)))) - } else { - reqMsg += fmt.Sprintf(" -d '%s'", bodyBytes) - } - } - - return reqMsg, nil -} - -// respToCurl 返回日志 -func respToCurl(resp *http.Response, st time.Time) (string, error) { - var ( - bodyBytes []byte - err error - ) - if resp.Body != nil { - bodyBytes, err = io.ReadAll(resp.Body) - if err != nil { - return "", err - } - resp.Body.Close() // nolint - - resp.Body = io.NopCloser(bytes.NewBuffer(bodyBytes)) - } - - if len(bodyBytes) > 1024 { - respMsg := fmt.Sprintf("[%s] %s %s...(Total %s)", - resp.Status, time.Since(st), bodyBytes[:1024], humanize.Bytes(uint64(len(bodyBytes)))) - return respMsg, nil - } - - if len(bodyBytes) > 0 { - respMsg := fmt.Sprintf("[%s] %s %s", resp.Status, time.Since(st), bodyBytes) - return respMsg, nil - } - - respMsg := fmt.Sprintf("[%s] %s", resp.Status, time.Since(st)) - return respMsg, nil -} - -// curlLogTransport print curl log transport -type curlLogTransport struct { - Transport http.RoundTripper -} - -// RoundTrip curlLog Transport -func (t *curlLogTransport) RoundTrip(req *http.Request) (*http.Response, error) { - st := time.Now() - - rid := getRequestID(req) - req.Header.Set(requestIDHeaderKey, rid) - - // 记录请求 - rbody, err := reqToCurl(req) - if err != nil { - return nil, err - } - slog.With("req_id", rid).Info("restyclient REQ", "body", rbody) - - resp, err := t.transport(req).RoundTrip(req) - if err != nil { - slog.With("req_id", rid).Error("restyclient RESP", "err", err) - return nil, err - } - - // 记录返回 - respBody, err := respToCurl(resp, st) - if err != nil { - return nil, err - } - slog.With("req_id", rid).Info("restyclient REQ", "body", respBody) - - return resp, nil -} - -func (t *curlLogTransport) transport(_ *http.Request) http.RoundTripper { //nolint:unparam - if t.Transport != nil { - return t.Transport - } - return http.DefaultTransport -} - -// NewCurlLogTransport make a new curl log transport, default transport can be nil -func NewCurlLogTransport(transport http.RoundTripper) http.RoundTripper { - return &curlLogTransport{Transport: transport} -} diff --git a/http/httpserver/io.go b/rest/httpserver/io.go similarity index 100% rename from http/httpserver/io.go rename to rest/httpserver/io.go diff --git a/http/httpserver/io_test.go b/rest/httpserver/io_test.go similarity index 100% rename from http/httpserver/io_test.go rename to rest/httpserver/io_test.go diff --git a/task/.gitignore b/task/.gitignore new file mode 100644 index 0000000..28f43be --- /dev/null +++ b/task/.gitignore @@ -0,0 +1 @@ +backup.db \ No newline at end of file diff --git a/task/README.md b/task/README.md new file mode 100644 index 0000000..5f5a94f --- /dev/null +++ b/task/README.md @@ -0,0 +1,124 @@ +# BCS异步任务流框架 + +## 背景 + +* 主要为了解决项目中复杂场景(集群管理任务、资源管理、联邦集群管理任务等)的分布式并发任务处理及任务编排场景，通过统一的框架实现松耦合、易扩展的特性的任务管理系统 + +## 方案 + +## 技术方案 + +* 基于 [go-machinery](https://github.com/RichardKnop/machinery) 的 [Workflows](https://github.com/RichardKnop/machinery?tab=readme-ov-file#workflows) 的 **Chains** 模式，通过上层任务抽象，实现异步任务框架处理 +* 依赖：消息队列rabbitmmq、数据库mongo + +### 任务框架 + +### 支持的组件 + +* Brokers: etcd +* Locks：etcd +* Backends: etcd + +### 任务框架实现功能 + +* 基于协程级别的轻量级任务执行 +* 支持水平扩展，提升任务处理并发量 +* 支持任务流处理 +* 支持子任务变量共享 +* 支持任务从当前失败节点重试 +* 支持任务取消 +* 支持任务跳过失败子任务 +* 支持指定任务节点运行 +* 支持自定义的任务回调机制 +* 可扩展的变量渲染 +* 子任务超时控制、任务超时控制机制 + +### 任务模型 + +抽象任务结构如下所示，Task是主任务，Step是工作流子任务，通过StepSequence控制执行的顺序。 + +``` +// Task task definition +type Task struct { + // index for task, client should set this field + TaskIndex string `json:"taskIndex" bson:"taskIndex"` + TaskID string `json:"taskId" bson:"taskId"` + TaskType string `json:"taskType" bson:"taskType"` + TaskName string `json:"taskName" bson:"taskName"` + // steps and params + CurrentStep string `json:"currentStep" bson:"currentStep"` + StepSequence []string `json:"stepSequence" bson:"stepSequence"` + Steps map[string]*Step `json:"steps" bson:"steps"` + CallBackFuncName string `json:"callBackFuncName" bson:"callBackFuncName"` + CommonParams map[string]string `json:"commonParams" bson:"commonParams"` + ExtraJson string `json:"extraJson" bson:"extraJson"` + + Status string `json:"status" bson:"status"` + Message string `json:"message" bson:"message"` + ForceTerminate bool `json:"forceTerminate" bson:"forceTerminate"` + Start string `json:"start" bson:"start"` + End string `json:"end" bson:"end"` + ExecutionTime uint32 `json:"executionTime" bson:"executionTime"` + MaxExecutionSeconds uint32 `json:"maxExecutionSeconds" bson:"maxExecutionSeconds"` + Creator string `json:"creator" bson:"creator"` + LastUpdate string `json:"lastUpdate" bson:"lastUpdate"` + Updater string `json:"updater" bson:"updater"` +} + +// Step step definition +type Step struct { + Name string `json:"name" bson:"name"` + Method string `json:"method" bson:"method"` + StepName string `json:"stepname" bson:"stepname"` + Params map[string]string `json:"params" bson:"params"` + // step extras for string json, need client step to parse + Extras string `json:"extras" bson:"extras"` + Status string `json:"status" bson:"status"` + Message string `json:"message" bson:"message"` + SkipOnFailed bool `json:"skipOnFailed" bson:"skipOnFailed"` + RetryCount uint32 `json:"retryCount" bson:"retryCount"` + + Start string `json:"start" bson:"start"` + End string `json:"end" bson:"end"` + ExecutionTime uint32 `json:"executionTime" bson:"executionTime"` + MaxExecutionSeconds uint32 `json:"maxExecutionSeconds" bson:"maxExecutionSeconds"` + LastUpdate string `json:"lastUpdate" bson:"lastUpdate"` +} + +// StepWorkerInterface that client must implement +type StepWorkerInterface interface { + GetMethod() string + DoWork(task *types.Task) error +} + +// CallbackInterface that client must implement +type CallbackInterface interface { + GetName() string + Callback(isSuccess bool, task *types.Task) +} + +// TaskMgr build task +type TaskMgr interface { + Name() string + Type() string + BuildTask(info types.TaskInfo, opts ...types.TaskOption) (*types.Task, error) + Steps(defineSteps []StepMgr) []*types.Step +} + +// StepMgr build step +type StepMgr interface { + Name() string + GetMethod() string + BuildStep(kvs []KeyValue, opts ...types.StepOption) *types.Step + DoWork(task *types.Task) error +} +``` +* Task 是主任务，控制子任务执行顺序以及子任务的执行参数，并存储子任务共享变量，同时负责子任务的切换以及主任务的状态更新。 +* Step 是工作流子任务，进一步抽象是 接口StepWorkerInterface，实现重要的业务逻辑。而通过对接口StepWorkerInterface抽象封装来实现任务切换和子任务状态更新 +* StepMgr 为构建step子任务 以及 step子任务的业务逻辑执行体 +* TaskMgr 为构建task任务 +* CallbackInterface 注册回调方法 + +### 示例代码 + +接入框架的示例代码可参考 task 目录下的 example 例子 \ No newline at end of file diff --git a/task/backends/etcd/etcd.go b/task/backends/etcd/etcd.go new file mode 100644 index 0000000..e50fa55 --- /dev/null +++ b/task/backends/etcd/etcd.go @@ -0,0 +1,357 @@ +/* + * Tencent is pleased to support the open source community by making Blueking Container Service available. + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * Licensed under the MIT License (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * http://opensource.org/licenses/MIT + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package etcd implement machinery v2 backend iface +package etcd + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "sync" + "time" + + "github.com/RichardKnop/machinery/v2/backends/iface" + "github.com/RichardKnop/machinery/v2/common" + "github.com/RichardKnop/machinery/v2/config" + "github.com/RichardKnop/machinery/v2/log" + "github.com/RichardKnop/machinery/v2/tasks" + clientv3 "go.etcd.io/etcd/client/v3" + "golang.org/x/sync/errgroup" +) + +const ( + groupKey = "/machinery/v2/backend/groups/%s" + taskKey = "/machinery/v2/backend/tasks/%s" +) + +type etcdBackend struct { + common.Backend + ctx context.Context + client *clientv3.Client +} + +// New .. +func New(ctx context.Context, conf *config.Config) (iface.Backend, error) { + etcdConf := clientv3.Config{ + Endpoints: []string{conf.ResultBackend}, + Context: ctx, + DialTimeout: time.Second * 5, + TLS: conf.TLSConfig, + } + client, err := clientv3.New(etcdConf) + if err != nil { + return nil, err + } + + backend := etcdBackend{ + Backend: common.NewBackend(conf), + ctx: ctx, + client: client, + } + + return &backend, nil + +} + +// InitGroup Group related functions +func (b *etcdBackend) InitGroup(groupUUID string, taskUUIDs []string) error { + lease, err := b.getLease() + if err != nil { + return err + } + + groupMeta := &tasks.GroupMeta{ + GroupUUID: groupUUID, + TaskUUIDs: taskUUIDs, + CreatedAt: time.Now().UTC(), + TTL: lease.TTL, + } + + encoded, err := json.Marshal(groupMeta) + if err != nil { + return err + } + + key := fmt.Sprintf(groupKey, groupUUID) + _, err = b.client.Put(b.ctx, key, string(encoded), clientv3.WithLease(lease.ID)) + return err +} + +// GroupCompleted .. +func (b *etcdBackend) GroupCompleted(groupUUID string, groupTaskCount int) (bool, error) { + groupMeta, err := b.getGroupMeta(groupUUID) + if err != nil { + return false, err + } + + taskStates, err := b.getStates(groupMeta.TaskUUIDs...) + if err != nil { + return false, err + } + + var countSuccessTasks = 0 + for _, taskState := range taskStates { + if taskState.IsCompleted() { + countSuccessTasks++ + } + } + + return countSuccessTasks == groupTaskCount, nil +} + +// GroupTaskStates .. +func (b *etcdBackend) GroupTaskStates(groupUUID string, groupTaskCount int) ([]*tasks.TaskState, error) { + groupMeta, err := b.getGroupMeta(groupUUID) + if err != nil { + return nil, err + } + if len(groupMeta.TaskUUIDs) != groupTaskCount { + return nil, fmt.Errorf("group task count not equal, %d != %d", len(groupMeta.TaskUUIDs), groupTaskCount) + } + + return b.getStates(groupMeta.TaskUUIDs...) +} + +// TriggerChord .. +func (b *etcdBackend) TriggerChord(groupUUID string) (bool, error) { + key := fmt.Sprintf(groupKey, groupUUID) + resp, err := b.client.Get(b.ctx, key) + if err != nil { + return false, err + } + if len(resp.Kvs) == 0 { + return false, fmt.Errorf("task %s not exist", groupUUID) + } + kv := resp.Kvs[0] + + meta := new(tasks.GroupMeta) + + decoder := json.NewDecoder(bytes.NewReader(kv.Value)) + decoder.UseNumber() + + if e := decoder.Decode(meta); e != nil { + return false, e + } + + if meta.ChordTriggered { + return false, nil + } + + lease, err := b.getLease() + if err != nil { + return false, err + } + + // Set flag to true + meta.ChordTriggered = true + meta.TTL = lease.TTL + + // Update the group meta + encoded, err := json.Marshal(&meta) + if err != nil { + return false, err + } + + cmp := clientv3.Compare(clientv3.ModRevision(key), "=", kv.ModRevision) + update := clientv3.OpPut(key, string(encoded), clientv3.WithLease(lease.ID)) + + txnresp, err := b.client.Txn(b.ctx).If(cmp).Then(update).Commit() + if err != nil { + return false, err + } + + // 有写入或者删除竞争 + if !txnresp.Succeeded { + return false, fmt.Errorf("trigger chord failed, groupId: %s", groupUUID) + } + + return true, nil + +} + +func (b *etcdBackend) getGroupMeta(groupUUID string) (*tasks.GroupMeta, error) { + key := fmt.Sprintf(groupKey, groupUUID) + resp, err := b.client.Get(b.ctx, key) + if err != nil { + return nil, err + } + if len(resp.Kvs) == 0 { + return nil, fmt.Errorf("task %s not exist", groupUUID) + } + kv := resp.Kvs[0] + + meta := new(tasks.GroupMeta) + + decoder := json.NewDecoder(bytes.NewReader(kv.Value)) + decoder.UseNumber() + if err := decoder.Decode(meta); err != nil { + return nil, err + } + return meta, nil +} + +// SetStatePending updates task state to PENDING +func (b *etcdBackend) SetStatePending(signature *tasks.Signature) error { + taskState := tasks.NewPendingTaskState(signature) + return b.updateState(taskState) +} + +// SetStateReceived updates task state to RECEIVED +func (b *etcdBackend) SetStateReceived(signature *tasks.Signature) error { + taskState := tasks.NewReceivedTaskState(signature) + b.mergeNewTaskState(taskState) + return b.updateState(taskState) +} + +// SetStateStarted updates task state to STARTED +func (b *etcdBackend) SetStateStarted(signature *tasks.Signature) error { + taskState := tasks.NewStartedTaskState(signature) + b.mergeNewTaskState(taskState) + return b.updateState(taskState) +} + +// SetStateRetry updates task state to RETRY +func (b *etcdBackend) SetStateRetry(signature *tasks.Signature) error { + taskState := tasks.NewRetryTaskState(signature) + b.mergeNewTaskState(taskState) + return b.updateState(taskState) +} + +// SetStateSuccess updates task state to SUCCESS +func (b *etcdBackend) SetStateSuccess(signature *tasks.Signature, results []*tasks.TaskResult) error { + taskState := tasks.NewSuccessTaskState(signature, results) + b.mergeNewTaskState(taskState) + return b.updateState(taskState) +} + +// SetStateFailure updates task state to FAILURE +func (b *etcdBackend) SetStateFailure(signature *tasks.Signature, err string) error { + taskState := tasks.NewFailureTaskState(signature, err) + b.mergeNewTaskState(taskState) + return b.updateState(taskState) +} + +// GetState .. +func (b *etcdBackend) GetState(taskUUID string) (*tasks.TaskState, error) { + return b.getState(b.ctx, taskUUID) +} + +func (b *etcdBackend) getState(ctx context.Context, taskUUID string) (*tasks.TaskState, error) { + key := fmt.Sprintf(taskKey, taskUUID) + resp, err := b.client.Get(ctx, key) + if err != nil { + return nil, err + } + if len(resp.Kvs) == 0 { + return nil, fmt.Errorf("task %s not exist", taskUUID) + } + kv := resp.Kvs[0] + + state := new(tasks.TaskState) + + decoder := json.NewDecoder(bytes.NewReader(kv.Value)) + decoder.UseNumber() + if err := decoder.Decode(state); err != nil { + return nil, err + } + return state, nil +} + +func (b *etcdBackend) mergeNewTaskState(newState *tasks.TaskState) { + state, err := b.GetState(newState.TaskUUID) + if err == nil { + newState.CreatedAt = state.CreatedAt + newState.TaskName = state.TaskName + } +} + +// PurgeState .. +func (b *etcdBackend) PurgeState(taskUUID string) error { + key := fmt.Sprintf(taskKey, taskUUID) + _, err := b.client.Delete(b.ctx, key) + return err +} + +// PurgeGroupMeta .. +func (b *etcdBackend) PurgeGroupMeta(groupUUID string) error { + key := fmt.Sprintf(groupKey, groupUUID) + _, err := b.client.Delete(b.ctx, key) + return err +} + +// getStates returns multiple task states +func (b *etcdBackend) getStates(taskUUIDs ...string) ([]*tasks.TaskState, error) { + eg, ctx := errgroup.WithContext(b.ctx) + eg.SetLimit(10) + taskStates := make([]*tasks.TaskState, 0, len(taskUUIDs)) + var mtx sync.Mutex + for _, taskUUID := range taskUUIDs { + t := taskUUID + eg.Go(func() error { + state, err := b.getState(ctx, t) + if err != nil { + return err + } + + mtx.Lock() + taskStates = append(taskStates, state) + mtx.Unlock() + return nil + }) + } + if err := eg.Wait(); err != nil { + return nil, err + } + + return taskStates, nil +} + +// updateState saves current task state +func (b *etcdBackend) updateState(taskState *tasks.TaskState) error { + lease, err := b.getLease() + if err != nil { + return err + } + taskState.TTL = lease.TTL + + encoded, err := json.Marshal(taskState) + if err != nil { + return err + } + + key := fmt.Sprintf(taskKey, taskState.TaskUUID) + _, err = b.client.Put(b.ctx, key, string(encoded), clientv3.WithLease(lease.ID)) + if err != nil { + return err + } + + log.DEBUG.Printf("update taskstate %s %s, %s", taskState.TaskName, taskState.TaskUUID, encoded) + return nil +} + +// getLease returns expiration for a stored task state +func (b *etcdBackend) getLease() (*clientv3.LeaseGrantResponse, error) { + expiresIn := b.GetConfig().ResultsExpireIn + if expiresIn <= 0 { + // expire results after 1 hour by default + expiresIn = config.DefaultResultsExpireIn + } + + resp, err := b.client.Grant(b.ctx, int64(expiresIn)) + if err != nil { + return nil, err + } + + return resp, nil +} diff --git a/task/brokers/etcd/delay_task.go b/task/brokers/etcd/delay_task.go new file mode 100644 index 0000000..5ed130c --- /dev/null +++ b/task/brokers/etcd/delay_task.go @@ -0,0 +1,276 @@ +/* + * Tencent is pleased to support the open source community by making Blueking Container Service available. + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * Licensed under the MIT License (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * http://opensource.org/licenses/MIT + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language governing permissions and + * limitations under the License. + */ + +package etcd + +import ( + "context" + "fmt" + "sort" + "strconv" + "strings" + "time" + + "github.com/RichardKnop/machinery/v2/log" + "go.etcd.io/etcd/api/v3/mvccpb" + clientv3 "go.etcd.io/etcd/client/v3" + "go.etcd.io/etcd/client/v3/concurrency" + "golang.org/x/sync/errgroup" +) + +const ( + // delayTaskMaxETA is the maximum eta duration for list&watch delayed task + delayTaskMaxETA = time.Minute * 10 +) + +type delayTask struct { + key string // {delayedTaskPrefix}/eta-{ms}/{queue}/{taskID} + taskKey string // {queue}/{taskID} + eta time.Time // eta-{ms} + kv *mvccpb.KeyValue + bindValue *mvccpb.KeyValue +} + +func makeDelayTask(kv *mvccpb.KeyValue) (*delayTask, error) { + key := string(kv.Key) + // {delayedTaskPrefix}/eta-{ms}/{queue}/{taskID} + parts := strings.Split(key, "/") + if len(parts) != 8 { + return nil, fmt.Errorf("invalid key") + } + + taskKey := fmt.Sprintf("%s/%s", parts[6], parts[7]) + + // eta-{ms} -> {ms} + etaStr := strings.TrimPrefix(parts[5], "eta-") + + etaMilli, err := strconv.Atoi(etaStr) + if err != nil { + return nil, fmt.Errorf("invalid eta") + } + + task := &delayTask{ + key: key, + taskKey: taskKey, + eta: time.UnixMilli(int64(etaMilli)), + kv: kv, + } + return task, nil +} + +func (b *etcdBroker) listWatchDelayedTask(ctx context.Context) error { + keyPrefix := fmt.Sprintf("%s/eta-", delayedTaskPrefix) + endTime := time.Now().Add(delayTaskMaxETA) + + // List + listCtx, listCancel := context.WithTimeout(ctx, time.Second*10) + defer listCancel() + + end := strconv.FormatInt(endTime.UnixMilli(), 10) + rangeOpts := []clientv3.OpOption{clientv3.WithRange(keyPrefix + end)} + resp, err := b.client.Get(listCtx, keyPrefix+"0", rangeOpts...) + if err != nil { + return err + } + + b.delayedMtx.Lock() + // 清空数据 + b.delayedTask = make(map[string]*delayTask) + + for _, ev := range resp.Kvs { + task, err := makeDelayTask(ev) + if err != nil { + log.ERROR.Printf("make delay task %s failed, err: %s", ev.Key, err) + continue + } + b.delayedTask[task.key] = task + } + b.delayedMtx.Unlock() + + // Watch + watchCtx, watchCancel := context.WithTimeout(ctx, delayTaskMaxETA) + defer watchCancel() + + eg, egCtx := errgroup.WithContext(watchCtx) + eg.Go(func() error { + watchOpts := []clientv3.OpOption{ + clientv3.WithPrefix(), + clientv3.WithKeysOnly(), + clientv3.WithRev(resp.Header.Revision), + } + wc := b.client.Watch(egCtx, keyPrefix, watchOpts...) + for wresp := range wc { + if wresp.Err() != nil { + return wresp.Err() + } + + b.delayedMtx.Lock() + for _, ev := range wresp.Events { + task, err := makeDelayTask(ev.Kv) + if err != nil { + log.ERROR.Printf("make delay task %s failed, err: %s", ev.Kv.Key, err) + continue + } + if ev.Type == clientv3.EventTypeDelete { + delete(b.delayedTask, task.key) + } + + if ev.Type == clientv3.EventTypePut { + b.delayedTask[task.key] = task + } + } + b.delayedMtx.Unlock() + } + return nil + }) + + eg.Go(func() error { + tick := time.NewTicker(time.Second) + defer tick.Stop() + + for { + select { + case <-egCtx.Done(): + return nil + case <-tick.C: + b.handleDelayTask(egCtx) + } + } + }) + + return eg.Wait() +} + +func (b *etcdBroker) handleDelayedTask(ctx context.Context) error { + s, err := concurrency.NewSession(b.client) + if err != nil { + return err + } + defer s.Close() // nolint + + m := concurrency.NewMutex(s, delayedTaskLockKey) + + // 最长等待watch时间获取锁 + lockCtx, lockCancel := context.WithTimeout(ctx, delayTaskMaxETA) + defer lockCancel() + + log.INFO.Printf("try acquire delayed task lock") + if err = m.Lock(lockCtx); err != nil { + log.INFO.Printf("try acquire delayed task lock failed, err: %s", err) + return err + } + log.INFO.Printf("acquire delayed task lock done") + + defer func() { + unlockCtx, unlockCancel := context.WithTimeout(context.Background(), time.Second*5) + defer unlockCancel() + + if err = m.Unlock(unlockCtx); err != nil { + log.ERROR.Printf("unlock delayed task failed, err: %s", err) + } + }() + + log.INFO.Printf("start handle delayed task") + err = b.listWatchDelayedTask(ctx) + log.INFO.Printf("handle delayed task done, err: %v", err) + + return err +} + +func (b *etcdBroker) handleDelayTask(ctx context.Context) { + now := time.Now() + taskList := []*delayTask{} + + b.delayedMtx.Lock() + for _, task := range b.delayedTask { + if task.eta.Before(now) { + taskList = append(taskList, task) + } + } + b.delayedMtx.Unlock() + + // 最老的任务最快处理 + sort.Slice(taskList, func(i, j int) bool { + return taskList[i].eta.Before(taskList[j].eta) + }) + + // 异步任务随时可能插入, 最多处理1分钟后重新获取任务列表(aka 异步任务到期后, 最多延迟1分钟放到pending队列) + ctx, cancel := context.WithTimeout(ctx, time.Minute) + defer cancel() + + for _, task := range taskList { + // 超时控制 + select { + case <-ctx.Done(): + return + default: + } + + if err := b.ensureDelayTaskBody(ctx, task); err != nil { + log.ERROR.Printf("ensure delay task body %s failed, diff=%s, err=%s", task.key, time.Since(task.eta), err) + continue + } + + if err := b.moveToPendingTask(ctx, task); err != nil { + log.ERROR.Printf("move delay task %s failed, diff=%s, err=%s", task.key, time.Since(task.eta), err) + continue + } + + b.delayedMtx.Lock() + delete(b.delayedTask, task.key) + b.delayedMtx.Unlock() + } +} + +func (b *etcdBroker) ensureDelayTaskBody(ctx context.Context, task *delayTask) error { + if task.bindValue != nil { + return nil + } + + delayKeyNotChange := clientv3.Compare(clientv3.ModRevision(task.key), "=", task.kv.ModRevision) + getReq := clientv3.OpGet(task.key) + resp, err := b.client.Txn(ctx).If(delayKeyNotChange).Then(getReq).Commit() + if err != nil { + return err + } + + if len(resp.Responses) != 1 { + return fmt.Errorf("tnx resp invalid, count=%d", len(resp.Responses)) + } + + getResp := resp.Responses[0].GetResponseRange() + if len(getResp.Kvs) == 0 || len(getResp.Kvs[0].Value) == 0 { + return fmt.Errorf("have no body") + } + + task.bindValue = getResp.Kvs[0] + return nil +} + +func (b *etcdBroker) moveToPendingTask(ctx context.Context, task *delayTask) error { + pendingKey := fmt.Sprintf("%s/%s", pendingTaskPrefix, task.taskKey) + delayKeyNotChange := clientv3.Compare(clientv3.ModRevision(task.key), "=", task.kv.ModRevision) + pendingKeyNotExist := clientv3.Compare(clientv3.CreateRevision(pendingKey), "=", 0) + + deleteReq := clientv3.OpDelete(task.key) + putReq := clientv3.OpPut(pendingKey, string(task.kv.Value)) + c, err := b.client.Txn(ctx).If(delayKeyNotChange, pendingKeyNotExist).Then(deleteReq, putReq).Commit() + if err != nil { + return err + } + if !c.Succeeded { + return fmt.Errorf("txn not success, maybe key conflict, will retry later") + } + + log.DEBUG.Printf("move delay task %s to pending queue done, diff=%s", task.key, time.Since(task.eta)) + return nil +} diff --git a/task/brokers/etcd/delay_task_test.go b/task/brokers/etcd/delay_task_test.go new file mode 100644 index 0000000..6f112a1 --- /dev/null +++ b/task/brokers/etcd/delay_task_test.go @@ -0,0 +1,68 @@ +/* + * Tencent is pleased to support the open source community by making Blueking Container Service available. + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * Licensed under the MIT License (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * http://opensource.org/licenses/MIT + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language governing permissions and + * limitations under the License. + */ + +package etcd + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "go.etcd.io/etcd/api/v3/mvccpb" +) + +func TestMakeDelayTask(t *testing.T) { + tests := []struct { + name string + input string + taskKey string + eta time.Time + }{ + { + name: "delayed_task1", + input: "/machinery/v2/broker/delayed_tasks/eta-1/machinery_tasks/d30986b4-6634-4013-bf56-88c0463450c2-test-0", + taskKey: "machinery_tasks/d30986b4-6634-4013-bf56-88c0463450c2-test-0", + eta: time.UnixMilli(1), + }, + { + name: "delayed_task2", + input: "/machinery/v2/broker/delayed_tasks/eta-0/machinery_tasks/d30986b4-6634-4013-bf56-88c0463450c2-test-0", + taskKey: "machinery_tasks/d30986b4-6634-4013-bf56-88c0463450c2-test-0", + eta: time.UnixMilli(0), + }, + { + name: "delayed_task3", + input: "/machinery/v2/broker/delayed_tasks/eta-1732356480593/machinery_tasks/d30986b4-6634-4013-bf56-88c0463450c2-test-0", + taskKey: "machinery_tasks/d30986b4-6634-4013-bf56-88c0463450c2-test-0", + eta: time.UnixMilli(1732356480593), + }, + { + name: "delayed_task4", + input: "/machinery/v2/broker/delayed_tasks/eta-1732356480583/machinery_tasks/d30986b4-6634-4013-bf56-88c0463450c2-test-0", + taskKey: "machinery_tasks/d30986b4-6634-4013-bf56-88c0463450c2-test-0", + eta: time.UnixMilli(1732356480583), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + kv := &mvccpb.KeyValue{Key: []byte(tt.input)} + task, err := makeDelayTask(kv) + require.NoError(t, err) + assert.Equal(t, tt.input, task.key) + assert.Equal(t, tt.taskKey, task.taskKey) + assert.Equal(t, tt.eta, task.eta) + }) + } +} diff --git a/task/brokers/etcd/delivery.go b/task/brokers/etcd/delivery.go new file mode 100644 index 0000000..2de0f77 --- /dev/null +++ b/task/brokers/etcd/delivery.go @@ -0,0 +1,177 @@ +/* + * Tencent is pleased to support the open source community by making Blueking Container Service available. + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * Licensed under the MIT License (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * http://opensource.org/licenses/MIT + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language governing permissions and + * limitations under the License. + */ + +package etcd + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "time" + + "github.com/RichardKnop/machinery/v2/brokers/errs" + "github.com/RichardKnop/machinery/v2/log" + "github.com/RichardKnop/machinery/v2/tasks" + clientv3 "go.etcd.io/etcd/client/v3" +) + +// Delivery task delivery with ack and nack +type Delivery interface { + Ack() + Nack() + Body() []byte + Signature() *tasks.Signature +} + +type deliver struct { + ctx context.Context + client *clientv3.Client + signature *tasks.Signature + value []byte + key string + node string + aliveCancel func() +} + +// NewDelivery create the task delivery +func NewDelivery(ctx context.Context, client *clientv3.Client, key string, node string) (Delivery, error) { + d := &deliver{ + ctx: ctx, + client: client, + key: key, + node: node, + } + + if err := d.tryAssign(key, node); err != nil { + return nil, err + } + + return d, nil +} + +func (d *deliver) tryAssign(key string, node string) error { + ctx, cancel := context.WithTimeout(d.ctx, time.Second*5) + defer cancel() + + grantResp, err := d.client.Grant(ctx, 60) + if err != nil { + return err + } + + runningKey := fmt.Sprintf("%s/%s", runningTaskPrefix, key) + pendingKey := fmt.Sprintf("%s/%s", pendingTaskPrefix, key) + + keyExist := clientv3.Compare(clientv3.CreateRevision(pendingKey), ">", 0) + assignNotExist := clientv3.Compare(clientv3.CreateRevision(runningKey), "=", 0) + + value := fmt.Sprintf("%s-%s", node, time.Now().Format(time.RFC3339)) + putReq := clientv3.OpPut(runningKey, value, clientv3.WithLease(grantResp.ID)) + getReq := clientv3.OpGet(pendingKey) + + resp, err := d.client.Txn(ctx).If(keyExist, assignNotExist).Then(putReq, getReq).Commit() + if err != nil { + return err + } + if !resp.Succeeded { + return fmt.Errorf("task %s not exist or already assign", key) + } + + if len(resp.Responses) < 2 { + return fmt.Errorf("task %s tnx resp invalid, count=%d", key, len(resp.Responses)) + } + getResp := resp.Responses[1].GetResponseRange() + if len(getResp.Kvs) == 0 || len(getResp.Kvs[0].Value) == 0 { + return fmt.Errorf("task %s have no body", key) + } + kv := getResp.Kvs[0] + + signature := new(tasks.Signature) + decoder := json.NewDecoder(bytes.NewReader(kv.Value)) + decoder.UseNumber() + if err = decoder.Decode(signature); err != nil { + return errs.NewErrCouldNotUnmarshalTaskSignature(kv.Value, err) + } + + aliveCtx, aliveCancel := context.WithCancel(d.ctx) + keepRespCh, err := d.client.KeepAlive(aliveCtx, grantResp.ID) + if err != nil { + aliveCancel() + return err + } + + go func() { + defer aliveCancel() + + for { + select { + case <-d.ctx.Done(): + return + case <-aliveCtx.Done(): + return + case _, ok := <-keepRespCh: + if !ok { + return + } + } + } + }() + + d.aliveCancel = aliveCancel + d.signature = signature + d.value = kv.Value + return nil +} + +// Ack acknowledged the task is done +func (d *deliver) Ack() { + defer d.aliveCancel() + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + + pendingKey := fmt.Sprintf("%s/%s", pendingTaskPrefix, d.key) + _, err := d.client.Delete(ctx, pendingKey) + if err != nil { + log.ERROR.Printf("ack task %s err: %s", d.key, err) + } + + runningKey := fmt.Sprintf("%s/%s", runningTaskPrefix, d.key) + _, err = d.client.Delete(ctx, runningKey) + if err != nil { + log.ERROR.Printf("ack task %s err: %s", d.key, err) + } +} + +// Nack negatively acknowledge the delivery of task should handle again +func (d *deliver) Nack() { + defer d.aliveCancel() + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + + runningKey := fmt.Sprintf("%s/%s", runningTaskPrefix, d.key) + _, err := d.client.Delete(ctx, runningKey) + if err != nil { + log.ERROR.Printf("nack task %s err: %s", d.key, err) + } +} + +// Signature return the task Signature +func (d *deliver) Signature() *tasks.Signature { + return d.signature +} + +// Body return the task body +func (d *deliver) Body() []byte { + return d.value +} diff --git a/task/brokers/etcd/etcd.go b/task/brokers/etcd/etcd.go new file mode 100644 index 0000000..0bd5570 --- /dev/null +++ b/task/brokers/etcd/etcd.go @@ -0,0 +1,532 @@ +/* + * Tencent is pleased to support the open source community by making Blueking Container Service available. + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * Licensed under the MIT License (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * http://opensource.org/licenses/MIT + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package etcd is broker use etcd +package etcd + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "runtime" + "strings" + "sync" + "time" + + "github.com/RichardKnop/machinery/v2/brokers/errs" + "github.com/RichardKnop/machinery/v2/brokers/iface" + "github.com/RichardKnop/machinery/v2/common" + "github.com/RichardKnop/machinery/v2/config" + "github.com/RichardKnop/machinery/v2/log" + "github.com/RichardKnop/machinery/v2/tasks" + clientv3 "go.etcd.io/etcd/client/v3" + "golang.org/x/sync/errgroup" +) + +const ( + pendingTaskPrefix = "/machinery/v2/broker/pending_tasks" + runningTaskPrefix = "/machinery/v2/broker/running_tasks" + delayedTaskPrefix = "/machinery/v2/broker/delayed_tasks" + delayedTaskLockKey = "/machinery/v2/lock/delayed_tasks" +) + +type etcdBroker struct { + common.Broker + ctx context.Context + client *clientv3.Client + wg sync.WaitGroup + pendingTask map[string]struct{} + runningTask map[string]struct{} + delayedTask map[string]*delayTask + mtx sync.RWMutex + delayedMtx sync.RWMutex +} + +// New .. +func New(ctx context.Context, conf *config.Config) (iface.Broker, error) { + etcdConf := clientv3.Config{ + Endpoints: []string{conf.Broker}, + Context: ctx, + DialTimeout: time.Second * 5, + TLS: conf.TLSConfig, + } + + client, err := clientv3.New(etcdConf) + if err != nil { + return nil, err + } + + broker := etcdBroker{ + Broker: common.NewBroker(conf), + ctx: ctx, + client: client, + pendingTask: make(map[string]struct{}), + runningTask: make(map[string]struct{}), + delayedTask: make(map[string]*delayTask), + } + + return &broker, nil + +} + +// nolint +// StartConsuming ... +func (b *etcdBroker) StartConsuming(consumerTag string, concurrency int, taskProcessor iface.TaskProcessor) (bool, error) { + if concurrency < 1 { + concurrency = runtime.NumCPU() + } + b.Broker.StartConsuming(consumerTag, concurrency, taskProcessor) + + log.INFO.Printf("[*] Waiting for messages, concurrency=%d. To exit press CTRL+C", concurrency) + + // Channel to which we will push tasks ready for processing by worker + deliveries := make(chan Delivery) + defer log.INFO.Printf("stop all consuming and handle done") + defer b.wg.Wait() + + ctx, cancel := context.WithCancel(b.ctx) + defer cancel() + + // list watch running task + b.wg.Add(1) + go func() { + defer func() { + cancel() + b.wg.Done() + log.INFO.Printf("list watch running task stopped") + }() + + for { + select { + case <-b.GetStopChan(): + return + case <-ctx.Done(): + return + default: + err := b.listWatchRunningTask(ctx) + if err != nil { + log.ERROR.Printf("list watch running task failed, err: %s", err) + time.Sleep(time.Second) + } + } + } + }() + + // list watch pending task + b.wg.Add(1) + go func() { + defer func() { + cancel() + b.wg.Done() + log.INFO.Printf("list watch pending task stopped") + }() + + queue := getQueue(b.GetConfig(), taskProcessor) + for { + select { + case <-b.GetStopChan(): + return + case <-ctx.Done(): + return + default: + err := b.listWatchPendingTask(ctx, queue) + if err != nil { + log.ERROR.Printf("list watch pending task failed, err: %s", err) + time.Sleep(time.Second) + } + } + } + }() + + // A receiving goroutine keeps popping messages from the queue by BLPOP + // If the message is valid and can be unmarshaled into a proper structure + // we send it to the deliveries channel + b.wg.Add(1) + go func() { + defer func() { + cancel() + close(deliveries) + b.wg.Done() + log.INFO.Printf("handle next task stopped") + }() + + for { + select { + case <-b.GetStopChan(): + return + case <-ctx.Done(): + return + default: + if !taskProcessor.PreConsumeHandler() { + continue + } + + task := b.nextTask(ctx, getQueue(b.GetConfig(), taskProcessor), consumerTag) + if task == nil { + time.Sleep(time.Second) + continue + } + + deliveries <- task + } + } + }() + + // A goroutine to watch for delayed tasks and push them to deliveries + // channel for consumption by the worker + b.wg.Add(1) + go func() { + defer func() { + cancel() + b.wg.Done() + log.INFO.Printf("handle delayed task stopped") + }() + + for { + select { + // A way to stop this goroutine from b.StopConsuming + case <-b.GetStopChan(): + return + case <-ctx.Done(): + return + default: + err := b.handleDelayedTask(ctx) + if err != nil && !errors.Is(err, context.DeadlineExceeded) { + log.ERROR.Printf("handle delayed task failed, err: %s", err) + } + time.Sleep(time.Second) + } + } + }() + + if err := b.consume(deliveries, concurrency, taskProcessor); err != nil { + log.WARNING.Printf("consume stopped, err=%v, retry=%t", err, b.GetRetry()) + return b.GetRetry(), err + } + + log.INFO.Printf("consume stopped, retry=%t", b.GetRetry()) + return b.GetRetry(), nil +} + +// consume takes delivered messages from the channel and manages a worker pool +// to process tasks concurrently +func (b *etcdBroker) consume(deliveries <-chan Delivery, concurrency int, taskProcessor iface.TaskProcessor) error { + eg, ctx := errgroup.WithContext(b.ctx) + + for i := 0; i < concurrency; i++ { + eg.Go(func() error { + for { + select { + case <-ctx.Done(): + return ctx.Err() + + case t, ok := <-deliveries: + if !ok { + return nil + } + + if err := b.consumeOne(t, taskProcessor); err != nil { + return err + } + } + } + }) + } + + return eg.Wait() +} + +// consumeOne processes a single message using TaskProcessor +func (b *etcdBroker) consumeOne(delivery Delivery, taskProcessor iface.TaskProcessor) error { + // If the task is not registered, we requeue it, + // there might be different workers for processing specific tasks + if !b.IsTaskRegistered(delivery.Signature().Name) { + log.INFO.Printf("Task not registered with this worker. Requeuing message: %s", delivery.Body()) + + if !delivery.Signature().IgnoreWhenTaskNotRegistered { + delivery.Nack() + } + return nil + } + + log.DEBUG.Printf("Received new message: %s", delivery.Body()) + defer delivery.Ack() + + return taskProcessor.Process(delivery.Signature()) +} + +// StopConsuming 停止 +func (b *etcdBroker) StopConsuming() { + b.Broker.StopConsuming() + + b.wg.Wait() +} + +// Publish put kvs to etcd stor +func (b *etcdBroker) Publish(ctx context.Context, signature *tasks.Signature) error { + // Adjust routing key (this decides which queue the message will be published to) + b.Broker.AdjustRoutingKey(signature) + + msg, err := json.Marshal(signature) + if err != nil { + return fmt.Errorf("JSON marshal error: %s", err) + } + + key := fmt.Sprintf("%s/%s/%s", pendingTaskPrefix, signature.RoutingKey, signature.UUID) + + // Check the ETA signature field, alway delay the task if not nil, + // prevent the key overwrite by slow ack request + if signature.ETA != nil { + key = fmt.Sprintf("%s/eta-%d/%s/%s", + delayedTaskPrefix, signature.ETA.UnixMilli(), signature.RoutingKey, signature.UUID) + } + + _, err = b.client.Put(ctx, key, string(msg)) + if err != nil { + log.ERROR.Printf("Publish queue[%s] new message: %s", key, string(msg)) + } else { + log.DEBUG.Printf("Publish queue[%s] new message: %s", key, string(msg)) + } + return err +} + +func (b *etcdBroker) getTasks(ctx context.Context, key string) ([]*tasks.Signature, error) { + resp, err := b.client.Get(ctx, key, clientv3.WithPrefix()) + if err != nil { + return nil, err + } + + result := make([]*tasks.Signature, 0, len(resp.Kvs)) + for _, kv := range resp.Kvs { + signature := new(tasks.Signature) + decoder := json.NewDecoder(bytes.NewReader(kv.Value)) + decoder.UseNumber() + if err := decoder.Decode(signature); err != nil { + return nil, errs.NewErrCouldNotUnmarshalTaskSignature(kv.Value, err) + } + + result = append(result, signature) + } + + return result, nil +} + +// GetPendingTasks 获取执行队列, 任务统计可使用 +func (b *etcdBroker) GetPendingTasks(queue string) ([]*tasks.Signature, error) { + if queue == "" { + queue = b.GetConfig().DefaultQueue + } + + key := fmt.Sprintf("%s/%s", pendingTaskPrefix, queue) + items, err := b.getTasks(b.ctx, key) + if err != nil { + return nil, err + } + + return items, nil +} + +// GetDelayedTasks 任务统计可使用 +func (b *etcdBroker) GetDelayedTasks() ([]*tasks.Signature, error) { + items, err := b.getTasks(b.ctx, delayedTaskPrefix) + if err != nil { + return nil, err + } + + return items, nil +} + +func (b *etcdBroker) nextTask(ctx context.Context, queue string, consumerTag string) Delivery { + b.mtx.Lock() + runningTask := make(map[string]struct{}, len(b.runningTask)) + for k, v := range b.runningTask { + runningTask[k] = v + } + pendingTask := make(map[string]struct{}, len(b.pendingTask)) + for k, v := range b.pendingTask { + pendingTask[k] = v + } + b.mtx.Unlock() + + for k := range pendingTask { + if !strings.Contains(k, queue) { + continue + } + + if _, ok := runningTask[k]; ok { + continue + } + + d, err := NewDelivery(ctx, b.client, k, consumerTag) + if err != nil { + continue + } + + b.mtx.Lock() + b.runningTask[k] = struct{}{} + b.mtx.Unlock() + + return d + } + + return nil +} + +func (b *etcdBroker) listWatchRunningTask(ctx context.Context) error { + // List + listCtx, listCancel := context.WithTimeout(ctx, time.Second*10) + defer listCancel() + resp, err := b.client.Get(listCtx, runningTaskPrefix, clientv3.WithPrefix(), clientv3.WithKeysOnly()) + if err != nil { + return err + } + + b.mtx.Lock() + b.runningTask = make(map[string]struct{}) + for _, kv := range resp.Kvs { + key := string(kv.Key) + taskKey := findTaskKey(key) + if taskKey == "" { + continue + } + b.runningTask[taskKey] = struct{}{} + } + b.mtx.Unlock() + + // Watch + watchCtx, watchCancel := context.WithTimeout(ctx, time.Minute*10) + defer watchCancel() + + watchOpts := []clientv3.OpOption{ + clientv3.WithPrefix(), + clientv3.WithKeysOnly(), + clientv3.WithRev(resp.Header.Revision), + } + wc := b.client.Watch(watchCtx, runningTaskPrefix, watchOpts...) + for wresp := range wc { + if wresp.Err() != nil { + return wresp.Err() + } + + b.mtx.Lock() + for _, ev := range wresp.Events { + key := string(ev.Kv.Key) + taskKey := findTaskKey(key) + if taskKey == "" { + continue + } + if ev.Type == clientv3.EventTypeDelete { + delete(b.runningTask, taskKey) + } + + if ev.Type == clientv3.EventTypePut { + b.runningTask[taskKey] = struct{}{} + } + } + b.mtx.Unlock() + } + + return nil +} + +func (b *etcdBroker) listWatchPendingTask(ctx context.Context, queue string) error { + keyPrefix := fmt.Sprintf("%s/%s", pendingTaskPrefix, queue) + + // List + listCtx, listCancel := context.WithTimeout(ctx, time.Second*10) + defer listCancel() + resp, err := b.client.Get(listCtx, keyPrefix, clientv3.WithPrefix(), clientv3.WithKeysOnly()) + if err != nil { + return err + } + + b.mtx.Lock() + b.pendingTask = make(map[string]struct{}) + for _, kv := range resp.Kvs { + key := string(kv.Key) + taskKey := findTaskKey(key) + if taskKey == "" { + continue + } + b.pendingTask[taskKey] = struct{}{} + } + b.mtx.Unlock() + + // Watch + watchCtx, watchCancel := context.WithTimeout(ctx, time.Minute*10) + defer watchCancel() + + watchOpts := []clientv3.OpOption{ + clientv3.WithPrefix(), + clientv3.WithKeysOnly(), + clientv3.WithRev(resp.Header.Revision), + } + wc := b.client.Watch(watchCtx, keyPrefix, watchOpts...) + for wresp := range wc { + if wresp.Err() != nil { + return wresp.Err() + } + + b.mtx.Lock() + for _, ev := range wresp.Events { + key := string(ev.Kv.Key) + taskKey := findTaskKey(key) + if taskKey == "" { + continue + } + if ev.Type == clientv3.EventTypeDelete { + delete(b.pendingTask, taskKey) + } + + if ev.Type == clientv3.EventTypePut { + b.pendingTask[taskKey] = struct{}{} + } + } + b.mtx.Unlock() + } + + return nil +} + +func getQueue(config *config.Config, taskProcessor iface.TaskProcessor) string { + customQueue := taskProcessor.CustomQueue() + if customQueue == "" { + return config.DefaultQueue + } + return customQueue +} + +// findTaskKey return {queue}/{taskID} +func findTaskKey(key string) string { + switch { + case strings.HasPrefix(key, pendingTaskPrefix+"/"): + return key[len(pendingTaskPrefix)+1:] + + case strings.HasPrefix(key, runningTaskPrefix+"/"): + return key[len(runningTaskPrefix)+1:] + + case strings.HasPrefix(key, delayedTaskPrefix): + // {delayedTaskPrefix}/eta-{ms}/{queue}/{taskID} + parts := strings.Split(key, "/") + if len(parts) != 8 { + log.WARNING.Printf("invalid delay task %s, just ignore", key) + return "" + } + return fmt.Sprintf("%s/%s", parts[6], parts[7]) + + default: + log.WARNING.Printf("invalid task %s, just ignore", key) + return "" + } +} diff --git a/task/brokers/etcd/etcd_test.go b/task/brokers/etcd/etcd_test.go new file mode 100644 index 0000000..d2ec160 --- /dev/null +++ b/task/brokers/etcd/etcd_test.go @@ -0,0 +1,221 @@ +/* + * Tencent is pleased to support the open source community by making Blueking Container Service available. + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * Licensed under the MIT License (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * http://opensource.org/licenses/MIT + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language governing permissions and + * limitations under the License. + */ + +package etcd + +import ( + "context" + "os" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/RichardKnop/machinery/v2/config" + "github.com/RichardKnop/machinery/v2/tasks" +) + +func TestFindTaskKey(t *testing.T) { + tests := []struct { + name string + input string + key string + }{ + { + name: "running_task1", + input: "/machinery/v2/broker/running_tasks/machinery_tasks/d30986b4-6634-4013-bf56-88c0463450c2-test-0", + key: "machinery_tasks/d30986b4-6634-4013-bf56-88c0463450c2-test-0", + }, + { + name: "pending_task1", + input: "/machinery/v2/broker/pending_tasks/machinery_tasks/d30986b4-6634-4013-bf56-88c0463450c2-test-0", + key: "machinery_tasks/d30986b4-6634-4013-bf56-88c0463450c2-test-0", + }, + { + name: "delayed_task1", + input: "/machinery/v2/broker/delayed_tasks/eta-0/machinery_tasks/d30986b4-6634-4013-bf56-88c0463450c2-test-0", + key: "machinery_tasks/d30986b4-6634-4013-bf56-88c0463450c2-test-0", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + k := findTaskKey(tt.input) + assert.Equal(t, tt.key, k) + }) + } +} + +func TestHandleDelayedTask(t *testing.T) { + endpoints := os.Getenv("ETCDCTL_ENDPOINTS") + if endpoints == "" { + t.Skip("ETCDCTL_ENDPOINTS is not set") + } + t.Parallel() + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*20) + defer cancel() + + eta := time.Now().Add(time.Second * 10) + mytask := &tasks.Signature{ + Name: "test_delay_task", + UUID: "test-delay-0", + RoutingKey: "test", + ETA: &eta, + } + broker, err := New(ctx, &config.Config{Broker: endpoints}) + etcdBroker := broker.(*etcdBroker) + require.NoError(t, err) + + err = broker.Publish(ctx, mytask) + require.NoError(t, err) + + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + + herr := etcdBroker.handleDelayedTask(ctx) + if herr != nil { + assert.ErrorIs(t, herr, context.DeadlineExceeded) + return + } + + assert.NoError(t, herr) + }() + + wg.Add(1) + go func() { + defer wg.Done() + // 排队等待 + time.Sleep(time.Second) + + herr := etcdBroker.handleDelayedTask(ctx) + if herr != nil { + assert.ErrorIs(t, herr, context.DeadlineExceeded) + return + } + + assert.NoError(t, herr) + + }() + + wg.Wait() + + // 完成后，上面的锁需要立即释放 + err = etcdBroker.handleDelayedTask(ctx) + if err != nil { + assert.ErrorIs(t, err, context.DeadlineExceeded) + return + } + + assert.NoError(t, err) +} + +func TestHandleDelayedMultiTask(t *testing.T) { + endpoints := os.Getenv("ETCDCTL_ENDPOINTS") + if endpoints == "" { + t.Skip("ETCDCTL_ENDPOINTS is not set") + } + t.Parallel() + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*20) + defer cancel() + + eta := time.Now().Add(time.Second * 10) + mytask := &tasks.Signature{ + Name: "test_delay_task", + UUID: "test-delay-00", + RoutingKey: "test", + ETA: &eta, + } + broker, err := New(ctx, &config.Config{Broker: endpoints}) + etcdBroker := broker.(*etcdBroker) + require.NoError(t, err) + + err = broker.Publish(ctx, mytask) + require.NoError(t, err) + + eta = time.Now().Add(time.Second * 5) + err = broker.Publish(ctx, &tasks.Signature{ + Name: "test_delay_task1", + UUID: "test-delay-01", + RoutingKey: "test", + ETA: &eta, + }) + require.NoError(t, err) + + var wg sync.WaitGroup + + wg.Add(1) + go func() { + defer wg.Done() + taskSlice, err := broker.GetDelayedTasks() + assert.NoError(t, err) + assert.True(t, len(taskSlice) >= 2) + // 排队等待 + time.Sleep(time.Second * 10) + + err = etcdBroker.handleDelayedTask(ctx) + if err != nil { + assert.ErrorIs(t, err, context.DeadlineExceeded) + return + } + + assert.NoError(t, err) + }() + + wg.Wait() +} + +func TestListWatchPendingTask(t *testing.T) { + endpoints := os.Getenv("ETCDCTL_ENDPOINTS") + if endpoints == "" { + t.Skip("ETCDCTL_ENDPOINTS is not set") + } + t.Parallel() + + st := time.Now() + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*15) + defer cancel() + + mytask := &tasks.Signature{ + Name: "test_task", + UUID: "test-0", + RoutingKey: "test", + } + broker, err := New(ctx, &config.Config{Broker: endpoints}) + etcdBroker := broker.(*etcdBroker) + require.NoError(t, err) + + err = broker.Publish(ctx, mytask) + require.NoError(t, err) + + var wg sync.WaitGroup + wg.Add(1) + + go func() { + defer wg.Done() + + err := etcdBroker.listWatchPendingTask(ctx, "test") + assert.NoError(t, err) + _, ok := etcdBroker.pendingTask["test/test-0"] + assert.True(t, ok) + assert.GreaterOrEqual(t, len(etcdBroker.pendingTask), 1) + duration := time.Since(st) + assert.True(t, duration > time.Second*15, "lock duration %s should be greater than 15s", duration) + }() + wg.Wait() +} diff --git a/task/builder.go b/task/builder.go new file mode 100644 index 0000000..5381129 --- /dev/null +++ b/task/builder.go @@ -0,0 +1,47 @@ +/* + * Tencent is pleased to support the open source community by making Blueking Container Service available. + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * Licensed under the MIT License (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * http://opensource.org/licenses/MIT + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language governing permissions and + * limitations under the License. + */ + +package task + +import ( + "fmt" + + "git.ifooth.com/common/pkg/task/types" +) + +// NewByTaskBuilder init task from builder +func NewByTaskBuilder(builder types.TaskBuilder, opts ...types.TaskOption) (*types.Task, error) { + // 声明step + steps, err := builder.Steps() + if err != nil { + return nil, err + } + + if len(steps) == 0 { + return nil, fmt.Errorf("task steps empty") + } + + task := types.NewTask(builder.TaskInfo(), opts...) + task.Steps = steps + task.CurrentStep = steps[0].GetName() + + // 自定义任务超时, commonParams, commonPayload等 + if err := builder.FinalizeTask(task); err != nil { + return nil, err + } + + if err := task.Validate(); err != nil { + return nil, err + } + + return task, nil +} diff --git a/task/example/call_back.go b/task/example/call_back.go new file mode 100644 index 0000000..d556028 --- /dev/null +++ b/task/example/call_back.go @@ -0,0 +1,40 @@ +/* + * Tencent is pleased to support the open source community by making Blueking Container Service available. + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * Licensed under the MIT License (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * http://opensource.org/licenses/MIT + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package main xxx +package main + +import ( + "fmt" + + istep "git.ifooth.com/common/pkg/task/steps/iface" +) + +const ( + callBackName = "callBack" +) + +type callBack struct{} + +// Callback 回调方法,根据任务成功状态更新实体对象状态 +func (cb *callBack) Callback(c *istep.Context, cbErr error) { + if cbErr != nil { + fmt.Println("success") + return + } + + fmt.Println("failure") +} + +func init() { + istep.RegisterCallback(istep.CallbackName(callBackName), &callBack{}) +} diff --git a/task/example/etcd/main.go b/task/example/etcd/main.go new file mode 100644 index 0000000..7432bff --- /dev/null +++ b/task/example/etcd/main.go @@ -0,0 +1,465 @@ +/* + * Tencent is pleased to support the open source community by making Blueking Container Service available. + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * Licensed under the MIT License (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * http://opensource.org/licenses/MIT + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language governing permissions and + * limitations under the License. + */ + +// program xxx defines the etcd main entry. +package main + +import ( + "context" + "errors" + "fmt" + "os" + "time" + + "github.com/RichardKnop/machinery/v2" + "github.com/RichardKnop/machinery/v2/config" + "github.com/RichardKnop/machinery/v2/example/tracers" + "github.com/RichardKnop/machinery/v2/log" + "github.com/RichardKnop/machinery/v2/tasks" + "github.com/google/uuid" + "github.com/urfave/cli" + + etcdbackend "git.ifooth.com/common/pkg/task/backends/etcd" + etcdbroker "git.ifooth.com/common/pkg/task/brokers/etcd" + exampletasks "git.ifooth.com/common/pkg/task/example/tasks" + etcdlock "git.ifooth.com/common/pkg/task/locks/etcd" +) + +var ( + app *cli.App +) + +func init() { + // Initialize a CLI app + app = cli.NewApp() + app.Name = "machinery" + app.Usage = "machinery worker and send example tasks with machinery send" + app.Version = "0.0.0" +} + +func main() { + // Set the CLI app commands + app.Commands = []cli.Command{ + { + Name: "worker", + Usage: "launch machinery worker", + Action: func(c *cli.Context) error { + if err := worker(); err != nil && !errors.Is(err, machinery.ErrWorkerQuitGracefully) { + return cli.NewExitError(err.Error(), 1) + } + return nil + }, + }, + { + Name: "send", + Usage: "send example tasks ", + Action: func(c *cli.Context) error { + if err := send(); err != nil { + return cli.NewExitError(err.Error(), 1) + } + return nil + }, + }, + } + + // Run the CLI app + _ = app.Run(os.Args) +} + +func startServer() (*machinery.Server, error) { + conf := &config.Config{ + DefaultQueue: "machinery_tasks", + Broker: "http://127.0.0.1:2379", + ResultBackend: "http://127.0.0.1:2379", + Lock: "http://127.0.0.1:2379", + ResultsExpireIn: 60, + } + + ctx := context.Background() + // Create server instance + // broker := redisbroker.NewGR(cnf, []string{"localhost:6379"}, 1) + broker, err := etcdbroker.New(ctx, conf) + if err != nil { + return nil, err + } + + // backend := redisbackend.NewGR(cnf, []string{"localhost:6379"}, 3) + backend, err := etcdbackend.New(ctx, conf) + if err != nil { + return nil, err + } + + // lock := redislock.New(cnf, []string{"localhost:6379"}, 3, 2) + lock, err := etcdlock.New(ctx, conf, 3) + if err != nil { + return nil, err + } + server := machinery.NewServer(conf, broker, backend, lock) + + // Register tasks + tasksMap := map[string]interface{}{ + "add": exampletasks.Add, + "multiply": exampletasks.Multiply, + "sum_ints": exampletasks.SumInts, + "sum_floats": exampletasks.SumFloats, + "concat": exampletasks.Concat, + "split": exampletasks.Split, + "panic_task": exampletasks.PanicTask, + "long_running_task": exampletasks.LongRunningTask, + } + + return server, server.RegisterTasks(tasksMap) +} + +func worker() error { + consumerTag := "machinery_worker" + + cleanup, err := tracers.SetupTracer(consumerTag) + if err != nil { + log.FATAL.Fatalln("Unable to instantiate a tracer:", err) + } + defer cleanup() + + server, err := startServer() + if err != nil { + return err + } + + // The second argument is a consumer tag + // Ideally, each worker should have a unique tag (worker1, worker2 etc) + worker := server.NewWorker(consumerTag, 0) + + // Here we inject some custom code for error handling, + // start and end of task hooks, useful for metrics for example. + errorHandler := func(err error) { + log.ERROR.Println("I am an error handler:", err) + } + + preTaskHandler := func(signature *tasks.Signature) { + log.INFO.Println("I am a start of task handler for:", signature.Name) + } + + postTaskHandler := func(signature *tasks.Signature) { + log.INFO.Println("I am an end of task handler for:", signature.Name) + } + + worker.SetPostTaskHandler(postTaskHandler) + worker.SetErrorHandler(errorHandler) + worker.SetPreTaskHandler(preTaskHandler) + + return worker.Launch() +} + +func send() error { // nolint + cleanup, err := tracers.SetupTracer("sender") + if err != nil { + log.FATAL.Fatalln("Unable to instantiate a tracer:", err) + } + defer cleanup() + + server, err := startServer() + if err != nil { + return err + } + + var ( + addTask0, addTask1, addTask2 tasks.Signature + multiplyTask0, multiplyTask1 tasks.Signature + sumIntsTask, sumFloatsTask, concatTask, splitTask tasks.Signature + panicTask tasks.Signature + longRunningTask tasks.Signature + ) + + var initTasks = func() { + addTask0 = tasks.Signature{ + Name: "add", + Args: []tasks.Arg{ + { + Type: "int64", + Value: 1, + }, + { + Type: "int64", + Value: 1, + }, + }, + } + + addTask1 = tasks.Signature{ + Name: "add", + Args: []tasks.Arg{ + { + Type: "int64", + Value: 2, + }, + { + Type: "int64", + Value: 2, + }, + }, + } + + addTask2 = tasks.Signature{ + Name: "add", + Args: []tasks.Arg{ + { + Type: "int64", + Value: 5, + }, + { + Type: "int64", + Value: 6, + }, + }, + } + + multiplyTask0 = tasks.Signature{ + Name: "multiply", + Args: []tasks.Arg{ + { + Type: "int64", + Value: 4, + }, + }, + } + + multiplyTask1 = tasks.Signature{ + Name: "multiply", + } + + sumIntsTask = tasks.Signature{ + Name: "sum_ints", + Args: []tasks.Arg{ + { + Type: "[]int64", + Value: []int64{1, 2}, + }, + }, + } + + sumFloatsTask = tasks.Signature{ + Name: "sum_floats", + Args: []tasks.Arg{ + { + Type: "[]float64", + Value: []float64{1.5, 2.7}, + }, + }, + } + + concatTask = tasks.Signature{ + Name: "concat", + Args: []tasks.Arg{ + { + Type: "[]string", + Value: []string{"foo", "bar"}, + }, + }, + } + + splitTask = tasks.Signature{ + Name: "split", + Args: []tasks.Arg{ + { + Type: "string", + Value: "foo", + }, + }, + } + + panicTask = tasks.Signature{ + Name: "panic_task", + } + + longRunningTask = tasks.Signature{ + Name: "long_running_task", + } + } + + /* + * Lets start a span representing this run of the `send` command and + * set a batch id as baggage so it can travel all the way into + * the worker functions. + */ + + ctx := context.Background() + batchID := uuid.New().String() + log.INFO.Println("Starting batch:", batchID) + /* + * First, let's try sending a single task + */ + initTasks() + + log.INFO.Println("Single task:") + + asyncResult, err := server.SendTaskWithContext(ctx, &addTask0) + if err != nil { + return fmt.Errorf("Could not send task: %s", err.Error()) + } + + results, err := asyncResult.Get(time.Millisecond * 5) + if err != nil { + return fmt.Errorf("Getting task result failed with error: %s", err.Error()) + } + log.INFO.Printf("1 + 1 = %v\n", tasks.HumanReadableResults(results)) + + /* + * Try couple of tasks with a slice argument and slice return value + */ + asyncResult, err = server.SendTaskWithContext(ctx, &sumIntsTask) + if err != nil { + return fmt.Errorf("Could not send task: %s", err.Error()) + } + + results, err = asyncResult.Get(time.Millisecond * 5) + if err != nil { + return fmt.Errorf("Getting task result failed with error: %s", err.Error()) + } + log.INFO.Printf("sum([1, 2]) = %v\n", tasks.HumanReadableResults(results)) + + asyncResult, err = server.SendTaskWithContext(ctx, &sumFloatsTask) + if err != nil { + return fmt.Errorf("Could not send task: %s", err.Error()) + } + + results, err = asyncResult.Get(time.Millisecond * 5) + if err != nil { + return fmt.Errorf("Getting task result failed with error: %s", err.Error()) + } + log.INFO.Printf("sum([1.5, 2.7]) = %v\n", tasks.HumanReadableResults(results)) + + asyncResult, err = server.SendTaskWithContext(ctx, &concatTask) + if err != nil { + return fmt.Errorf("Could not send task: %s", err.Error()) + } + + results, err = asyncResult.Get(time.Millisecond * 5) + if err != nil { + return fmt.Errorf("Getting task result failed with error: %s", err.Error()) + } + log.INFO.Printf("concat([\"foo\", \"bar\"]) = %v\n", tasks.HumanReadableResults(results)) + + asyncResult, err = server.SendTaskWithContext(ctx, &splitTask) + if err != nil { + return fmt.Errorf("Could not send task: %s", err.Error()) + } + + results, err = asyncResult.Get(time.Millisecond * 5) + if err != nil { + return fmt.Errorf("Getting task result failed with error: %s", err.Error()) + } + log.INFO.Printf("split([\"foo\"]) = %v\n", tasks.HumanReadableResults(results)) + + /* + * Now let's explore ways of sending multiple tasks + */ + + // Now let's try a parallel execution + initTasks() + log.INFO.Println("Group of tasks (parallel execution):") + + group, err := tasks.NewGroup(&addTask0, &addTask1, &addTask2) + if err != nil { + return fmt.Errorf("Error creating group: %s", err.Error()) + } + + asyncResults, err := server.SendGroupWithContext(ctx, group, 10) + if err != nil { + return fmt.Errorf("Could not send group: %s", err.Error()) + } + + for _, asyncResult := range asyncResults { + results, err = asyncResult.Get(time.Millisecond * 5) + if err != nil { + return fmt.Errorf("Getting task result failed with error: %s", err.Error()) + } + log.INFO.Printf( + "%v + %v = %v\n", + asyncResult.Signature.Args[0].Value, + asyncResult.Signature.Args[1].Value, + tasks.HumanReadableResults(results), + ) + } + + // Now let's try a group with a chord + initTasks() + log.INFO.Println("Group of tasks with a callback (chord):") + + group, err = tasks.NewGroup(&addTask0, &addTask1, &addTask2) + if err != nil { + return fmt.Errorf("Error creating group: %s", err.Error()) + } + + chord, err := tasks.NewChord(group, &multiplyTask1) + if err != nil { + return fmt.Errorf("Error creating chord: %s", err) + } + + chordAsyncResult, err := server.SendChordWithContext(ctx, chord, 10) + if err != nil { + return fmt.Errorf("Could not send chord: %s", err.Error()) + } + + results, err = chordAsyncResult.Get(time.Millisecond * 5) + if err != nil { + return fmt.Errorf("Getting chord result failed with error: %s", err.Error()) + } + log.INFO.Printf("(1 + 1) * (2 + 2) * (5 + 6) = %v\n", tasks.HumanReadableResults(results)) + + // Now let's try chaining task results + initTasks() + log.INFO.Println("Chain of tasks:") + + chain, err := tasks.NewChain(&addTask0, &addTask1, &addTask2, &multiplyTask0) + if err != nil { + return fmt.Errorf("Error creating chain: %s", err) + } + + chainAsyncResult, err := server.SendChainWithContext(ctx, chain) + if err != nil { + return fmt.Errorf("Could not send chain: %s", err.Error()) + } + + results, err = chainAsyncResult.Get(time.Millisecond * 5) + if err != nil { + return fmt.Errorf("Getting chain result failed with error: %s", err.Error()) + } + log.INFO.Printf("(((1 + 1) + (2 + 2)) + (5 + 6)) * 4 = %v\n", tasks.HumanReadableResults(results)) + + // Let's try a task which throws panic to make sure stack trace is not lost + initTasks() + asyncResult, err = server.SendTaskWithContext(ctx, &panicTask) + if err != nil { + return fmt.Errorf("Could not send task: %s", err.Error()) + } + + _, err = asyncResult.Get(time.Millisecond * 5) + if err == nil { + return errors.New("Error should not be nil if task panicked") + } + log.INFO.Printf("Task panicked and returned error = %v\n", err.Error()) + + // Let's try a long running task + initTasks() + asyncResult, err = server.SendTaskWithContext(ctx, &longRunningTask) + if err != nil { + return fmt.Errorf("Could not send task: %s", err.Error()) + } + + results, err = asyncResult.Get(time.Millisecond * 5) + if err != nil { + return fmt.Errorf("Getting long running task result failed with error: %s", err.Error()) + } + log.INFO.Printf("Long running task returned = %v\n", tasks.HumanReadableResults(results)) + + return nil +} diff --git a/task/example/example_task.go b/task/example/example_task.go new file mode 100644 index 0000000..12b8437 --- /dev/null +++ b/task/example/example_task.go @@ -0,0 +1,92 @@ +/* + * Tencent is pleased to support the open source community by making Blueking Container Service available. + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * Licensed under the MIT License (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * http://opensource.org/licenses/MIT + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package main xxx +package main + +import ( + "fmt" + + istep "git.ifooth.com/common/pkg/task/steps/iface" + "git.ifooth.com/common/pkg/task/types" +) + +/****************************************************************** +************ 构建演示任务 *********** +******************************************************************/ + +var ( + // ExampleTask task + ExampleTask types.TaskName = "测试任务" + // TestTask task for test + TestTask types.TaskType = "TestTask" +) + +// NewExampleTask build example task +func NewExampleTask(a, b string) *Example { + return &Example{ + a: a, + b: b, + } +} + +// Example task +type Example struct { + a string + b string +} + +// Name 任务名称 +func (st *Example) Name() string { + return ExampleTask.String() +} + +// Type 任务类型 +func (st *Example) Type() string { + return TestTask.String() +} + +// Steps 构建任务step +func (st *Example) Steps() []*types.Step { + steps := make([]*types.Step, 0) + + // step1: sum step + step1 := SumStep{}.BuildStep([]istep.KeyValue{ + { + Key: sumA, + Value: st.a, + }, + { + Key: sumB, + Value: st.b, + }, + }, types.WithMaxExecutionSeconds(10)) + + // step2: hello step + step2 := HelloStep{}.BuildStep(nil) + + steps = append(steps, step1, step2) + return steps +} + +// BuildTask build task +func (st *Example) BuildTask(info types.TaskInfo, opts ...types.TaskOption) (*types.Task, error) { + t := types.NewTask(info, opts...) + if len(st.Steps()) == 0 { + return nil, fmt.Errorf("task steps empty") + } + + t.Steps = st.Steps() + t.CurrentStep = t.Steps[0].GetName() + + return t, nil +} diff --git a/task/example/hello_step.go b/task/example/hello_step.go new file mode 100644 index 0000000..f48c938 --- /dev/null +++ b/task/example/hello_step.go @@ -0,0 +1,67 @@ +/* + * Tencent is pleased to support the open source community by making Blueking Container Service available. + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * Licensed under the MIT License (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * http://opensource.org/licenses/MIT + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package main xxx +package main + +import ( + "fmt" + + istep "git.ifooth.com/common/pkg/task/steps/iface" + "git.ifooth.com/common/pkg/task/types" +) + +const ( + stepName = "你好" + method = "hello" +) + +// NewHelloStep hello step +func NewHelloStep() *HelloStep { + return &HelloStep{} +} + +// HelloStep hello step +type HelloStep struct{} + +// Alias stepAlias +func (s HelloStep) Alias() string { + return stepName +} + +// GetName method name +func (s HelloStep) GetName() string { + return method +} + +// Execute for worker exec task +func (s HelloStep) Execute(c *istep.Context) error { + fmt.Printf("%s %s %s\n", c.GetTaskID(), c.GetTaskType(), c.GetTaskName()) + return nil +} + +// BuildStep build step +func (s HelloStep) BuildStep(kvs []istep.KeyValue, opts ...types.StepOption) *types.Step { + step := types.NewStep(s.GetName(), method, opts...) + + // build step paras + for _, v := range kvs { + step.AddParam(v.Key.String(), v.Value) + } + + return step +} + +func init() { + // register step + istep.Register(method, NewHelloStep()) +} diff --git a/task/example/main.go b/task/example/main.go new file mode 100644 index 0000000..bbbb5fa --- /dev/null +++ b/task/example/main.go @@ -0,0 +1,146 @@ +/* + * Tencent is pleased to support the open source community by making Blueking Container Service available. + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * Licensed under the MIT License (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * http://opensource.org/licenses/MIT + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package main xxx +package main + +import ( + "context" + "fmt" + "os" + "os/signal" + "syscall" + "time" + + "git.ifooth.com/common/pkg/task" + etcdbackend "git.ifooth.com/common/pkg/task/backends/etcd" + etcdbroker "git.ifooth.com/common/pkg/task/brokers/etcd" + etcdlock "git.ifooth.com/common/pkg/task/locks/etcd" + istep "git.ifooth.com/common/pkg/task/steps/iface" + mysqlstore "git.ifooth.com/common/pkg/task/stores/mysql" + "git.ifooth.com/common/pkg/task/types" + "github.com/RichardKnop/machinery/v2/config" +) + +/* +场景测试 +1. 正常分发任务并成功执行 +2. 任务执行失败并暂停执行任务 +3. 重试失败任务 / 设置skipOnFailed并重试成功 +4. 任务跳过失败测试 +5. step超时控制 +6. task超时控制 +7. 任务回调机制 +*/ + +var ( + moduleName = "example" + mysqlDSN = "root:%s@tcp(127.0.0.1:3306)/tasks?charset=utf8mb4&parseTime=True&loc=Local" +) + +// nolint +func main() { + pwd := os.Getenv("MYSQL_PASSWORD") + + ctx := context.Background() + + broker, err := etcdbroker.New(ctx, &config.Config{}) + if err != nil { + panic(err) + } + lock, err := etcdlock.New(ctx, &config.Config{}, 3) + if err != nil { + panic(lock) + } + + dns := fmt.Sprintf(mysqlDSN, pwd) + store, err := mysqlstore.New(dns) + if err != nil { + panic(err) + } + err = store.EnsureTable(ctx) + if err != nil { + panic(err) + } + + backend, err := etcdbackend.New(ctx, &config.Config{}) + if err != nil { + panic(err) + } + + btm := task.NewTaskManager() + config := &task.ManagerConfig{ + ModuleName: moduleName, + WorkerNum: 100, + Broker: broker, + Backend: backend, + Lock: lock, + Store: store, + } + + // init task manager + err = btm.Init(config) + if err != nil { + panic(err) + } + + // run task manager + go func() { + _ = btm.Run() + }() + + // wait task server run + time.Sleep(3 * time.Second) + + // build tak && run + sum := NewExampleTask("3", "5") + + info := types.TaskInfo{ + TaskIndex: "example", + TaskType: "example-test", + TaskName: "example", + Creator: "bcs", + } + sumTask, err := sum.BuildTask(info, types.WithTaskMaxExecutionSeconds(0), + types.WithTaskCallback(callBackName)) + if err != nil { + fmt.Println(err) + return + } + err = btm.Dispatch(sumTask) + if err != nil { + fmt.Println(err) + return + } + + // listening OS shutdown singal + signalChan := make(chan os.Signal, 1) + signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM) + <-signalChan + + btm.Stop() + + fmt.Printf("Got OS shutdown signal, shutting down server gracefully...") +} + +// nolint +func registerSteps() []istep.StepExecutor { + steps := make([]istep.StepExecutor, 0) + + sum := NewSumStep() + steps = append(steps, sum) + + hello := NewHelloStep() + steps = append(steps, hello) + + return steps +} diff --git a/task/example/sum_step.go b/task/example/sum_step.go new file mode 100644 index 0000000..d5abca4 --- /dev/null +++ b/task/example/sum_step.go @@ -0,0 +1,84 @@ +/* + * Tencent is pleased to support the open source community by making Blueking Container Service available. + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * Licensed under the MIT License (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * http://opensource.org/licenses/MIT + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package main xxx +package main + +import ( + "fmt" + "strconv" + + istep "git.ifooth.com/common/pkg/task/steps/iface" + "git.ifooth.com/common/pkg/task/types" +) + +const ( + stepSumName = "求和任务" + sumMethod = "sum" +) + +var ( + sumA istep.ParamKey = "sumA" + sumB istep.ParamKey = "sumB" + sumC istep.ParamKey = "sumC" +) + +// NewSumStep sum step +func NewSumStep() *SumStep { + return &SumStep{} +} + +// SumStep sum step +type SumStep struct{} + +// Alias step name +func (s SumStep) Alias() string { + return stepSumName +} + +// GetName step name +func (s SumStep) GetName() string { + return sumMethod +} + +// Execute for worker exec task +func (s SumStep) Execute(c *istep.Context) error { + a, _ := c.GetParam(sumA.String()) + b, _ := c.GetParam(sumB.String()) + + a1, _ := strconv.Atoi(a) + b1, _ := strconv.Atoi(b) + + c1 := a1 + b1 + _ = c.AddCommonParam(sumC.String(), fmt.Sprintf("%v", c1)) + + fmt.Printf("%s %s %s sumC: %v\n", c.GetTaskID(), c.GetTaskType(), c.GetName(), c) + + return nil +} + +// BuildStep build step +func (s SumStep) BuildStep(kvs []istep.KeyValue, opts ...types.StepOption) *types.Step { + step := types.NewStep(s.GetName(), sumMethod, opts...) + + // build step paras + for _, v := range kvs { + step.AddParam(v.Key.String(), v.Value) + } + + return step +} + +func init() { + // register step + istep.Register(sumMethod, NewSumStep()) +} diff --git a/task/example/tasks/tasks.go b/task/example/tasks/tasks.go new file mode 100644 index 0000000..64d96ed --- /dev/null +++ b/task/example/tasks/tasks.go @@ -0,0 +1,88 @@ +/* + * Tencent is pleased to support the open source community by making Blueking Container Service available. + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * Licensed under the MIT License (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * http://opensource.org/licenses/MIT + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package tasks ... +package tasks + +import ( + "errors" + "strings" + "time" + + "github.com/RichardKnop/machinery/v2/log" +) + +// Add ... +func Add(args ...int64) (int64, error) { + sum := int64(0) + for _, arg := range args { + sum += arg + } + return sum, nil +} + +// Multiply ... +func Multiply(args ...int64) (int64, error) { + sum := int64(1) + for _, arg := range args { + sum *= arg + } + return sum, nil +} + +// SumInts ... +func SumInts(numbers []int64) (int64, error) { + var sum int64 + for _, num := range numbers { + sum += num + } + return sum, nil +} + +// SumFloats ... +func SumFloats(numbers []float64) (float64, error) { + var sum float64 + for _, num := range numbers { + sum += num + } + return sum, nil +} + +// Concat ... +func Concat(strs []string) (string, error) { + var res string + for _, s := range strs { + res += s + } + return res, nil +} + +// Split ... +func Split(str string) ([]string, error) { + return strings.Split(str, ""), nil +} + +// PanicTask ... +func PanicTask() (string, error) { + panic(errors.New("oops")) +} + +// LongRunningTask ... +func LongRunningTask() error { + log.INFO.Print("Long running task started") + for i := 0; i < 10; i++ { + log.INFO.Print(10 - i) + time.Sleep(1 * time.Second) + } + log.INFO.Print("Long running task finished") + return nil +} diff --git a/task/go.mod b/task/go.mod new file mode 100644 index 0000000..ae510ba --- /dev/null +++ b/task/go.mod @@ -0,0 +1,84 @@ +module git.ifooth.com/common/pkg/task + +go 1.25 + +require ( + github.com/RichardKnop/machinery/v2 v2.0.16 + github.com/google/uuid v1.6.0 + github.com/prometheus/client_golang v1.20.5 + github.com/stretchr/testify v1.10.0 + github.com/urfave/cli v1.22.17 + go.etcd.io/etcd/api/v3 v3.6.6 + go.etcd.io/etcd/client/v3 v3.6.6 + golang.org/x/sync v0.12.0 + gorm.io/driver/mysql v1.6.0 + gorm.io/gorm v1.31.1 +) + +require ( + cloud.google.com/go v0.75.0 // indirect + cloud.google.com/go/pubsub v1.10.0 // indirect + filippo.io/edwards25519 v1.1.0 // indirect + github.com/RichardKnop/logging v0.0.0-20190827224416-1a693bdd4fae // indirect + github.com/aws/aws-sdk-go-v2 v1.38.1 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 // indirect + github.com/aws/aws-sdk-go-v2/service/dynamodb v1.42.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.10.15 // indirect + github.com/aws/aws-sdk-go-v2/service/sqs v1.38.5 // indirect + github.com/aws/smithy-go v1.22.5 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/coreos/go-semver v0.3.1 // indirect + github.com/coreos/go-systemd/v22 v22.5.0 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.7 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/go-sql-driver/mysql v1.8.1 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/golang/snappy v0.0.4 // indirect + github.com/gomodule/redigo v1.9.2 // indirect + github.com/googleapis/gax-go/v2 v2.0.5 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 // indirect + github.com/jinzhu/inflection v1.0.0 // indirect + github.com/jinzhu/now v1.1.5 // indirect + github.com/jstemmer/go-junit-report v0.9.1 // indirect + github.com/kelseyhightower/envconfig v1.4.0 // indirect + github.com/klauspost/compress v1.17.9 // indirect + github.com/montanaflynn/stats v0.7.1 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/opentracing/opentracing-go v1.2.0 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.62.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect + github.com/rabbitmq/amqp091-go v1.10.0 // indirect + github.com/robfig/cron/v3 v3.0.1 // indirect + github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/xdg-go/pbkdf2 v1.0.0 // indirect + github.com/xdg-go/scram v1.1.2 // indirect + github.com/xdg-go/stringprep v1.0.4 // indirect + github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect + go.etcd.io/etcd/client/pkg/v3 v3.6.6 // indirect + go.mongodb.org/mongo-driver v1.17.0 // indirect + go.opencensus.io v0.22.5 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.27.0 // indirect + golang.org/x/crypto v0.36.0 // indirect + golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5 // indirect + golang.org/x/mod v0.21.0 // indirect + golang.org/x/net v0.38.0 // indirect + golang.org/x/oauth2 v0.27.0 // indirect + golang.org/x/sys v0.31.0 // indirect + golang.org/x/text v0.23.0 // indirect + golang.org/x/tools v0.26.0 // indirect + google.golang.org/api v0.39.0 // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/genproto v0.0.0-20210207032614-bba0dbe2a9ea // indirect + google.golang.org/grpc v1.71.1 // indirect + google.golang.org/protobuf v1.36.5 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/task/go.sum b/task/go.sum new file mode 100644 index 0000000..d8d0517 --- /dev/null +++ b/task/go.sum @@ -0,0 +1,613 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.75.0 h1:XgtDnVJRCPEUG21gjFiRPz4zI1Mjg16R+NYQjfmU4XY= +cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/pubsub v1.10.0 h1:JK22g5uNpscGPthjJE/D0siWtA6UlU4Cb6pLcyJkzyQ= +cloud.google.com/go/pubsub v1.10.0/go.mod h1:eNpTrkOy7dCpkNyaSNetMa6udbgecJMd0ZsTJS/cuNo= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= +filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/toml v1.5.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/RichardKnop/logging v0.0.0-20190827224416-1a693bdd4fae h1:DcFpTQBYQ9Ct2d6sC7ol0/ynxc2pO1cpGUM+f4t5adg= +github.com/RichardKnop/logging v0.0.0-20190827224416-1a693bdd4fae/go.mod h1:rJJ84PyA/Wlmw1hO+xTzV2wsSUon6J5ktg0g8BF2PuU= +github.com/RichardKnop/machinery/v2 v2.0.16 h1:CXMNQHgcQla3D4JpRjK8jYZvd9Pk8B4rO3uhMjmZV8M= +github.com/RichardKnop/machinery/v2 v2.0.16/go.mod h1:LPnvwtB0xEfhmOEI9zVPmHij7AyReyXUtLPIn9u3VG4= +github.com/aws/aws-sdk-go-v2 v1.38.1 h1:j7sc33amE74Rz0M/PoCpsZQ6OunLqys/m5antM0J+Z8= +github.com/aws/aws-sdk-go-v2 v1.38.1/go.mod h1:9Q0OoGQoboYIAJyslFyF1f5K1Ryddop8gqMhWx/n4Wg= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 h1:ZK5jHhnrioRkUNOc+hOgQKlUL5JeC3S6JgLxtQ+Rm0Q= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34/go.mod h1:p4VfIceZokChbA9FzMbRGz5OV+lekcVtHlPKEO0gSZY= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 h1:SZwFm17ZUNNg5Np0ioo/gq8Mn6u9w19Mri8DnJ15Jf0= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34/go.mod h1:dFZsC0BLo346mvKQLWmoJxT+Sjp+qcVR1tRVHQGOH9Q= +github.com/aws/aws-sdk-go-v2/service/dynamodb v1.42.4 h1:5GjCSGIpndYU/tVABz+4XnAcluU6wrjlPzAAgFUDG98= +github.com/aws/aws-sdk-go-v2/service/dynamodb v1.42.4/go.mod h1:yYaWRnVSPyAmexW5t7G3TcuYoalYfT+xQwzWsvtUQ7M= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 h1:eAh2A4b5IzM/lum78bZ590jy36+d/aFLgKF/4Vd1xPE= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3/go.mod h1:0yKJC/kb8sAnmlYa6Zs3QVYqaC8ug2AbnNChv5Ox3uA= +github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.10.15 h1:M1R1rud7HzDrfCdlBQ7NjnRsDNEhXO/vGhuD189Ggmk= +github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.10.15/go.mod h1:uvFKBSq9yMPV4LGAi7N4awn4tLY+hKE35f8THes2mzQ= +github.com/aws/aws-sdk-go-v2/service/sqs v1.38.5 h1:KNgVWw8qbPzjYnIF1gL0EAszy6VKGnmUK6VSm1huYY8= +github.com/aws/aws-sdk-go-v2/service/sqs v1.38.5/go.mod h1:Bar4MrRxeqdn6XIh8JGfiXuFRmyrrsZNTJotxEJmWW0= +github.com/aws/smithy-go v1.22.5 h1:P9ATCXPMb2mPjYBgueqJNCA5S9UfktsW0tTxi+a7eqw= +github.com/aws/smithy-go v1.22.5/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= +github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= +github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= +github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/cpuguy83/go-md2man/v2 v2.0.7 h1:zbFlGlXEAKlwXpmvle3d8Oe3YnkKIK4xSRTd3sHPnBo= +github.com/cpuguy83/go-md2man/v2 v2.0.7/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= +github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/gomodule/redigo v1.9.2 h1:HrutZBLhSIU8abiSfW8pj8mPhOyMYjZT/wcA4/L9L9s= +github.com/gomodule/redigo v1.9.2/go.mod h1:KsU3hiK/Ay8U42qpaJk+kuNa3C+spxapWpM+ywhcgtw= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 h1:5ZPtiqj0JL5oKWmcsq4VMaAW5ukBEgSGXEN89zeH1Jo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3/go.mod h1:ndYquD05frm2vACXE1nsccT4oJzjhw2arTS2cpUD1PI= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= +github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= +github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= +github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE= +github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= +github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= +github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io= +github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw= +github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o= +github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= +github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/urfave/cli v1.22.17 h1:SYzXoiPfQjHBbkYxbew5prZHS1TOLT3ierW8SYLqtVQ= +github.com/urfave/cli v1.22.17/go.mod h1:b0ht0aqgH/6pBYzzxURyrM4xXNgsoT/n2ZzwQiEhNVo= +github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= +github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= +github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= +github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= +github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 h1:ilQV1hzziu+LLM3zUTJ0trRztfwgjqKnBWNtSRkbmwM= +github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfSfmXjznFBSZNN13rSJjlIOI1fUNAtF7rmI= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.etcd.io/etcd/api/v3 v3.6.6 h1:mcaMp3+7JawWv69p6QShYWS8cIWUOl32bFLb6qf8pOQ= +go.etcd.io/etcd/api/v3 v3.6.6/go.mod h1:f/om26iXl2wSkcTA1zGQv8reJRSLVdoEBsi4JdfMrx4= +go.etcd.io/etcd/client/pkg/v3 v3.6.6 h1:uoqgzSOv2H9KlIF5O1Lsd8sW+eMLuV6wzE3q5GJGQNs= +go.etcd.io/etcd/client/pkg/v3 v3.6.6/go.mod h1:YngfUVmvsvOJ2rRgStIyHsKtOt9SZI2aBJrZiWJhCbI= +go.etcd.io/etcd/client/v3 v3.6.6 h1:G5z1wMf5B9SNexoxOHUGBaULurOZPIgGPsW6CN492ec= +go.etcd.io/etcd/client/v3 v3.6.6/go.mod h1:36Qv6baQ07znPR3+n7t+Rk5VHEzVYPvFfGmfF4wBHV8= +go.mongodb.org/mongo-driver v1.17.0 h1:Hp4q2MCjvY19ViwimTs00wHi7G4yzxh4/2+nTx8r40k= +go.mongodb.org/mongo-driver v1.17.0/go.mod h1:wwWm/+BuOddhcq3n68LKRmgk2wXzmF6s0SFOa0GINL4= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5 h1:dntmOdLpSpHlVqbW5Eay97DelsZHe+55D+xC6i0dDS0= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/otel v1.34.0 h1:zRLXxLCgL1WyKsPVrgbSdMN4c0FMkDAskSTQP+0hdUY= +go.opentelemetry.io/otel v1.34.0/go.mod h1:OWFPOQ+h4G8xpyjgqo4SxJYdDQ/qmRH+wivy7zzx9oI= +go.opentelemetry.io/otel/metric v1.34.0 h1:+eTR3U0MyfWjRDhmFMxe2SsW64QrZ84AOhvqS7Y+PoQ= +go.opentelemetry.io/otel/metric v1.34.0/go.mod h1:CEDrp0fy2D0MvkXE+dPV7cMi8tWZwX3dmaIhwPOaqHE= +go.opentelemetry.io/otel/sdk v1.34.0 h1:95zS4k/2GOy069d321O8jWgYsW3MzVV+KuSPKp7Wr1A= +go.opentelemetry.io/otel/sdk v1.34.0/go.mod h1:0e/pNiaMAqaykJGKbi+tSjWfNNHMTxoC9qANsCzbyxU= +go.opentelemetry.io/otel/sdk/metric v1.34.0 h1:5CeK9ujjbFVL5c1PhLuStg1wxA7vQv7ce1EK0Gyvahk= +go.opentelemetry.io/otel/sdk/metric v1.34.0/go.mod h1:jQ/r8Ze28zRKoNRdkjCZxfs6YvBTG1+YIqyFVFYec5w= +go.opentelemetry.io/otel/trace v1.34.0 h1:+ouXS2V8Rd4hp4580a8q23bg0azF2nI8cqLYnC8mh/k= +go.opentelemetry.io/otel/trace v1.34.0/go.mod h1:Svm7lSjQD7kG7KJ/MUHPVXSDGz2OX4h0M2jHBhmSfRE= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= +golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5 h1:2M3HP5CCK1Si9FQhwnzYhXdG6DXeebvUHFpre8QvbyI= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.21.0 h1:vvrHzRwRfVKSiLrG+d4FMl/Qi4ukBCE6kZlTUkDYRT0= +golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= +golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210113205817-d3ed898aa8a3/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.27.0 h1:da9Vo7/tDv5RH/7nZDz1eMGS/q1Vv1N/7FCrBhI9I3M= +golang.org/x/oauth2 v0.27.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= +golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= +golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= +golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ= +golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.39.0 h1:zHCTXf0NeDdKTgcSQpT+ZflWAqHsEp1GmdpxW09f3YM= +google.golang.org/api v0.39.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210207032614-bba0dbe2a9ea h1:N98SvVh7Hdle2lgUVFuIkf0B3u29CUakMUQa7Hwz8Wc= +google.golang.org/genproto v0.0.0-20210207032614-bba0dbe2a9ea/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= +google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.71.1 h1:ffsFWr7ygTUscGPI0KKK6TLrGz0476KUvvsbqWK0rPI= +google.golang.org/grpc v1.71.1/go.mod h1:H0GRtasmQOh9LkFoCPDu3ZrwUtD1YGE+b2vYBYd/8Ec= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= +google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gorm.io/driver/mysql v1.6.0 h1:eNbLmNTpPpTOVZi8MMxCi2aaIm0ZpInbORNXDwyLGvg= +gorm.io/driver/mysql v1.6.0/go.mod h1:D/oCC2GWK3M/dqoLxnOlaNKmXz8WNTfcS9y5ovaSqKo= +gorm.io/gorm v1.31.1 h1:7CA8FTFz/gRfgqgpeKIBcervUn3xSyPUmr6B2WXJ7kg= +gorm.io/gorm v1.31.1/go.mod h1:XyQVbO2k6YkOis7C2437jSit3SsDK72s7n7rsSHd+Gs= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/task/locks/etcd/etcd.go b/task/locks/etcd/etcd.go new file mode 100644 index 0000000..280048c --- /dev/null +++ b/task/locks/etcd/etcd.go @@ -0,0 +1,130 @@ +/* + * Tencent is pleased to support the open source community by making Blueking Container Service available. + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * Licensed under the MIT License (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * http://opensource.org/licenses/MIT + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package etcd implement the lock interface for etcd. +package etcd + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + + "github.com/RichardKnop/machinery/v2/config" + "github.com/RichardKnop/machinery/v2/locks/iface" + "github.com/RichardKnop/machinery/v2/log" + clientv3 "go.etcd.io/etcd/client/v3" + "go.etcd.io/etcd/client/v3/concurrency" +) + +const ( + lockKey = "/machinery/v2/lock/%s" +) + +var ( + // ErrLockFailed .. + ErrLockFailed = errors.New("etcd lock: failed to acquire lock") +) + +type etcdLock struct { + ctx context.Context + client *clientv3.Client + retries int +} + +// New .. +func New(ctx context.Context, conf *config.Config, retries int) (iface.Lock, error) { + etcdConf := clientv3.Config{ + Endpoints: []string{conf.Lock}, + Context: ctx, + DialTimeout: time.Second * 5, + TLS: conf.TLSConfig, + } + + client, err := clientv3.New(etcdConf) + if err != nil { + return nil, err + } + + lock := etcdLock{ + ctx: ctx, + client: client, + retries: retries, + } + + return &lock, nil +} + +// LockWithRetries lock with retries, if TTL is < 1s, the default 1s TTL will be used. +func (l *etcdLock) LockWithRetries(key string, unixTsToExpireNs int64) error { + i := 0 + for ; i < l.retries; i++ { + err := l.Lock(key, unixTsToExpireNs) + if err == nil { + // 成功拿到锁，返回 + return nil + } + + log.DEBUG.Printf("acquired lock=%s failed, retries=%d, err=%s", key, i, err) + time.Sleep(time.Millisecond * 100) + } + + log.INFO.Printf("acquired lock=%s failed, retries=%d", key, i) + return ErrLockFailed +} + +// Lock If TTL is < 1s, the default 1s TTL will be used. +func (l *etcdLock) Lock(key string, unixTsToExpireNs int64) error { + now := time.Now().UnixNano() + expireTTL := time.Duration(unixTsToExpireNs - now) + + // etcd ttl单位是s,往上取整 + ttl := time.Duration(int(expireTTL.Seconds())) * time.Second + if ttl < expireTTL { + ttl += time.Second + } + + // etcd 不能设置小于1s的ttl + if ttl < time.Second { + ttl = time.Second + } + + s, err := concurrency.NewSession(l.client, concurrency.WithTTL(int(ttl.Seconds()))) + if err != nil { + return err + } + defer s.Orphan() + + k := fmt.Sprintf(lockKey, strings.TrimRight(key, "/")) + m := concurrency.NewMutex(s, k) + + ctx, cancel := context.WithTimeout(l.ctx, time.Second*5) + defer cancel() + + // 阻塞等待锁 + if err := m.Lock(ctx); err != nil { + _ = s.Close() + if errors.Is(err, context.DeadlineExceeded) { + return ErrLockFailed + } + return err + } + + log.INFO.Printf("acquired lock=%s, duration=%s", key, ttl) + return nil +} + +// GetLockExpireNs 获取锁的过期时间 +func GetLockExpireNs(duration time.Duration) int64 { + return time.Now().Add(duration).UnixNano() +} diff --git a/task/locks/etcd/etcd_test.go b/task/locks/etcd/etcd_test.go new file mode 100644 index 0000000..6976abf --- /dev/null +++ b/task/locks/etcd/etcd_test.go @@ -0,0 +1,113 @@ +/* + * Tencent is pleased to support the open source community by making Blueking Container Service available. + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * Licensed under the MIT License (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * http://opensource.org/licenses/MIT + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language governing permissions and + * limitations under the License. + */ + +package etcd + +import ( + "context" + "os" + "sync" + "testing" + "time" + + "github.com/RichardKnop/machinery/v2/config" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestLock(t *testing.T) { + endpoints := os.Getenv("ETCDCTL_ENDPOINTS") + if endpoints == "" { + t.Skip("ETCDCTL_ENDPOINTS is not set") + } + t.Parallel() + + locker, err := New(context.Background(), &config.Config{Lock: endpoints}, 3) + require.NoError(t, err) + + lockDuration := time.Second * 10 + err = locker.Lock("test_lock", GetLockExpireNs(lockDuration)) + assert.NoError(t, err) + + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + + st := time.Now() + err = locker.Lock("test_lock", GetLockExpireNs(lockDuration)) + assert.ErrorIs(t, err, ErrLockFailed) + + time.Sleep(time.Second * 5) + err = locker.Lock("test_lock", GetLockExpireNs(lockDuration)) + assert.NoError(t, err) + duration := time.Since(st) + assert.True(t, duration > lockDuration, "lock duration %s should be greater than %s", duration, lockDuration) + }() + wg.Wait() +} + +func TestLockWithRetries(t *testing.T) { + endpoints := os.Getenv("ETCDCTL_ENDPOINTS") + if endpoints == "" { + t.Skip("ETCDCTL_ENDPOINTS is not set") + } + t.Parallel() + + locker, err := New(context.Background(), &config.Config{Lock: endpoints}, 3) + require.NoError(t, err) + + lockDuration := time.Second * 10 + err = locker.Lock("test_retry_lock", GetLockExpireNs(lockDuration)) + assert.NoError(t, err) + st := time.Now() + + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + + err = locker.LockWithRetries("test_retry_lock", GetLockExpireNs(lockDuration)) + assert.NoError(t, err) + duration := time.Since(st) + assert.True(t, duration > lockDuration, "lock duration %s should be greater than %s", duration, lockDuration) + }() + wg.Wait() +} + +func TestLockWithMs(t *testing.T) { + endpoints := os.Getenv("ETCDCTL_ENDPOINTS") + if endpoints == "" { + t.Skip("ETCDCTL_ENDPOINTS is not set") + } + t.Parallel() + + locker, err := New(context.Background(), &config.Config{Lock: endpoints}, 3) + require.NoError(t, err) + + lockDuration := time.Millisecond * 10 + err = locker.Lock("test_retry_lock_ms", GetLockExpireNs(lockDuration)) + assert.NoError(t, err) + + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + + st := time.Now() + err = locker.LockWithRetries("test_retry_lock_ms", GetLockExpireNs(lockDuration)) + assert.NoError(t, err) + duration := time.Since(st) + assert.True(t, duration > lockDuration, "lock duration %s should be greater than %s", duration, lockDuration) + }() + wg.Wait() +} diff --git a/task/manager.go b/task/manager.go new file mode 100644 index 0000000..e290b92 --- /dev/null +++ b/task/manager.go @@ -0,0 +1,461 @@ +/* + * Tencent is pleased to support the open source community by making Blueking Container Service available. + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * Licensed under the MIT License (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * http://opensource.org/licenses/MIT + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package task is a package for task management +package task + +import ( + "context" + "errors" + "fmt" + "runtime/debug" + "sync" + "time" + + "github.com/RichardKnop/machinery/v2" + ibackend "github.com/RichardKnop/machinery/v2/backends/iface" + ibroker "github.com/RichardKnop/machinery/v2/brokers/iface" + "github.com/RichardKnop/machinery/v2/config" + ilock "github.com/RichardKnop/machinery/v2/locks/iface" + "github.com/RichardKnop/machinery/v2/log" + "github.com/RichardKnop/machinery/v2/tasks" + + irevoker "git.ifooth.com/common/pkg/task/revokers/iface" + istep "git.ifooth.com/common/pkg/task/steps/iface" + istore "git.ifooth.com/common/pkg/task/stores/iface" + "git.ifooth.com/common/pkg/task/types" +) + +const ( + // DefaultWorkerConcurrency default worker concurrency + DefaultWorkerConcurrency = 10 +) + +// BrokerConfig config for go-machinery broker + +// TaskManager manager for task server +type TaskManager struct { // nolint + moduleName string + lock sync.Locker + server *machinery.Server + worker *machinery.Worker + + workerNum int + stepExecutors map[istep.StepName]istep.StepExecutor + callbackExecutors map[istep.CallbackName]istep.CallbackExecutor + cfg *ManagerConfig + store istore.Store + + ctx context.Context + cancel context.CancelFunc +} + +// ManagerConfig options for manager +type ManagerConfig struct { + ModuleName string + WorkerName string + WorkerNum int + Broker ibroker.Broker + Revoker irevoker.Revoker + Backend ibackend.Backend + Lock ilock.Lock + Store istore.Store + ServerConfig *config.Config +} + +// NewTaskManager create new manager +func NewTaskManager() *TaskManager { + ctx, cancel := context.WithCancel(context.Background()) + + m := &TaskManager{ + ctx: ctx, + cancel: cancel, + lock: &sync.Mutex{}, + workerNum: DefaultWorkerConcurrency, + stepExecutors: istep.GetRegisters(), // get all step workers + cfg: &ManagerConfig{}, + } + return m +} + +// Init init machinery server and worker +func (m *TaskManager) Init(cfg *ManagerConfig) error { + err := m.validate(cfg) + if err != nil { + return err + } + + if cfg.ServerConfig == nil { + cfg.ServerConfig = &config.Config{ + ResultsExpireIn: 3600 * 48, + NoUnixSignals: true, + } + } + m.cfg = cfg + m.store = cfg.Store + + if m.stepExecutors == nil { + m.stepExecutors = make(map[istep.StepName]istep.StepExecutor) + } + + m.callbackExecutors = istep.GetCallbackRegisters() + + m.moduleName = cfg.ModuleName + if cfg.WorkerNum != 0 { + m.workerNum = cfg.WorkerNum + } + + if err := m.initGlobalStorage(); err != nil { + return err + } + + if err := m.initServer(); err != nil { + return err + } + + if err := m.initWorker(cfg.WorkerName, cfg.WorkerNum); err != nil { + return err + } + + return nil +} + +func (m *TaskManager) initGlobalStorage() error { + globalStorage = m.store + return nil +} + +func (m *TaskManager) validate(c *ManagerConfig) error { + // module name check + if c.ModuleName == "" { + return fmt.Errorf("module name is empty") + } + + return nil +} + +func (m *TaskManager) initServer() error { + m.server = machinery.NewServer(m.cfg.ServerConfig, m.cfg.Broker, m.cfg.Backend, m.cfg.Lock) + + return nil +} + +// register step workers and init workers +func (m *TaskManager) initWorker(workerName string, workerNum int) error { + // register all workers + if err := m.registerStepWorkers(); err != nil { + return fmt.Errorf("register workers failed, err: %s", err.Error()) + } + + m.worker = m.server.NewWorker(workerName, workerNum) + + preTaskHandler := func(signature *tasks.Signature) { + log.INFO.Printf("start task[%s] handler for: %s", signature.UUID, signature.Name) + } + postTaskHandler := func(signature *tasks.Signature) { + log.INFO.Printf("end task[%s] handler for: %s", signature.UUID, signature.Name) + } + errorHandler := func(err error) { + log.INFO.Printf("task error handler: %s", err) + } + + m.worker.SetPreTaskHandler(preTaskHandler) + m.worker.SetPostTaskHandler(postTaskHandler) + m.worker.SetErrorHandler(errorHandler) + + return nil +} + +// Run start worker +func (m *TaskManager) Run() error { + return m.worker.Launch() +} + +// GetTaskWithID get task by taskid +func (m *TaskManager) GetTaskWithID(ctx context.Context, taskId string) (*types.Task, error) { + return GetGlobalStorage().GetTask(ctx, taskId) +} + +// ListTask list tasks with options, returns a paginated list of tasks +func (m *TaskManager) ListTask(ctx context.Context, opt *istore.ListOption) (*istore.Pagination[types.Task], error) { + return GetGlobalStorage().ListTask(ctx, opt) +} + +// UpdateTask update task +// ! warning: modify task status will cause task status not consistent +func (m *TaskManager) UpdateTask(ctx context.Context, task *types.Task) error { + return GetGlobalStorage().UpdateTask(ctx, task) +} + +// RetryAll reset status to running and dispatch all tasks +func (m *TaskManager) RetryAll(task *types.Task) error { + task.SetStatus(types.TaskStatusRunning) + task.SetMessage("task retrying") + + if err := GetGlobalStorage().UpdateTask(context.Background(), task); err != nil { + return err + } + return m.dispatchAt(task, "") +} + +// RetryAt reset status to running and dispatch tasks which begin with stepName +func (m *TaskManager) RetryAt(task *types.Task, stepName string) error { + task.SetStatus(types.TaskStatusRunning) + task.SetMessage("task retrying") + + if err := GetGlobalStorage().UpdateTask(context.Background(), task); err != nil { + return err + } + return m.dispatchAt(task, stepName) +} + +// Revoke revoke the task +func (m *TaskManager) Revoke(task *types.Task) error { + // task revoke + if m.cfg == nil || m.cfg.Revoker == nil { + return fmt.Errorf("task revoker is required") + } + + task.SetStatus(types.TaskStatusRevoked) + task.SetMessage("task has been revoked") + + if err := GetGlobalStorage().UpdateTask(context.Background(), task); err != nil { + return err + } + return m.cfg.Revoker.Revoke(context.Background(), task.TaskID) +} + +// Dispatch dispatch task +func (m *TaskManager) Dispatch(task *types.Task) error { + if err := task.Validate(); err != nil { + return err + } + + if err := GetGlobalStorage().CreateTask(context.Background(), task); err != nil { + return err + } + + return m.dispatchAt(task, "") +} + +func (m *TaskManager) transTaskToSignature(task *types.Task, stepNameBegin string) []*tasks.Signature { + var signatures []*tasks.Signature + + for _, step := range task.Steps { + // skip steps which before begin step, empty str not skip any steps + if step.Name != "" && stepNameBegin != "" && step.Name != stepNameBegin { + continue + } + + // build signature from step + signature := &tasks.Signature{ + UUID: fmt.Sprintf("%s-%s", task.TaskID, step.Name), + Name: step.Executor, + ETA: step.ETA, + // two parameters: taskID, stepName + Args: []tasks.Arg{ + { + Name: "task_id", + Type: "string", + Value: task.GetTaskID(), + }, + { + Name: "step_name", + Type: "string", + Value: step.Name, + }, + }, + + IgnoreWhenTaskNotRegistered: true, + } + + signatures = append(signatures, signature) + } + + return signatures +} + +// dispatchAt task to machinery +func (m *TaskManager) dispatchAt(task *types.Task, stepNameBegin string) error { + signatures := m.transTaskToSignature(task, stepNameBegin) + + m.lock.Lock() + defer m.lock.Unlock() + + // sending to workers + chain, err := tasks.NewChain(signatures...) + if err != nil { + log.ERROR.Printf("taskManager[%s] DispatchChainTask NewChain failed: %v", task.GetTaskID(), err) + return err + } + + // send chain to machinery & ctx for tracing + _, err = m.server.SendChainWithContext(context.Background(), chain) + if err != nil { + return fmt.Errorf("send chain to machinery failed: %s", err.Error()) + } + + return nil +} + +// registerStepWorkers build machinery workers for all step worker +func (m *TaskManager) registerStepWorkers() error { + allTasks := make(map[string]interface{}, 0) + for stepName := range m.stepExecutors { + name := string(stepName) + if _, ok := allTasks[name]; ok { + return fmt.Errorf("task %s already exists", name) + } + allTasks[name] = m.doWork + } + err := m.server.RegisterTasks(allTasks) + return err +} + +// doWork machinery 通用处理函数 +func (m *TaskManager) doWork(taskID string, stepName string) error { // nolint + defer RecoverPrintStack(fmt.Sprintf("%s-%s", taskID, stepName)) + + log.INFO.Printf("start to execute task[%s] stepName[%s]", taskID, stepName) + + state, err := m.getTaskState(taskID, stepName) + if err != nil { + log.ERROR.Printf("task[%s] stepName[%s] getTaskState failed: %v", + taskID, stepName, err) + return err + } + + // step executed success + if state.step == nil { + log.INFO.Printf("task[%s] stepName[%s] already exec successful && skip", taskID, stepName) + return nil + } + + step := state.step + stepExecutor, ok := m.stepExecutors[istep.StepName(step.Executor)] + if !ok { + log.ERROR.Printf("task[%s] stepName[%s] executor[%s] not found", taskID, stepName, state.step.Executor) + return fmt.Errorf("step executor[%s] not found", step.Executor) + } + + start := time.Now() + + // metrics + collectMetricStart(state) + defer collectMetricEnd(state) + + // step timeout + stepCtx, stepCancel := GetTimeOutCtx(context.Background(), step.MaxExecutionSeconds) + defer stepCancel() + + // task revoke + revokeCtx := context.TODO() + if m.cfg != nil && m.cfg.Revoker != nil { + revokeCtx = m.cfg.Revoker.RevokeCtx(taskID) + } + + // task timeout + t := state.task.GetStartTime() + taskCtx, taskCancel := GetDeadlineCtx(context.Background(), &t, state.task.MaxExecutionSeconds) + defer taskCancel() + + tmpCh := make(chan error, 1) + go func() { + defer func() { + if r := recover(); r != nil { + log.ERROR.Printf("[%s-%s][recover] panic: %v, stack %s", taskID, stepName, r, debug.Stack()) + tmpCh <- fmt.Errorf("%w by a panic: %v", istep.ErrRevoked, r) + } + }() + + // call step worker + execCtx := istep.NewContext(stepCtx, GetGlobalStorage(), state.GetTask(), step) + tmpCh <- stepExecutor.Execute(execCtx) + }() + + select { + case stepErr := <-tmpCh: + log.INFO.Printf("task %s step %s exec done, duration=%s, err=%v", + taskID, stepName, time.Since(start), stepErr) + + // update task & step status + if stepErr == nil { + state.updateStepSuccess(start) + return nil + } + state.updateStepFailure(start, stepErr, nil) + + // 单步骤主动revoke或者没有重试次数时, 不再重试 + if !errors.Is(stepErr, istep.ErrRevoked) && step.GetRetryCount() < step.MaxRetries { + retryIn := time.Second * time.Duration(retryNext(int(step.GetRetryCount()))) + log.INFO.Printf("retry task %s step %s, err=%s, retried=%d, maxRetries=%d, retryIn=%s", + taskID, stepName, stepErr, step.GetRetryCount(), step.MaxRetries, retryIn) + return tasks.NewErrRetryTaskLater(stepErr.Error(), retryIn) + } + + if step.GetSkipOnFailed() { + return nil + } + + retErr := fmt.Errorf("task %s step %s running failed, err=%w", taskID, stepName, stepErr) + return retErr + + case <-stepCtx.Done(): + // step timeout + stepErr := fmt.Errorf("step exec timeout") + state.updateStepFailure(start, stepErr, nil) + + if step.GetRetryCount() < step.MaxRetries { + retryIn := time.Second * time.Duration(retryNext(int(step.GetRetryCount()))) + log.INFO.Printf("retry task %s step %s, err=%s, retried=%d, maxRetries=%d, retryIn=%s", + taskID, stepName, stepErr, step.GetRetryCount(), step.MaxRetries, retryIn) + return tasks.NewErrRetryTaskLater(stepErr.Error(), retryIn) + } + + if step.GetSkipOnFailed() { + return nil + } + + retErr := fmt.Errorf("task %s step %s running failed, err=%w", taskID, stepName, stepErr) + return retErr + + case <-revokeCtx.Done(): + // task revoke + stepErr := fmt.Errorf("task has been revoked") + state.updateStepFailure(start, stepErr, &taskEndStatus{status: types.TaskStatusRevoked}) + + // 取消指令, 不再重试 + retErr := fmt.Errorf("task %s step %s running failed, err=%w", taskID, stepName, stepErr) + return retErr + + case <-taskCtx.Done(): + // task timeout + stepErr := fmt.Errorf("task exec timeout") + state.updateStepFailure(start, stepErr, &taskEndStatus{status: types.TaskStatusTimeout}) + + // 整个任务结束 + retErr := fmt.Errorf("task %s step %s running failed, err=%w", taskID, stepName, stepErr) + return retErr + + case <-m.ctx.Done(): + // task manager stop, try later + log.INFO.Printf("task manager stop, task %s step %s will retry later", taskID, stepName) + return tasks.NewErrRetryTaskLater("task manager stop", time.Second*10) + } +} + +// Stop running +func (m *TaskManager) Stop() { + // should set NoUnixSignals + m.worker.Quit() + m.cancel() +} diff --git a/task/manager_test.go b/task/manager_test.go new file mode 100644 index 0000000..2d4ba8e --- /dev/null +++ b/task/manager_test.go @@ -0,0 +1,110 @@ +/* + * Tencent is pleased to support the open source community by making Blueking Container Service available. + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * Licensed under the MIT License (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * http://opensource.org/licenses/MIT + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language governing permissions and + * limitations under the License. + */ + +package task + +import ( + "context" + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + hellostep "git.ifooth.com/common/pkg/task/steps/hello" + istep "git.ifooth.com/common/pkg/task/steps/iface" + "git.ifooth.com/common/pkg/task/stores/mem" + mysqlstore "git.ifooth.com/common/pkg/task/stores/mysql" + "git.ifooth.com/common/pkg/task/types" +) + +func TestDoWork(t *testing.T) { + // 使用结构体注册 + // istep.Register("hello", hellostep.NewHello()) + + // 使用函数注册 + // istep.Register("sum", istep.StepWorkerFunc(hellostep.Sum)) + + mgr := TaskManager{ + ctx: context.Background(), + store: mem.New(), + stepExecutors: istep.GetRegisters(), + } + mgr.initGlobalStorage() + + info := types.TaskInfo{ + TaskType: "example-test", + TaskName: "example", + Creator: "bcs", + } + + steps := []*types.Step{ + types.NewStep("test", "hello"), + types.NewStep("test1", "sum").AddParam(hellostep.SumA.String(), "1").AddParam(hellostep.SumB.String(), "2"), + } + + task := types.NewTask(info) + task.Steps = steps + + require.NoError(t, GetGlobalStorage().CreateTask(context.Background(), task)) + + for _, s := range steps { + err := mgr.doWork(task.TaskID, s.Name) + assert.NoError(t, err) + } +} + +func TestDoWorkWithMySQL(t *testing.T) { + if os.Getenv("MYSQL_DSN") == "" { + t.Skip("skip test without mysql dsn") + } + + // 使用结构体注册 + istep.Register("hello", hellostep.NewHello()) + + // 使用函数注册 + istep.Register("sum", istep.StepExecutorFunc(hellostep.Sum)) + + store, err := mysqlstore.New(os.Getenv("MYSQL_DSN")) + require.NoError(t, err) + + ctx := context.Background() + require.NoError(t, store.EnsureTable(ctx)) + + mgr := TaskManager{ + ctx: context.Background(), + store: store, + stepExecutors: istep.GetRegisters(), + } + mgr.initGlobalStorage() + + info := types.TaskInfo{ + TaskType: "example-test", + TaskName: "example", + Creator: "bcs", + } + + steps := []*types.Step{ + types.NewStep("test", "hello"), + types.NewStep("test1", "sum").AddParam(hellostep.SumA.String(), "1").AddParam(hellostep.SumB.String(), "2"), + } + + task := types.NewTask(info) + task.Steps = steps + + require.NoError(t, GetGlobalStorage().CreateTask(context.Background(), task)) + + for _, s := range steps { + err := mgr.doWork(task.TaskID, s.Name) + assert.NoError(t, err) + } +} diff --git a/task/metrics.go b/task/metrics.go new file mode 100644 index 0000000..ce45efe --- /dev/null +++ b/task/metrics.go @@ -0,0 +1,97 @@ +/* + * Tencent is pleased to support the open source community by making Blueking Container Service available. + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * Licensed under the MIT License (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * http://opensource.org/licenses/MIT + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language governing permissions and + * limitations under the License. + */ + +package task + +import ( + "github.com/prometheus/client_golang/prometheus" + + "git.ifooth.com/common/pkg/task/types" +) + +var ( + // 当前step执行数量 + stepRunningCount = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "step_running_count", + Help: "The number of running step.", + }, []string{"task_type", "executor"}) + + // step执行总数 + stepExecuteTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "step_execute_total", + Help: "Counter of step execute count.", + }, []string{"task_type", "executor", "status"}) + + // step执行耗时 + stepExecuteDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Name: "step_execute_duration_seconds", + Help: "Histogram of duration for step execute.", + Buckets: []float64{1, 10, 30, 60, 60 * 5, 60 * 10, 60 * 30, 3600, 3600 * 2, 3600 * 4, 3600 * 8}, + }, []string{"task_type", "executor", "status"}) + + // task执行总数 + taskExecuteTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "task_execute_total", + Help: "Counter of task execute.", + }, []string{"task_type", "status"}) + + // task执行耗时 + taskExecuteDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Name: "task_execute_duration_seconds", + Help: "Histogram of duration for task execute.", + Buckets: []float64{1, 10, 30, 60, 60 * 5, 60 * 10, 60 * 30, 3600, 3600 * 2, 3600 * 4, 3600 * 8}, + }, []string{"task_type", "status"}) +) + +func init() { + prometheus.MustRegister(stepRunningCount) + prometheus.MustRegister(stepExecuteTotal) + prometheus.MustRegister(stepExecuteDuration) + prometheus.MustRegister(taskExecuteTotal) + prometheus.MustRegister(taskExecuteDuration) +} + +// collectMetricStart metrics for task start +func collectMetricStart(state *State) { + stepRunningCount.WithLabelValues( + state.task.GetTaskType(), + state.step.Executor).Inc() +} + +// collectMetricEnd metrics for task end +func collectMetricEnd(state *State) { + // 任务状态完成时, 记录执行结果 + if state.task.GetStatus() != types.TaskStatusInit && state.task.GetStatus() != types.TaskStatusRunning { + taskExecuteTotal.WithLabelValues( + state.task.GetTaskType(), + state.task.GetStatus()).Inc() + + taskExecuteDuration.WithLabelValues( + state.task.GetTaskType(), + state.task.GetStatus()).Observe(state.task.GetExecutionTime().Seconds()) + } + + // 任务步骤完成时, 记录执行结果 + stepRunningCount.WithLabelValues( + state.task.GetTaskType(), + state.step.Executor).Dec() + + stepExecuteTotal.WithLabelValues( + state.task.GetTaskType(), + state.step.Executor, + state.step.GetStatus()).Inc() + + stepExecuteDuration.WithLabelValues( + state.task.GetTaskType(), + state.step.Executor, + state.step.GetStatus()).Observe(state.step.GetExecutionTime().Seconds()) +} diff --git a/task/revokers/etcd/etcd.go b/task/revokers/etcd/etcd.go new file mode 100644 index 0000000..1125bef --- /dev/null +++ b/task/revokers/etcd/etcd.go @@ -0,0 +1,100 @@ +/* + * Tencent is pleased to support the open source community by making Blueking Container Service available. + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * Licensed under the MIT License (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * http://opensource.org/licenses/MIT + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package etcd is revoker use etcd +package etcd + +import ( + "context" + "sync" + "time" + + "github.com/RichardKnop/machinery/v2/config" + "github.com/RichardKnop/machinery/v2/log" + clientv3 "go.etcd.io/etcd/client/v3" + + "git.ifooth.com/common/pkg/task/revokers/iface" +) + +type etcdRevoker struct { + ctx context.Context + client *clientv3.Client + mtx sync.Mutex + wg sync.WaitGroup + revokeSignMap map[string]*revokeSign +} + +// New .. +func New(ctx context.Context, conf *config.Config) (iface.Revoker, error) { + etcdConf := clientv3.Config{ + Endpoints: []string{conf.Broker}, // 复用broker的etcd配置 + Context: ctx, + DialTimeout: time.Second * 5, + TLS: conf.TLSConfig, + } + + client, err := clientv3.New(etcdConf) + if err != nil { + return nil, err + } + + revoker := etcdRevoker{ + ctx: ctx, + client: client, + revokeSignMap: map[string]*revokeSign{}, + } + + go revoker.Run() + + return &revoker, nil +} + +func (r *etcdRevoker) Run() { + // list and watch revoke sign + r.wg.Add(1) + go func() { + defer r.wg.Done() + + for { + select { + case <-r.ctx.Done(): + return + default: + err := r.listWatchRevoke(r.ctx) + if err != nil { + log.ERROR.Printf("list and watch revoke failed, err: %s", err) + time.Sleep(time.Second) + } + } + } + }() + + // cleanup revoke sign + r.wg.Add(1) + go func() { + defer r.wg.Done() + + ticker := time.NewTicker(time.Minute * 10) + defer ticker.Stop() + + for { + select { + case <-r.ctx.Done(): + return + case <-ticker.C: + r.cleanupRevokeSign() + } + } + }() + + r.wg.Wait() +} diff --git a/task/revokers/etcd/revoke.go b/task/revokers/etcd/revoke.go new file mode 100644 index 0000000..cd6a9e4 --- /dev/null +++ b/task/revokers/etcd/revoke.go @@ -0,0 +1,153 @@ +/* + * Tencent is pleased to support the open source community by making Blueking Container Service available. + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * Licensed under the MIT License (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * http://opensource.org/licenses/MIT + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language governing permissions and + * limitations under the License. + */ + +package etcd + +import ( + "context" + "path/filepath" + "time" + + "github.com/RichardKnop/machinery/v2/log" + "go.etcd.io/etcd/api/v3/mvccpb" + clientv3 "go.etcd.io/etcd/client/v3" +) + +const ( + revokePrefix = "/machinery/v2/revoker/tasks" +) + +type revokeSign struct { + taskID string + registerTime time.Time + ctx context.Context + cancel context.CancelFunc +} + +// Revoke etcd revoker send sign +func (b *etcdRevoker) Revoke(ctx context.Context, taskID string) error { + key := revokePrefix + "/" + taskID + + // 2分钟自动过期 + lease, err := b.client.Grant(ctx, 120) + if err != nil { + return err + } + + _, err = b.client.Put(ctx, key, time.Now().Format(time.RFC3339), clientv3.WithLease(lease.ID)) + if err != nil { + return err + } + + return nil +} + +// RevokeCtx etcd revoker ctx +func (b *etcdRevoker) RevokeCtx(taskID string) context.Context { + b.mtx.Lock() + defer b.mtx.Unlock() + + sign, ok := b.revokeSignMap[taskID] + if ok { + sign.registerTime = time.Now() + return sign.ctx + } + + ctx, cancel := context.WithCancel(context.Background()) + sign = &revokeSign{ + taskID: taskID, + registerTime: time.Now(), + ctx: ctx, + cancel: cancel, + } + b.revokeSignMap[taskID] = sign + + return sign.ctx +} + +func (b *etcdRevoker) tryRevoke(kv *mvccpb.KeyValue) { + key := string(kv.Key) + taskID := filepath.Base(key) + + b.mtx.Lock() + sign, ok := b.revokeSignMap[taskID] + if ok { + sign.cancel() + delete(b.revokeSignMap, taskID) + } + b.mtx.Unlock() + + if !ok { + return + } + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + + _, err := b.client.Delete(ctx, key) + if err != nil { + log.ERROR.Printf("revoke %s failed: %s", key, err) + } +} + +func (b *etcdRevoker) listWatchRevoke(ctx context.Context) error { + // List + listCtx, listCancel := context.WithTimeout(ctx, time.Second*10) + defer listCancel() + + resp, err := b.client.Get(listCtx, revokePrefix, clientv3.WithPrefix(), clientv3.WithKeysOnly()) + if err != nil { + return err + } + + for _, kv := range resp.Kvs { + b.tryRevoke(kv) + } + + // Watch + watchCtx, watchCancel := context.WithTimeout(ctx, time.Minute*60) + defer watchCancel() + + watchOpts := []clientv3.OpOption{ + clientv3.WithPrefix(), + clientv3.WithKeysOnly(), + clientv3.WithRev(resp.Header.Revision), + } + wc := b.client.Watch(watchCtx, revokePrefix, watchOpts...) + for wresp := range wc { + if wresp.Err() != nil { + return wresp.Err() + } + + for _, ev := range wresp.Events { + if ev.Type != clientv3.EventTypePut { + continue + } + + b.tryRevoke(ev.Kv) + } + } + + return nil +} + +func (b *etcdRevoker) cleanupRevokeSign() { + b.mtx.Lock() + defer b.mtx.Unlock() + + for taskID, sign := range b.revokeSignMap { + if time.Since(sign.registerTime) > time.Hour*24*3 { + sign.cancel() + delete(b.revokeSignMap, taskID) + } + } +} diff --git a/task/revokers/iface/interfaces.go b/task/revokers/iface/interfaces.go new file mode 100644 index 0000000..2ffc5b1 --- /dev/null +++ b/task/revokers/iface/interfaces.go @@ -0,0 +1,24 @@ +/* + * Tencent is pleased to support the open source community by making Blueking Container Service available. + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * Licensed under the MIT License (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * http://opensource.org/licenses/MIT + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package iface defines the interface for a task revocation +package iface + +import ( + "context" +) + +// Revoker is an interface for task revocation +type Revoker interface { + Revoke(ctx context.Context, taskID string) error + RevokeCtx(taskID string) context.Context +} diff --git a/task/state.go b/task/state.go new file mode 100644 index 0000000..42933af --- /dev/null +++ b/task/state.go @@ -0,0 +1,336 @@ +/* + * Tencent is pleased to support the open source community by making Blueking Container Service available. + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * Licensed under the MIT License (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * http://opensource.org/licenses/MIT + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language governing permissions and + * limitations under the License. + */ + +package task + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/RichardKnop/machinery/v2/log" + + istep "git.ifooth.com/common/pkg/task/steps/iface" + "git.ifooth.com/common/pkg/task/types" +) + +// taskEndStatus task结束状态,处理超时和revoke +type taskEndStatus struct { + status string + messsage string +} + +// getTaskStateAndCurrentStep get task state and current step +func (m *TaskManager) getTaskState(taskId, stepName string) (*State, error) { + task, err := GetGlobalStorage().GetTask(context.Background(), taskId) + if err != nil { + return nil, fmt.Errorf("get task %s information failed, %s", taskId, err.Error()) + } + + if task.CommonParams == nil { + task.CommonParams = make(map[string]string, 0) + } + + state := NewState(task, stepName) + if state.isTaskTerminated() { + return nil, fmt.Errorf("task %s is terminated, step %s skip", taskId, stepName) + } + step, err := state.isReadyToStep(stepName) + if err != nil { + return nil, fmt.Errorf("task %s step %s is not ready, %w", taskId, stepName, err) + } + + if step == nil { + // step successful and skip + log.INFO.Printf("task %s step %s already execute successful", taskId, stepName) + return state, nil + } + state.step = step + + // inject call back func + if state.task.GetCallback() != "" && len(m.callbackExecutors) > 0 { + name := istep.CallbackName(state.task.GetCallback()) + if cbExecutor, ok := m.callbackExecutors[name]; ok { + state.cbExecutor = cbExecutor + } else { + log.WARNING.Println("task %s callback %s not registered, just ignore", taskId, name) + } + } + + return state, nil +} + +// State is a struct for task state +type State struct { + task *types.Task + step *types.Step + stepName string + cbExecutor istep.CallbackExecutor +} + +// NewState return state relative to task +func NewState(task *types.Task, stepName string) *State { + return &State{ + task: task, + stepName: stepName, + } +} + +// isTaskTerminated is terminated +func (s *State) isTaskTerminated() bool { + status := s.task.GetStatus() + if status == types.TaskStatusFailure || + status == types.TaskStatusSuccess || + status == types.TaskStatusRevoked || + status == types.TaskStatusTimeout { + return true + } + return false +} + +// isReadyToStep check if step is ready to step +func (s *State) isReadyToStep(stepName string) (*types.Step, error) { + nowTime := time.Now() + + switch s.task.GetStatus() { + case types.TaskStatusInit: + s.task.SetStartTime(nowTime) + case types.TaskStatusRunning: + default: + return nil, fmt.Errorf("task %s is not running, state is %s", s.task.GetTaskID(), s.task.GetStatus()) + } + + // validate step existence + curStep, ok := s.task.GetStep(stepName) + if !ok { + return nil, fmt.Errorf("step %s is not exist", stepName) + } + s.task.SetCurrentStep(stepName).SetLastUpdate(nowTime) + + defer func() { + // update Task in storage + if err := GetGlobalStorage().UpdateTask(context.Background(), s.task); err != nil { + log.ERROR.Printf("task %s update step %s failed: %s", s.task.TaskID, curStep.GetName(), err.Error()) + } + }() + + // return nil & nil means step had been executed + // if task retring and so on, shoud update task status and ignore callback because task actually not execute + if curStep.IsCompleted() { + // task success + taskStartTime := s.task.GetStartTime() + if curStep.GetStatus() == types.TaskStatusSuccess { + if s.isLastStep(curStep) { + s.task.SetEndTime(nowTime). + SetExecutionTime(taskStartTime, nowTime). + SetStatus(types.TaskStatusSuccess). + SetMessage("task finished successfully") + } + // step is success, skip + return nil, nil + } + + // task failed + failMsg := fmt.Sprintf("step %s running failed", curStep.Name) + if s.isLastStep(curStep) { + if curStep.GetSkipOnFailed() { + s.task.SetEndTime(nowTime). + SetExecutionTime(taskStartTime, nowTime). + SetStatus(types.TaskStatusSuccess). + SetMessage("task finished successfully") + return nil, nil + } + + s.task.SetEndTime(nowTime). + SetExecutionTime(taskStartTime, nowTime). + SetStatus(types.TaskStatusFailure). + SetMessage(failMsg) + return nil, fmt.Errorf(failMsg) + } + + if curStep.GetSkipOnFailed() { + return nil, nil + } + + s.task.SetEndTime(nowTime). + SetExecutionTime(taskStartTime, nowTime). + SetStatus(types.TaskStatusFailure). + SetMessage(failMsg) + return nil, fmt.Errorf(failMsg) + } + + // not first time to execute current step + if curStep.GetStatus() == types.TaskStatusFailure { + curStep.AddRetryCount(1) + } + + curStep = curStep.SetStartTime(nowTime). + SetStatus(types.TaskStatusRunning). + SetMessage("step ready to run"). + SetLastUpdate(nowTime) + + s.task.SetStatus(types.TaskStatusRunning).SetMessage("task running") + return curStep, nil +} + +// updateStepSuccess update step status to success +func (s *State) updateStepSuccess(start time.Time) { + endTime := time.Now() + + defer func() { + // update Task in storage + if err := GetGlobalStorage().UpdateTask(context.Background(), s.task); err != nil { + log.ERROR.Printf("task %s update step %s to success failed: %s", s.task.TaskID, s.step.GetName(), err.Error()) + } + }() + + s.step.SetEndTime(endTime). + SetExecutionTime(start, endTime). + SetStatus(types.TaskStatusSuccess). + SetMessage(fmt.Sprintf("step %s running successfully", s.step.Name)). + SetLastUpdate(endTime) + + taskStartTime := s.task.GetStartTime() + s.task.SetStatus(types.TaskStatusRunning). + SetExecutionTime(taskStartTime, endTime). + SetMessage(fmt.Sprintf("step %s running successfully", s.step.Name)). + SetLastUpdate(endTime) + + // last step + if s.isLastStep(s.step) { + s.task.SetEndTime(endTime). + SetStatus(types.TaskStatusSuccess). + SetMessage("task finished successfully") + + // callback + if s.cbExecutor != nil { + c := istep.NewContext(context.Background(), GetGlobalStorage(), s.task, s.step) + s.cbExecutor.Callback(c, nil) + } + } +} + +// updateStepFailure update step status to failure +func (s *State) updateStepFailure(start time.Time, stepErr error, taskStatus *taskEndStatus) { + defer func() { + // update Task in storage + if err := GetGlobalStorage().UpdateTask(context.Background(), s.task); err != nil { + log.ERROR.Printf("task %s update step %s to failure failed: %s", s.task.TaskID, s.step.GetName(), err.Error()) + } + }() + + endTime := time.Now() + + stepFailMsg := fmt.Sprintf("running failed, err=%s", stepErr) + taskFailMsg := fmt.Sprintf("step %s running failed, err=%s", s.step.Name, stepErr) + if s.step.MaxRetries > 0 { + stepFailMsg = fmt.Sprintf("running failed, err=%s, retried=%d, maxRetries=%d", + stepErr, s.step.GetRetryCount(), s.step.MaxRetries) + taskFailMsg = fmt.Sprintf("step %s running failed, err=%s, retried=%d, maxRetries=%d", + s.step.Name, stepErr, s.step.GetRetryCount(), s.step.MaxRetries) + } + + s.step.SetEndTime(endTime). + SetExecutionTime(start, endTime). + SetStatus(types.TaskStatusFailure). + SetMessage(stepFailMsg). + SetLastUpdate(endTime) + + taskStartTime := s.task.GetStartTime() + s.task.SetExecutionTime(taskStartTime, endTime). + SetLastUpdate(endTime) + + // 任务超时, 整体结束 + if taskStatus != nil { + if taskStatus.messsage != "" { + taskFailMsg = taskStatus.messsage + } + s.task.SetEndTime(endTime). + SetStatus(taskStatus.status). + SetMessage(taskFailMsg) + + // callback + if s.cbExecutor != nil { + c := istep.NewContext(context.Background(), GetGlobalStorage(), s.task, s.step) + s.cbExecutor.Callback(c, stepErr) + } + return + } + + // last step failed and skipOnFailed is true, update task status to success + if s.isLastStep(s.step) { + if s.step.GetSkipOnFailed() { + // ignore error + stepErr = nil + s.task.SetEndTime(endTime). + SetStatus(types.TaskStatusSuccess). + SetMessage("task finished successfully") + } else { + s.task.SetEndTime(endTime). + SetStatus(types.TaskStatusFailure). + SetMessage(taskFailMsg) + } + + // callback + if s.cbExecutor != nil { + c := istep.NewContext(context.Background(), GetGlobalStorage(), s.task, s.step) + s.cbExecutor.Callback(c, stepErr) + } + return + } + + // 重试流程中 + if !errors.Is(stepErr, istep.ErrRevoked) && s.step.GetRetryCount() < s.step.MaxRetries { + s.task.SetStatus(types.TaskStatusRunning).SetMessage(taskFailMsg) + return + } + + // 忽略错误 + if s.step.GetSkipOnFailed() { + msg := fmt.Sprintf("step %s running failed, with skip on failed", s.step.Name) + s.task.SetStatus(types.TaskStatusRunning).SetMessage(msg) + return + } + + // 重试次数用完且没有忽略错误 + s.task.SetEndTime(endTime). + SetStatus(types.TaskStatusFailure). + SetMessage(taskFailMsg) + + // callback + if s.cbExecutor != nil { + c := istep.NewContext(context.Background(), GetGlobalStorage(), s.task, s.step) + s.cbExecutor.Callback(c, stepErr) + } +} + +func (s *State) isLastStep(step *types.Step) bool { + count := len(s.task.Steps) + // 没有step也就没有后续流程, 返回true + if count == 0 { + return true + } + + // 非最后一步 + if step.GetName() != s.task.Steps[count-1].Name { + return false + } + + // 最后一步还需要看重试次数 + return step.IsCompleted() +} + +// GetTask get task +func (s *State) GetTask() *types.Task { + return s.task +} diff --git a/task/state_test.go b/task/state_test.go new file mode 100644 index 0000000..e6d55e6 --- /dev/null +++ b/task/state_test.go @@ -0,0 +1,52 @@ +/* + * Tencent is pleased to support the open source community by making Blueking Container Service available. + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * Licensed under the MIT License (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * http://opensource.org/licenses/MIT + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language governing permissions and + * limitations under the License. + */ + +package task + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + + "git.ifooth.com/common/pkg/task/stores/mem" + "git.ifooth.com/common/pkg/task/types" +) + +func TestIsReadyToStep(t *testing.T) { + globalStorage = mem.New() + + info := types.TaskInfo{ + TaskType: "example-test", + TaskName: "example", + Creator: "bcs", + } + task := types.NewTask(info) + stepName := "step1" + state := NewState(task, stepName) + step, err := state.isReadyToStep(stepName) + if assert.Error(t, err) { + assert.True(t, strings.Contains(err.Error(), "not exist")) + assert.Nil(t, step) + } + + steps := []*types.Step{ + types.NewStep("step1", "hello"), + } + task.Steps = steps + step, err = state.isReadyToStep(stepName) + + assert.NoError(t, err) + assert.Equal(t, stepName, step.Name) + assert.Equal(t, types.TaskStatusRunning, step.Status) + assert.Equal(t, types.TaskStatusRunning, task.Status) +} diff --git a/task/steps/bksops/bksops.go b/task/steps/bksops/bksops.go new file mode 100644 index 0000000..c924368 --- /dev/null +++ b/task/steps/bksops/bksops.go @@ -0,0 +1,44 @@ +/* + * Tencent is pleased to support the open source community by making Blueking Container Service available. + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * Licensed under the MIT License (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * http://opensource.org/licenses/MIT + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package bksops defines the step implemented. +package bksops + +import ( + istep "git.ifooth.com/common/pkg/task/steps/iface" +) + +type sops struct { + bkAppCode string + bkAppSecret string +} + +func newSops(bkAppCode, bkAppSecret string) *sops { + return &sops{bkAppCode, bkAppSecret} +} + +func (s *sops) Execute(c *istep.Context) error { + return nil +} + +// 注意, Step名称不能修改 +const ( + // BKSopsStep ... + BKSopsStep istep.StepName = "BK_SOPS" +) + +// Register ... +func Register(bkAppCode, bkAppSecret string) { + s := newSops(bkAppCode, bkAppSecret) + + istep.Register(BKSopsStep, istep.StepExecutor(s)) +} diff --git a/task/steps/hello/callback.go b/task/steps/hello/callback.go new file mode 100644 index 0000000..baedbbe --- /dev/null +++ b/task/steps/hello/callback.go @@ -0,0 +1,36 @@ +/* + * Tencent is pleased to support the open source community by making Blueking Container Service available. + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * Licensed under the MIT License (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * http://opensource.org/licenses/MIT + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package hello defines the hello step. +package hello + +import ( + "fmt" + + istep "git.ifooth.com/common/pkg/task/steps/iface" +) + +// Callback ... +func Callback(c *istep.Context, err error) { + if err != nil { + fmt.Println(err) + } +} + +type callback struct { +} + +func (cb *callback) Callback(c *istep.Context, err error) { + if err != nil { + fmt.Println(err) + } +} diff --git a/task/steps/hello/hello.go b/task/steps/hello/hello.go new file mode 100644 index 0000000..bd826bb --- /dev/null +++ b/task/steps/hello/hello.go @@ -0,0 +1,52 @@ +/* + * Tencent is pleased to support the open source community by making Blueking Container Service available. + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * Licensed under the MIT License (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * http://opensource.org/licenses/MIT + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package hello defines the hello step. +package hello + +import ( + "fmt" + + istep "git.ifooth.com/common/pkg/task/steps/iface" +) + +// hello hello +type hello struct{} + +// NewHello ... +func NewHello() istep.StepExecutor { + return &hello{} +} + +// DoWork for worker exec task +func (s *hello) Execute(c *istep.Context) error { + fmt.Println("Hello") + // time.Sleep(30 * time.Second) + if err := c.AddCommonParam("name", "hello"); err != nil { + return err + } + return nil +} + +func init() { + // 使用结构体注册 + istep.Register("hello", NewHello()) + + // 使用函数注册 + istep.Register("sum", istep.StepExecutorFunc(Sum)) + + // 回调使用结构体注册 + istep.RegisterCallback("callback_fun", &callback{}) + + // 回调使用函数注册 + istep.RegisterCallback("callback", istep.CallbackExecutorFunc(Callback)) +} diff --git a/task/steps/hello/sum.go b/task/steps/hello/sum.go new file mode 100644 index 0000000..1820096 --- /dev/null +++ b/task/steps/hello/sum.go @@ -0,0 +1,60 @@ +/* + * Tencent is pleased to support the open source community by making Blueking Container Service available. + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * Licensed under the MIT License (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * http://opensource.org/licenses/MIT + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language governing permissions and + * limitations under the License. + */ + +package hello + +import ( + "fmt" + "strconv" + + istep "git.ifooth.com/common/pkg/task/steps/iface" +) + +var ( + // SumA istep.ParamKey = "sumA" + SumA istep.ParamKey = "sumA" + // SumB ... + SumB istep.ParamKey = "sumB" + // SumC ... + SumC istep.ParamKey = "sumC" +) + +// Sum ... +func Sum(c *istep.Context) error { + // time.Sleep(30 * time.Second) + + a, ok := c.GetParam(SumA.String()) + if !ok { + return fmt.Errorf("%w: param=%s", istep.ErrParamNotFound, SumA.String()) + } + + b, ok := c.GetParam(SumB.String()) + if !ok { + return fmt.Errorf("%w: param=%s", istep.ErrParamNotFound, SumB.String()) + } + + a1, err := strconv.Atoi(a) + if err != nil { + return err + } + + b1, err := strconv.Atoi(b) + if err != nil { + return err + } + + c1 := a1 + b1 + _ = c.AddCommonParam(SumC.String(), fmt.Sprintf("%v", c1)) + + fmt.Printf("%s %s %s sumC: %v\n", c.GetTaskID(), c.GetTaskType(), c.GetName(), c) + return nil +} diff --git a/task/steps/iface/context.go b/task/steps/iface/context.go new file mode 100644 index 0000000..452fd0d --- /dev/null +++ b/task/steps/iface/context.go @@ -0,0 +1,166 @@ +/* + * Tencent is pleased to support the open source community by making Blueking Container Service available. + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * Licensed under the MIT License (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * http://opensource.org/licenses/MIT + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language governing permissions and + * limitations under the License. + */ + +package iface + +import ( + "context" + "errors" + "time" + + istore "git.ifooth.com/common/pkg/task/stores/iface" + "git.ifooth.com/common/pkg/task/types" +) + +var ( + // ErrRevoked step has been revoked + ErrRevoked = errors.New("revoked") +) + +// Context 当前执行的任务 +type Context struct { + ctx context.Context + store istore.Store + task *types.Task + currentStep *types.Step +} + +// NewContext ... +func NewContext(ctx context.Context, store istore.Store, task *types.Task, currentStep *types.Step) *Context { + return &Context{ + ctx: ctx, + store: store, + task: task, + currentStep: currentStep, + } +} + +// Context returns the step's context +func (c *Context) Context() context.Context { + return c.ctx +} + +// GetTaskID get task id +func (c *Context) GetTaskID() string { + return c.task.GetTaskID() +} + +// GetTaskName get task name +func (c *Context) GetTaskName() string { + return c.task.GetTaskName() +} + +// GetTaskType get task type +func (c *Context) GetTaskType() string { + return c.task.GetTaskType() +} + +// GetTaskIndex get task index +func (c *Context) GetTaskIndex() string { + return c.task.GetTaskIndex() +} + +// GetTaskIndexType get task index type +func (c *Context) GetTaskIndexType() string { + return c.task.GetTaskIndexType() +} + +// GetTaskStatus get task status +func (c *Context) GetTaskStatus() string { + return c.task.GetStatus() +} + +// GetCommonParams get task common params +func (c *Context) GetCommonParams() map[string]string { + return c.task.GetCommonParams() +} + +// GetCommonParam get current task param +func (c *Context) GetCommonParam(key string) (string, bool) { + return c.task.GetCommonParam(key) +} + +// AddCommonParam add task common param and save to store +func (c *Context) AddCommonParam(k, v string) error { + _ = c.task.AddCommonParam(k, v) + return c.store.UpdateTask(c.ctx, c.task) +} + +// GetCommonPayload unmarshal task common payload to struct obj +func (c *Context) GetCommonPayload(obj any) error { + return c.task.GetCommonPayload(obj) +} + +// SetCommonPayload marshal struct obj to task common payload and save to store +func (c *Context) SetCommonPayload(obj any) error { + if err := c.task.SetCommonPayload(obj); err != nil { + return err + } + + return c.store.UpdateTask(c.ctx, c.task) +} + +// GetName get current step name +func (c *Context) GetName() string { + return c.currentStep.GetName() +} + +// GetStatus get current step status +func (c *Context) GetStatus() string { + return c.currentStep.GetStatus() +} + +// GetRetryCount get current step retry count +func (c *Context) GetRetryCount() uint32 { + return c.currentStep.GetRetryCount() +} + +// GetParam get current step param by key +func (c *Context) GetParam(key string) (string, bool) { + return c.currentStep.GetParam(key) +} + +// AddParam set step param by key,value and save to store +func (c *Context) AddParam(key string, value string) error { + _ = c.currentStep.AddParam(key, value) + return c.store.UpdateTask(c.ctx, c.task) +} + +// GetParams return all step params +func (c *Context) GetParams() map[string]string { + return c.currentStep.GetParams() +} + +// SetParams set all step params and save to store +func (c *Context) SetParams(params map[string]string) error { + c.currentStep.SetParams(params) + return c.store.UpdateTask(c.ctx, c.task) +} + +// GetPayload return unmarshal step payload +func (c *Context) GetPayload(obj any) error { + return c.currentStep.GetPayload(obj) +} + +// GetStartTime return step start time +func (c *Context) GetStartTime() time.Time { + return c.currentStep.Start +} + +// SetPayload marshal struct obj to step payload and save to store +func (c *Context) SetPayload(obj any) error { + if err := c.currentStep.SetPayload(obj); err != nil { + return err + } + + return c.store.UpdateTask(c.ctx, c.task) +} diff --git a/task/steps/iface/interfaces.go b/task/steps/iface/interfaces.go new file mode 100644 index 0000000..fc31dab --- /dev/null +++ b/task/steps/iface/interfaces.go @@ -0,0 +1,69 @@ +/* + * Tencent is pleased to support the open source community by making Blueking Container Service available. + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * Licensed under the MIT License (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * http://opensource.org/licenses/MIT + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package iface is a package for task step interface +package iface + +import ( + "errors" +) + +var ( + // ErrParamNotFound 参数未找到 + ErrParamNotFound = errors.New("param not found") +) + +// StepExecutor that client must implement +type StepExecutor interface { + Execute(*Context) error +} + +// The StepExecutorFunc type is an adapter to allow the use of +// ordinary functions as a Executor. If f is a function +// with the appropriate signature, StepExecutorFunc(f) is a +// Executor that calls f. +type StepExecutorFunc func(*Context) error + +// Execute calls f(c) +func (f StepExecutorFunc) Execute(c *Context) error { + return f(c) +} + +// CallbackExecutor that callback client must implement +type CallbackExecutor interface { + Callback(*Context, error) +} + +// The CallbackExecutorFunc type is an adapter to allow the use of +// ordinary functions as a Executor. If f is a function +// with the appropriate signature, CallbackExecutorFunc(f) is a +// Executor that calls f. +type CallbackExecutorFunc func(*Context, error) + +// Callback calls f(c, cbErr) +func (f CallbackExecutorFunc) Callback(c *Context, cbErr error) { + f(c, cbErr) +} + +// KeyValue key-value paras +type KeyValue struct { + Key ParamKey + Value string +} + +// ParamKey xxx +type ParamKey string + +// String xxx +func (pk ParamKey) String() string { + return string(pk) +} diff --git a/task/steps/iface/register.go b/task/steps/iface/register.go new file mode 100644 index 0000000..8f7aefc --- /dev/null +++ b/task/steps/iface/register.go @@ -0,0 +1,89 @@ +/* + * Tencent is pleased to support the open source community by making Blueking Container Service available. + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * Licensed under the MIT License (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * http://opensource.org/licenses/MIT + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language governing permissions and + * limitations under the License. + */ + +package iface + +import ( + "sync" +) + +// StepName 步骤名称, 通过这个查找Executor, 必须全局唯一 +type StepName string + +// String ... +func (s StepName) String() string { + return string(s) +} + +// CallbackName 步骤名称, 通过这个查找callback Executor, 必须全局唯一 +type CallbackName string + +// String ... +func (cb CallbackName) String() string { + return string(cb) +} + +var ( + stepMu sync.RWMutex + steps = make(map[StepName]StepExecutor) + callBacks = make(map[CallbackName]CallbackExecutor) +) + +// Register makes a StepExecutor available by the provided name. +// If Register is called twice with the same name or if StepExecutor is nil, +// it panics. +func Register(name StepName, step StepExecutor) { + stepMu.Lock() + defer stepMu.Unlock() + + if step == nil { + panic("task: Register step is nil") + } + + if _, dup := steps[name]; dup { + panic("task: Register step twice for executor " + name) + } + + steps[name] = step +} + +// GetRegisters get all steps instance +func GetRegisters() map[StepName]StepExecutor { + stepMu.Lock() + defer stepMu.Unlock() + + return steps +} + +// RegisterCallback ... +func RegisterCallback(name CallbackName, cb CallbackExecutor) { + stepMu.Lock() + defer stepMu.Unlock() + + if cb == nil { + panic("task: Register callback is nil") + } + + if _, dup := callBacks[name]; dup { + panic("task: Register callback twice for executor " + name) + } + + callBacks[name] = cb +} + +// GetCallbackRegisters get all steps instance +func GetCallbackRegisters() map[CallbackName]CallbackExecutor { + stepMu.Lock() + defer stepMu.Unlock() + + return callBacks +} diff --git a/task/storage.go b/task/storage.go new file mode 100644 index 0000000..4c3888b --- /dev/null +++ b/task/storage.go @@ -0,0 +1,25 @@ +/* + * Tencent is pleased to support the open source community by making Blueking Container Service available. + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * Licensed under the MIT License (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * http://opensource.org/licenses/MIT + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language governing permissions and + * limitations under the License. + */ + +package task + +import istore "git.ifooth.com/common/pkg/task/stores/iface" + +var ( + // globalStorage used for state and task manager + globalStorage istore.Store +) + +// GetGlobalStorage for cluster manager storage tools +func GetGlobalStorage() istore.Store { + return globalStorage +} diff --git a/task/stores/iface/interfaces.go b/task/stores/iface/interfaces.go new file mode 100644 index 0000000..3978d9a --- /dev/null +++ b/task/stores/iface/interfaces.go @@ -0,0 +1,61 @@ +/* + * Tencent is pleased to support the open source community by making Blueking Container Service available. + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * Licensed under the MIT License (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * http://opensource.org/licenses/MIT + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package iface defines the interface for store. +package iface + +import ( + "context" + "time" + + "git.ifooth.com/common/pkg/task/types" +) + +// ListOption ... +type ListOption struct { + TaskID string + TaskType string + TaskName string + TaskIndex string + TaskIndexType string + CurrentStep string + Status string + Creator string + CreatedGte *time.Time // CreatedGte create time greater or equal to + CreatedLte *time.Time // CreatedLte create time less or equal to + Sort map[string]int // Sort map for sort list results + Offset int64 // Offset offset for list results + Limit int64 // Limit limit for list results +} + +// Pagination generic pagination for list results +type Pagination[T any] struct { + Count int64 `json:"count"` + Items []*T `json:"items"` +} + +// PatchOption 主要实时更新params, payload信息 +type PatchOption struct { + Task *types.Task + CurrentStep *types.Step +} + +// Store model for TaskManager +type Store interface { + EnsureTable(ctx context.Context, dst ...any) error + CreateTask(ctx context.Context, task *types.Task) error + ListTask(ctx context.Context, opt *ListOption) (*Pagination[types.Task], error) + GetTask(ctx context.Context, taskID string) (*types.Task, error) + DeleteTask(ctx context.Context, taskID string) error + UpdateTask(ctx context.Context, task *types.Task) error + // PatchTask(ctx context.Context, opt *PatchOption) error // Patcask 更新params/payload信息 +} diff --git a/task/stores/mem/mem.go b/task/stores/mem/mem.go new file mode 100644 index 0000000..0ee0073 --- /dev/null +++ b/task/stores/mem/mem.go @@ -0,0 +1,77 @@ +/* + * Tencent is pleased to support the open source community by making Blueking Container Service available. + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * Licensed under the MIT License (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * http://opensource.org/licenses/MIT + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package mem implemented storage interface. +package mem + +import ( + "context" + "fmt" + "sync" + + "git.ifooth.com/common/pkg/task/stores/iface" + "git.ifooth.com/common/pkg/task/types" +) + +type memStore struct { + mtx sync.Mutex + tasks map[string]*types.Task +} + +// New new memStore +func New() iface.Store { + s := &memStore{ + tasks: make(map[string]*types.Task), + } + return s +} + +// EnsureTable 创建db表 +func (s *memStore) EnsureTable(ctx context.Context, dst ...any) error { + return nil +} + +func (s *memStore) CreateTask(ctx context.Context, task *types.Task) error { + s.mtx.Lock() + defer s.mtx.Unlock() + + s.tasks[task.GetTaskID()] = task + return nil +} + +func (s *memStore) ListTask(ctx context.Context, opt *iface.ListOption) (*iface.Pagination[types.Task], error) { + return nil, types.ErrNotImplemented +} + +func (s *memStore) UpdateTask(ctx context.Context, task *types.Task) error { + s.tasks[task.GetTaskID()] = task + return nil +} + +func (s *memStore) DeleteTask(ctx context.Context, taskID string) error { + return types.ErrNotImplemented +} + +func (s *memStore) GetTask(ctx context.Context, taskID string) (*types.Task, error) { + s.mtx.Lock() + defer s.mtx.Unlock() + + t, ok := s.tasks[taskID] + if ok { + return t, nil + } + return nil, fmt.Errorf("not found") +} + +func (s *memStore) PatchTask(ctx context.Context, opt *iface.PatchOption) error { + return types.ErrNotImplemented +} diff --git a/task/stores/mysql/helper.go b/task/stores/mysql/helper.go new file mode 100644 index 0000000..82cb2e8 --- /dev/null +++ b/task/stores/mysql/helper.go @@ -0,0 +1,185 @@ +/* + * Tencent is pleased to support the open source community by making Blueking Container Service available. + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * Licensed under the MIT License (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * http://opensource.org/licenses/MIT + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language governing permissions and + * limitations under the License. + */ + +package mysql + +import ( + "gorm.io/gorm" + + "git.ifooth.com/common/pkg/task/types" +) + +func getStepRecord(t *types.Task) []*StepRecord { + records := make([]*StepRecord, 0, len(t.Steps)) + for _, step := range t.Steps { + record := &StepRecord{ + TaskID: t.TaskID, + Name: step.Name, + Alias: step.Alias, + Executor: step.Executor, + Payload: step.Payload, + Status: step.Status, + Message: step.Message, + SkipOnFailed: step.SkipOnFailed, + ETA: step.ETA, + RetryCount: step.RetryCount, + MaxRetries: step.MaxRetries, + Params: step.Params, + Start: step.Start, + End: step.End, + ExecutionTime: step.ExecutionTime, + MaxExecutionSeconds: step.MaxExecutionSeconds, + } + records = append(records, record) + } + + return records +} +func getTaskRecord(t *types.Task) *TaskRecord { + stepSequence := make([]string, 0, len(t.Steps)) + for i := range t.Steps { + stepSequence = append(stepSequence, t.Steps[i].Name) + } + + record := &TaskRecord{ + TaskID: t.TaskID, + TaskType: t.TaskType, + TaskIndex: t.TaskIndex, + TaskIndexType: t.TaskIndexType, + TaskName: t.TaskName, + CurrentStep: t.CurrentStep, + StepSequence: stepSequence, + CallbackName: t.CallbackName, + CommonParams: t.CommonParams, + CommonPayload: t.CommonPayload, + Status: t.Status, + Message: t.Message, + Start: t.Start, + End: t.End, + ExecutionTime: t.ExecutionTime, + MaxExecutionSeconds: t.MaxExecutionSeconds, + Creator: t.Creator, + Updater: t.Updater, + } + return record +} + +func toTask(task *TaskRecord, steps []*StepRecord) *types.Task { + t := &types.Task{ + TaskID: task.TaskID, + TaskType: task.TaskType, + TaskIndex: task.TaskIndex, + TaskIndexType: task.TaskIndexType, + TaskName: task.TaskName, + CurrentStep: task.CurrentStep, + CallbackName: task.CallbackName, + CommonParams: task.CommonParams, + CommonPayload: task.CommonPayload, + Status: task.Status, + Message: task.Message, + Start: task.Start, + End: task.End, + ExecutionTime: task.ExecutionTime, + MaxExecutionSeconds: task.MaxExecutionSeconds, + CreatedAt: task.CreatedAt, + LastUpdate: task.UpdatedAt, + Creator: task.Creator, + Updater: task.Updater, + } + + stepMap := map[string]*StepRecord{} + for _, step := range steps { + stepMap[step.Name] = step + } + + t.Steps = make([]*types.Step, 0, len(task.StepSequence)) + for _, step := range task.StepSequence { + step, ok := stepMap[step] + if !ok { + continue + } + t.Steps = append(t.Steps, step.ToStep()) + } + return t +} + +var ( + // updateTaskField task 支持更新的字段 + updateTaskField = []string{ + "CurrentStep", + "CommonParams", + "CommonPayload", + "Status", + "Message", + "Start", + "End", + "ExecutionTime", + "Updater", + } + + // updateStepField step 支持更新的字段 + updateStepField = []string{ + "Params", + "Payload", + "Status", + "Message", + "Start", + "End", + "ExecutionTime", + "RetryCount", + } +) + +func getUpdateTaskRecord(t *types.Task) *TaskRecord { + record := &TaskRecord{ + CurrentStep: t.CurrentStep, + CommonParams: t.CommonParams, + CommonPayload: t.CommonPayload, + Status: t.Status, + Message: t.Message, + Start: t.Start, + End: t.End, + ExecutionTime: t.ExecutionTime, + Updater: t.Updater, + } + return record +} + +func getUpdateStepRecord(t *types.Step) *StepRecord { + record := &StepRecord{ + Params: t.Params, + Payload: t.Payload, + Status: t.Status, + Message: t.Message, + Start: t.Start, + End: t.End, + ExecutionTime: t.ExecutionTime, + RetryCount: t.RetryCount, + } + return record +} + +// FindByPage 分页查询 +func FindByPage[T any](db *gorm.DB, offset int, limit int) (result []*T, count int64, err error) { + err = db.Offset(offset).Limit(limit).Find(&result).Error + if err != nil { + return + } + + if size := len(result); 0 < limit && 0 < size && size < limit { + count = int64(size + offset) + return + } + + err = db.Offset(-1).Limit(-1).Count(&count).Error + return +} diff --git a/task/stores/mysql/mysql.go b/task/stores/mysql/mysql.go new file mode 100644 index 0000000..19a7138 --- /dev/null +++ b/task/stores/mysql/mysql.go @@ -0,0 +1,194 @@ +/* + * Tencent is pleased to support the open source community by making Blueking Container Service available. + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * Licensed under the MIT License (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * http://opensource.org/licenses/MIT + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package mysql implemented storage interface. +package mysql + +import ( + "context" + + "gorm.io/driver/mysql" + "gorm.io/gorm" + "gorm.io/gorm/logger" + + "git.ifooth.com/common/pkg/task/stores/iface" + "git.ifooth.com/common/pkg/task/types" +) + +type mysqlStore struct { + dsn string + debug bool + db *gorm.DB +} + +type option func(*mysqlStore) + +// WithDebug 是否显示sql语句 +func WithDebug(debug bool) option { + return func(s *mysqlStore) { + s.debug = debug + } +} + +// New init mysql iface.Store +func New(dsn string, opts ...option) (iface.Store, error) { + store := &mysqlStore{dsn: dsn, debug: false} + for _, opt := range opts { + opt(store) + } + + // 是否显示sql语句 + level := logger.Warn + if store.debug { + level = logger.Info + } + + db, err := gorm.Open(mysql.Open(store.dsn), + &gorm.Config{Logger: logger.Default.LogMode(level)}, + ) + if err != nil { + return nil, err + } + store.db = db + + return store, nil +} + +// EnsureTable implement istore EnsureTable interface +func (s *mysqlStore) EnsureTable(ctx context.Context, dst ...any) error { + // 没有自定义数据, 使用默认表结构 + if len(dst) == 0 { + dst = []any{&TaskRecord{}, &StepRecord{}} + } + return s.db.WithContext(ctx).AutoMigrate(dst...) +} + +// CreateTask implement istore CreateTask interface +func (s *mysqlStore) CreateTask(ctx context.Context, task *types.Task) error { + return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + record := getTaskRecord(task) + if err := tx.Create(record).Error; err != nil { + return err + } + + steps := getStepRecord(task) + if err := tx.CreateInBatches(steps, 100).Error; err != nil { + return err + } + + return nil + }) +} + +// ListTask implement istore ListTask interface +func (s *mysqlStore) ListTask(ctx context.Context, opt *iface.ListOption) (*iface.Pagination[types.Task], error) { + tx := s.db.WithContext(ctx) + + // 条件过滤 0值gorm自动忽略查询 + tx = tx.Where(&TaskRecord{ + TaskID: opt.TaskID, + TaskType: opt.TaskType, + TaskName: opt.TaskName, + TaskIndex: opt.TaskIndex, + TaskIndexType: opt.TaskIndexType, + Status: opt.Status, + CurrentStep: opt.CurrentStep, + Creator: opt.Creator, + }) + + // mysql store 使用创建时间过滤 + if opt.CreatedGte != nil { + tx = tx.Where("created_at >= ?", opt.CreatedGte) + } + if opt.CreatedLte != nil { + tx = tx.Where("created_at <= ?", opt.CreatedLte) + } + + // 只使用id排序 + tx = tx.Order("id DESC") + + result, count, err := FindByPage[TaskRecord](tx, int(opt.Offset), int(opt.Limit)) + if err != nil { + return nil, err + } + + items := make([]*types.Task, 0, len(result)) + for _, record := range result { + items = append(items, toTask(record, []*StepRecord{})) + } + + return &iface.Pagination[types.Task]{ + Count: count, + Items: items, + }, nil +} + +// UpdateTask implement istore UpdateTask interface +func (s *mysqlStore) UpdateTask(ctx context.Context, task *types.Task) error { + return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + updateTask := getUpdateTaskRecord(task) + if err := tx.Model(&TaskRecord{}). + Where("task_id = ?", task.TaskID). + Select(updateTaskField). + Updates(updateTask).Error; err != nil { + return err + } + + for _, step := range task.Steps { + if step.Name != task.CurrentStep { + continue + } + updateStep := getUpdateStepRecord(step) + if err := tx.Model(&StepRecord{}). + Where("task_id = ? AND name= ?", task.TaskID, step.Name). + Select(updateStepField). + Updates(updateStep).Error; err != nil { + return err + } + } + return nil + }) +} + +// DeleteTask implement istore DeleteTask interface +func (s *mysqlStore) DeleteTask(ctx context.Context, taskID string) error { + return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + if err := tx.Where("task_id = ?", taskID).Delete(&TaskRecord{}).Error; err != nil { + return err + } + + if err := tx.Where("task_id = ?", taskID).Delete(&StepRecord{}).Error; err != nil { + return err + } + return nil + }) +} + +// GetTask implement istore GetTask interface +func (s *mysqlStore) GetTask(ctx context.Context, taskID string) (*types.Task, error) { + tx := s.db.WithContext(ctx) + taskRecord := TaskRecord{} + if err := tx.Where("task_id = ?", taskID).First(&taskRecord).Error; err != nil { + return nil, err + } + + stepRecord := []*StepRecord{} + if err := tx.Where("task_id = ?", taskID).Find(&stepRecord).Error; err != nil { + return nil, err + } + return toTask(&taskRecord, stepRecord), nil +} + +// PatchTask implement istore PatchTask interface +func (s *mysqlStore) PatchTask(ctx context.Context, opt *iface.PatchOption) error { + return types.ErrNotImplemented +} diff --git a/task/stores/mysql/table.go b/task/stores/mysql/table.go new file mode 100644 index 0000000..5d9a616 --- /dev/null +++ b/task/stores/mysql/table.go @@ -0,0 +1,161 @@ +/* + * Tencent is pleased to support the open source community by making Blueking Container Service available. + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * Licensed under the MIT License (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * http://opensource.org/licenses/MIT + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language governing permissions and + * limitations under the License. + */ + +package mysql + +import ( + "time" + + "gorm.io/gorm" + + "git.ifooth.com/common/pkg/task/types" +) + +/** +字段规范: +1. 字段名使用驼峰命名法，表字段使用 _ 分隔 +2. bool/int/float/datetime 等类型使用默认字段类型 +3. string 类型必须指定类型和长度 +4. index 固定varchar(191), (mysql 5.6索引长度限制767byte, utf8mb4下最长191) +**/ + +var ( + // UnixZeroTime mysql 8.0 版本以上不能写入, 使用unix 0时作为zero time + // https://dev.mysql.com/doc/refman/8.0/en/datetime.html + UnixZeroTime = time.Unix(0, 0) +) + +// BaseModel 添加 CreatedAt 索引 +type BaseModel struct { + gorm.Model + CreatedAt time.Time `gorm:"index"` +} + +// TaskRecord 任务记录 +type TaskRecord struct { + BaseModel + TaskID string `json:"taskID" gorm:"type:varchar(191);uniqueIndex:idx_task_id"` // 唯一索引 + TaskType string `json:"taskType" gorm:"type:varchar(191);index:idx_task_type"` + TaskIndex string `json:"TaskIndex" gorm:"type:varchar(191);index:idx_task_index"` + TaskIndexType string `json:"TaskIndexType" gorm:"type:varchar(191);index:idx_task_index"` + TaskName string `json:"taskName" gorm:"type:varchar(255)"` + CurrentStep string `json:"currentStep" gorm:"type:varchar(255)"` + StepSequence []string `json:"stepSequence" gorm:"type:text;serializer:json"` + CallbackName string `json:"callbackName" gorm:"type:varchar(255)"` + CommonParams map[string]string `json:"commonParams" gorm:"type:text;serializer:json"` + CommonPayload string `json:"commonPayload" gorm:"type:text"` + Status string `json:"status" gorm:"type:varchar(191);index:idx_status"` + Message string `json:"message" gorm:"type:text"` + ExecutionTime uint32 `json:"executionTime"` + MaxExecutionSeconds uint32 `json:"maxExecutionSeconds"` + Start time.Time `json:"start"` + End time.Time `json:"end"` + Creator string `json:"creator" gorm:"type:varchar(255)"` + Updater string `json:"updater" gorm:"type:varchar(255)"` +} + +// TableName .. +func (t *TaskRecord) TableName() string { + return "task_records" +} + +// BeforeCreate .. +func (t *TaskRecord) BeforeCreate(tx *gorm.DB) error { + if t.Start.IsZero() { + t.Start = UnixZeroTime + } + if t.End.IsZero() { + t.End = UnixZeroTime + } + return nil +} + +// BeforeUpdate .. +func (t *TaskRecord) BeforeUpdate(tx *gorm.DB) error { + if t.Start.IsZero() { + t.Start = UnixZeroTime + } + if t.End.IsZero() { + t.End = UnixZeroTime + } + return nil +} + +// StepRecord 步骤记录 +type StepRecord struct { + gorm.Model + TaskID string `json:"taskID" gorm:"type:varchar(191);uniqueIndex:idx_task_id_step_name"` + Name string `json:"name" gorm:"type:varchar(191);uniqueIndex:idx_task_id_step_name"` + Alias string `json:"alias" gorm:"type:varchar(255)"` + Executor string `json:"executor" gorm:"type:varchar(255)"` + Params map[string]string `json:"input" gorm:"type:text;serializer:json"` + Payload string `json:"payload" gorm:"type:text"` + Status string `json:"status" gorm:"type:varchar(255)"` + Message string `json:"message" gorm:"type:text"` + ETA *time.Time `json:"eta"` + SkipOnFailed bool `json:"skipOnFailed"` + RetryCount uint32 `json:"retryCount"` + MaxRetries uint32 `json:"maxRetries"` + ExecutionTime uint32 `json:"executionTime"` + MaxExecutionSeconds uint32 `json:"maxExecutionSeconds"` + Start time.Time `json:"start"` + End time.Time `json:"end"` +} + +// TableName .. +func (t *StepRecord) TableName() string { + return "task_step_records" +} + +// BeforeCreate .. +func (t *StepRecord) BeforeCreate(tx *gorm.DB) error { + if t.Start.IsZero() { + t.Start = UnixZeroTime + } + if t.End.IsZero() { + t.End = UnixZeroTime + } + return nil +} + +// BeforeUpdate .. +func (t *StepRecord) BeforeUpdate(tx *gorm.DB) error { + if t.Start.IsZero() { + t.Start = UnixZeroTime + } + if t.End.IsZero() { + t.End = UnixZeroTime + } + return nil +} + +// ToStep 类型转换 +func (t *StepRecord) ToStep() *types.Step { + return &types.Step{ + Name: t.Name, + Alias: t.Alias, + Executor: t.Executor, + Params: t.Params, + Payload: t.Payload, + Status: t.Status, + Message: t.Message, + ETA: t.ETA, + SkipOnFailed: t.SkipOnFailed, + RetryCount: t.RetryCount, + MaxRetries: t.MaxRetries, + ExecutionTime: t.ExecutionTime, + MaxExecutionSeconds: t.MaxExecutionSeconds, + Start: t.Start, + End: t.End, + LastUpdate: t.UpdatedAt, + } +} diff --git a/task/types/step.go b/task/types/step.go new file mode 100644 index 0000000..e44c7ef --- /dev/null +++ b/task/types/step.go @@ -0,0 +1,293 @@ +/* + * Tencent is pleased to support the open source community by making Blueking Container Service available. + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * Licensed under the MIT License (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * http://opensource.org/licenses/MIT + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package types for task +package types + +import ( + "encoding/json" + "time" +) + +// StepOptions xxx +type StepOptions struct { + MaxRetries uint32 + SkipFailed bool + MaxExecutionSeconds uint32 +} + +// StepOption xxx +type StepOption func(opt *StepOptions) + +// WithMaxRetries xxx +func WithMaxRetries(count uint32) StepOption { + return func(opt *StepOptions) { + opt.MaxRetries = count + } +} + +// WithStepSkipFailed xxx +func WithStepSkipFailed(skip bool) StepOption { + return func(opt *StepOptions) { + opt.SkipFailed = skip + } +} + +// WithMaxExecutionSeconds xxx +func WithMaxExecutionSeconds(execSecs uint32) StepOption { + return func(opt *StepOptions) { + opt.MaxExecutionSeconds = execSecs + } +} + +// NewStep return a new step by default params +func NewStep(name string, executor string, opts ...StepOption) *Step { + defaultOptions := &StepOptions{MaxRetries: 0} + for _, opt := range opts { + opt(defaultOptions) + } + + return &Step{ + Name: name, + Executor: executor, + Params: map[string]string{}, + Payload: DefaultPayloadContent, + Status: TaskStatusNotStarted, + Message: "", + RetryCount: 0, + SkipOnFailed: defaultOptions.SkipFailed, + MaxRetries: defaultOptions.MaxRetries, + MaxExecutionSeconds: defaultOptions.MaxExecutionSeconds, + } +} + +// GetName return task name +func (s *Step) GetName() string { + return s.Name +} + +// GetAlias return task alias +func (s *Step) GetAlias() string { + return s.Alias +} + +// SetAlias set task alias +func (s *Step) SetAlias(alias string) *Step { + s.Alias = alias + return s +} + +// GetParam return step param by key +func (s *Step) GetParam(key string) (string, bool) { + if value, ok := s.Params[key]; ok { + return value, true + } + return "", false +} + +// AddParam set step param by key,value +func (s *Step) AddParam(key, value string) *Step { + if s.Params == nil { + s.Params = make(map[string]string, 0) + } + s.Params[key] = value + return s +} + +// GetParams return all step params +func (s *Step) GetParams() map[string]string { + if s.Params == nil { + s.Params = make(map[string]string, 0) + } + return s.Params +} + +// SetParams set step params by map +func (s *Step) SetParams(params map[string]string) { + if s.Params == nil { + s.Params = make(map[string]string, 0) + } + for key, value := range params { + s.Params[key] = value + } +} + +// SetNewParams replace all params by new params +func (s *Step) SetNewParams(params map[string]string) *Step { + s.Params = params + return s +} + +// GetPayload unmarshal step payload to struct obj +func (s *Step) GetPayload(obj any) error { + if len(s.Payload) == 0 { + s.Payload = DefaultPayloadContent + } + return json.Unmarshal([]byte(s.Payload), obj) +} + +// SetPayload marshal struct obj to step payload +func (s *Step) SetPayload(obj any) error { + result, err := json.Marshal(obj) + if err != nil { + return err + } + s.Payload = string(result) + return nil +} + +// GetStatus return step status +func (s *Step) GetStatus() string { + return s.Status +} + +// IsCompleted return step completed or not +func (s *Step) IsCompleted() bool { + // 已经完成 + if s.Status == TaskStatusSuccess { + return true + } + + // 失败需要看重试次数 + if s.Status == TaskStatusFailure { + // 还有重试次数 + if s.MaxRetries > 0 && s.RetryCount < s.MaxRetries { + return false + } + return true + } + + return false +} + +// SetStatus set status +func (s *Step) SetStatus(stat string) *Step { + s.Status = stat + return s +} + +// GetMessage get step message +func (s *Step) GetMessage() string { + return s.Message +} + +// SetMessage set step message +func (s *Step) SetMessage(msg string) *Step { + s.Message = msg + return s +} + +// GetSkipOnFailed get step skipOnFailed +func (s *Step) GetSkipOnFailed() bool { + return s.SkipOnFailed +} + +// SetSkipOnFailed set step skipOnFailed +func (s *Step) SetSkipOnFailed(skipOnFailed bool) *Step { + s.SkipOnFailed = skipOnFailed + return s +} + +// SetMaxTries set step max retry count +func (s *Step) SetMaxTries(count uint32) *Step { + s.MaxRetries = count + return s +} + +// GetRetryCount get step retry count +func (s *Step) GetRetryCount() uint32 { + return s.RetryCount +} + +// AddRetryCount add step retry count +func (s *Step) AddRetryCount(count uint32) *Step { + s.RetryCount += count + return s +} + +// SetCountdown step eta with countdown(seconds) +func (s *Step) SetCountdown(c int) *Step { + // 默认就是立即执行, 0值忽略 + if c <= 0 { + return s + } + + t := time.Now().Add(time.Duration(c) * time.Second) + s.ETA = &t + return s +} + +// SetETA step estimated time of arrival +func (s *Step) SetETA(t time.Time) *Step { + if t.Before(time.Now()) { + return s + } + + s.ETA = &t + return s +} + +// GetStartTime get start time +func (s *Step) GetStartTime() time.Time { + return s.Start +} + +// SetStartTime update start time +func (s *Step) SetStartTime(t time.Time) *Step { + s.Start = t + return s +} + +// GetEndTime get end time +func (s *Step) GetEndTime() time.Time { + return s.End +} + +// SetEndTime set end time +func (s *Step) SetEndTime(t time.Time) *Step { + // set end time + s.End = t + return s +} + +// GetExecutionTime set execution time +func (s *Step) GetExecutionTime() time.Duration { + return time.Duration(s.ExecutionTime) * time.Millisecond +} + +// SetExecutionTime set execution time +func (s *Step) SetExecutionTime(start time.Time, end time.Time) *Step { + s.ExecutionTime = uint32(end.Sub(start).Milliseconds()) + return s +} + +// GetMaxExecution get max execution seconds +func (s *Step) GetMaxExecution() time.Duration { + return time.Duration(s.MaxExecutionSeconds) * time.Second +} + +// SetMaxExecution set max execution seconds +func (s *Step) SetMaxExecution(duration time.Duration) *Step { + s.MaxExecutionSeconds = uint32(duration.Seconds()) + return s +} + +// GetLastUpdate get last update time +func (s *Step) GetLastUpdate() time.Time { + return s.LastUpdate +} + +// SetLastUpdate set last update time +func (s *Step) SetLastUpdate(t time.Time) *Step { + s.LastUpdate = t + return s +} diff --git a/task/types/task.go b/task/types/task.go new file mode 100644 index 0000000..1d1ceab --- /dev/null +++ b/task/types/task.go @@ -0,0 +1,362 @@ +/* + * Tencent is pleased to support the open source community by making Blueking Container Service available. + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * Licensed under the MIT License (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * http://opensource.org/licenses/MIT + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package types for task +package types + +import ( + "encoding/json" + "fmt" + "time" + + "github.com/google/uuid" +) + +// TaskBuilder ... +type TaskBuilder interface { // nolint + TaskInfo() TaskInfo + Steps() ([]*Step, error) // Steps init step and define StepSequence + FinalizeTask(t *Task) error // FinalizeTask for custom task +} + +// TaskOptions xxx +type TaskOptions struct { + CallbackName string + MaxExecutionSeconds uint32 +} + +// TaskOption xxx +type TaskOption func(opt *TaskOptions) + +// WithTaskCallback xxx +func WithTaskCallback(callbackName string) TaskOption { + return func(opt *TaskOptions) { + opt.CallbackName = callbackName + } +} + +// WithTaskMaxExecutionSeconds xxx +func WithTaskMaxExecutionSeconds(timeout uint32) TaskOption { + return func(opt *TaskOptions) { + opt.MaxExecutionSeconds = timeout + } +} + +// TaskInfo task basic info definition +type TaskInfo struct { + TaskType string + TaskName string + TaskIndex string // TaskIndex for resource index + TaskIndexType string + Creator string +} + +// NewTask create new task by default +func NewTask(o TaskInfo, opts ...TaskOption) *Task { + defaultOptions := &TaskOptions{CallbackName: "", MaxExecutionSeconds: 0} + for _, opt := range opts { + opt(defaultOptions) + } + + now := time.Now() + return &Task{ + TaskID: uuid.NewString(), + TaskType: o.TaskType, + TaskName: o.TaskName, + TaskIndex: o.TaskIndex, + TaskIndexType: o.TaskIndexType, + Status: TaskStatusInit, + Steps: make([]*Step, 0), + Creator: o.Creator, + Updater: o.Creator, + LastUpdate: now, + CommonParams: make(map[string]string, 0), + CommonPayload: DefaultPayloadContent, + CallbackName: defaultOptions.CallbackName, + Message: DefaultTaskMessage, + MaxExecutionSeconds: defaultOptions.MaxExecutionSeconds, + } +} + +// GetTaskID get task id +func (t *Task) GetTaskID() string { + return t.TaskID +} + +// GetTaskType get task type +func (t *Task) GetTaskType() string { + return t.TaskType +} + +// GetTaskName get task name +func (t *Task) GetTaskName() string { + return t.TaskName +} + +// GetTaskIndex get task index +func (t *Task) GetTaskIndex() string { + return t.TaskIndex +} + +// GetTaskIndexType get task index type +func (t *Task) GetTaskIndexType() string { + return t.TaskIndexType +} + +// GetStep get step by name +func (t *Task) GetStep(stepName string) (*Step, bool) { + for _, step := range t.Steps { + if step.Name == stepName { + return step, true + } + } + return nil, false +} + +// AddStep add step to task +func (t *Task) AddStep(step *Step) *Task { + if step == nil { + t.Steps = make([]*Step, 0) + } + + t.Steps = append(t.Steps, step) + return t +} + +// GetCommonParams return all common params +func (t *Task) GetCommonParams() map[string]string { + if t.CommonParams == nil { + t.CommonParams = make(map[string]string, 0) + } + return t.CommonParams +} + +// GetCommonParam get common params +func (t *Task) GetCommonParam(key string) (string, bool) { + if t.CommonParams == nil { + t.CommonParams = make(map[string]string, 0) + return "", false + } + if value, ok := t.CommonParams[key]; ok { + return value, true + } + return "", false +} + +// AddCommonParam add common params +func (t *Task) AddCommonParam(k, v string) *Task { + if t.CommonParams == nil { + t.CommonParams = make(map[string]string, 0) + } + t.CommonParams[k] = v + return t +} + +// GetCallback set callback function name +func (t *Task) GetCallback() string { + return t.CallbackName +} + +// SetCallback set callback function name +func (t *Task) SetCallback(callBackName string) *Task { + t.CallbackName = callBackName + return t +} + +// GetCommonPayload unmarshal common payload to struct obj +func (t *Task) GetCommonPayload(obj any) error { + if len(t.CommonPayload) == 0 { + t.CommonPayload = DefaultPayloadContent + } + return json.Unmarshal([]byte(t.CommonPayload), obj) +} + +// SetCommonPayload marshal struct obj to common payload +func (t *Task) SetCommonPayload(obj any) error { + result, err := json.Marshal(obj) + if err != nil { + return err + } + t.CommonPayload = string(result) + return nil +} + +// GetStatus get status +func (t *Task) GetStatus() string { + return t.Status +} + +// SetStatus set status +func (t *Task) SetStatus(status string) *Task { + t.Status = status + return t +} + +// GetMessage set message +func (t *Task) GetMessage() string { + return t.Message +} + +// SetMessage set message +func (t *Task) SetMessage(msg string) *Task { + t.Message = msg + return t +} + +// GetStartTime get start time +func (t *Task) GetStartTime() time.Time { + return t.Start +} + +// SetStartTime set start time +func (t *Task) SetStartTime(time time.Time) *Task { + t.Start = time + return t +} + +// GetEndTime get end time +func (t *Task) GetEndTime() time.Time { + return t.End +} + +// SetEndTime set end time +func (t *Task) SetEndTime(time time.Time) *Task { + t.End = time + return t +} + +// GetExecutionTime get execution time +func (t *Task) GetExecutionTime() time.Duration { + return time.Duration(t.ExecutionTime) +} + +// SetExecutionTime set execution time +func (t *Task) SetExecutionTime(start time.Time, end time.Time) *Task { + t.ExecutionTime = uint32(end.Sub(start).Milliseconds()) + return t +} + +// GetMaxExecution get max execution seconds +func (t *Task) GetMaxExecution() time.Duration { + return time.Duration(t.MaxExecutionSeconds) * time.Second +} + +// SetMaxExecution set max execution seconds +func (t *Task) SetMaxExecution(duration time.Duration) *Task { + t.MaxExecutionSeconds = uint32(duration.Seconds()) + return t +} + +// GetCreator get creator +func (t *Task) GetCreator() string { + return t.Creator +} + +// SetCreator set creator +func (t *Task) SetCreator(creator string) *Task { + t.Creator = creator + return t +} + +// GetUpdater get updater +func (t *Task) GetUpdater() string { + return t.Updater +} + +// SetUpdater set updater +func (t *Task) SetUpdater(updater string) *Task { + t.Updater = updater + return t +} + +// GetLastUpdate get last update time +func (t *Task) GetLastUpdate() (time.Time, error) { + return t.LastUpdate, nil +} + +// SetLastUpdate set last update time +func (t *Task) SetLastUpdate(lastUpdate time.Time) *Task { + t.LastUpdate = lastUpdate + return t +} + +// GetCurrentStep get current step +func (t *Task) GetCurrentStep() string { + return t.CurrentStep +} + +// SetCurrentStep set current step +func (t *Task) SetCurrentStep(stepName string) *Task { + t.CurrentStep = stepName + return t +} + +// GetStepParam get step params +func (t *Task) GetStepParam(stepName, key string) (string, bool) { + step, ok := t.GetStep(stepName) + if !ok { + return "", false + } + return step.GetParam(key) +} + +// AddStepParams add step params +func (t *Task) AddStepParams(stepName string, k, v string) error { + step, ok := t.GetStep(stepName) + if !ok { + return fmt.Errorf("step %s not exist", stepName) + } + step.AddParam(k, v) + return nil +} + +// AddStepParamsBatch add step params batch +func (t *Task) AddStepParamsBatch(stepName string, params map[string]string) error { + step, ok := t.GetStep(stepName) + if !ok { + return fmt.Errorf("step %s not exist", stepName) + } + + for k, v := range params { + step.AddParam(k, v) + } + return nil +} + +// Validate 校验 task +func (t *Task) Validate() error { + if t.TaskName == "" { + return fmt.Errorf("task name is required") + } + + if len(t.Steps) == 0 { + return fmt.Errorf("task steps empty") + } + + uniq := map[string]struct{}{} + for _, s := range t.Steps { + if s.Name == "" { + return fmt.Errorf("step name is required") + } + + if s.Executor == "" { + return fmt.Errorf("step executor is required") + } + + if _, ok := uniq[s.Name]; ok { + return fmt.Errorf("step name %s is not unique", s.Name) + } + uniq[s.Name] = struct{}{} + } + + return nil +} diff --git a/task/types/type.go b/task/types/type.go new file mode 100644 index 0000000..40d7070 --- /dev/null +++ b/task/types/type.go @@ -0,0 +1,113 @@ +/* + * Tencent is pleased to support the open source community by making Blueking Container Service available. + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * Licensed under the MIT License (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * http://opensource.org/licenses/MIT + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package types for task +package types + +import ( + "errors" + "time" +) + +const ( + // TaskTimeFormat task time format, e.g. 2006-01-02T15:04:05Z07:00 + TaskTimeFormat = time.RFC3339 + + // DefaultMaxExecuteTimeSeconds default max execute time for 1 hour + DefaultMaxExecuteTimeSeconds = 3600 + // DefaultTaskMessage default message + DefaultTaskMessage = "task initializing" + // DefaultPayloadContent default json extras content + DefaultPayloadContent = "{}" +) + +const ( + // TaskStatusInit INIT task status + TaskStatusInit = "INITIALIZING" + // TaskStatusRunning running task status + TaskStatusRunning = "RUNNING" + // TaskStatusSuccess task success + TaskStatusSuccess = "SUCCESS" + // TaskStatusFailure task failed + TaskStatusFailure = "FAILURE" + // TaskStatusTimeout task run timeout + TaskStatusTimeout = "TIMEOUT" + // TaskStatusRevoked task has been revoked + TaskStatusRevoked = "REVOKED" + // TaskStatusNotStarted force task terminate + TaskStatusNotStarted = "NOTSTARTED" +) + +var ( + // ErrNotImplemented not implemented error + ErrNotImplemented = errors.New("not implemented") +) + +// Task task definition +type Task struct { + TaskIndex string `json:"taskIndex"` + TaskIndexType string `json:"taskIndexType"` + TaskID string `json:"taskID"` + TaskType string `json:"taskType"` + TaskName string `json:"taskName"` + CurrentStep string `json:"currentStep"` + Steps []*Step `json:"steps"` + CallbackName string `json:"callbackName"` + CommonParams map[string]string `json:"commonParams"` + CommonPayload string `json:"commonPayload"` + Status string `json:"status"` + Message string `json:"message"` + ExecutionTime uint32 `json:"executionTime"` + MaxExecutionSeconds uint32 `json:"maxExecutionSeconds"` + Creator string `json:"creator"` + Updater string `json:"updater"` + Start time.Time `json:"start"` + End time.Time `json:"end"` + CreatedAt time.Time `json:"createdAt"` + LastUpdate time.Time `json:"lastUpdate"` +} + +// Step step definition +type Step struct { + Name string `json:"name"` + Alias string `json:"alias"` + Executor string `json:"executor"` + Params map[string]string `json:"params"` + Payload string `json:"payload"` + Status string `json:"status"` + Message string `json:"message"` + ETA *time.Time `json:"eta"` // 延迟执行时间(Estimated Time of Arrival) + SkipOnFailed bool `json:"skipOnFailed"` + RetryCount uint32 `json:"retryCount"` + MaxRetries uint32 `json:"maxRetries"` + ExecutionTime uint32 `json:"executionTime"` + MaxExecutionSeconds uint32 `json:"maxExecutionSeconds"` + Start time.Time `json:"start"` + End time.Time `json:"end"` + LastUpdate time.Time `json:"lastUpdate"` +} + +// TaskType taskType +type TaskType string // nolint + +// String toString +func (tt TaskType) String() string { + return string(tt) +} + +// TaskName xxx +type TaskName string // nolint + +// String xxx +func (tn TaskName) String() string { + return string(tn) +} diff --git a/task/utils.go b/task/utils.go new file mode 100644 index 0000000..88baf9e --- /dev/null +++ b/task/utils.go @@ -0,0 +1,113 @@ +/* + * Tencent is pleased to support the open source community by making Blueking Container Service available. + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * Licensed under the MIT License (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * http://opensource.org/licenses/MIT + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language governing permissions and + * limitations under the License. + */ + +package task + +import ( + "context" + "errors" + "runtime/debug" + "time" + + "github.com/RichardKnop/machinery/v2/log" + "github.com/RichardKnop/machinery/v2/retry" +) + +// RecoverPrintStack capture panic and print stack +func RecoverPrintStack(proc string) { + if r := recover(); r != nil { + log.ERROR.Printf("[%s][recover] panic: %v, stack %s", proc, r, debug.Stack()) + return + } +} + +// GetTimeOutCtx get timeout context +func GetTimeOutCtx(ctx context.Context, seconds uint32) (context.Context, context.CancelFunc) { + if seconds > 0 { + return context.WithTimeout(ctx, time.Duration(seconds)*time.Second) + } + + return context.WithCancel(ctx) +} + +// GetDeadlineCtx get daedline context +func GetDeadlineCtx(ctx context.Context, t *time.Time, seconds uint32) (context.Context, context.CancelFunc) { + if t == nil || seconds <= 0 { + return context.WithCancel(ctx) + } + + return context.WithDeadline(context.Background(), t.Add(time.Duration(seconds)*time.Second)) +} + +var ( + // ErrEndLoop xxx + ErrEndLoop = errors.New("end loop") +) + +// LoopOption init LoopOptions +type LoopOption func(loop *LoopOptions) + +// LoopInterval set LoopOptions interval parameter +func LoopInterval(duration time.Duration) LoopOption { + return func(loop *LoopOptions) { + if duration != 0 { + loop.interval = duration + } + } +} + +// LoopOptions loop parameter +type LoopOptions struct { + interval time.Duration +} + +// LoopDoFunc execute func do for interval +func LoopDoFunc(ctx context.Context, do func() error, ops ...LoopOption) error { + opt := &LoopOptions{interval: time.Second} + + for _, o := range ops { + o(opt) + } + + coldStart := make(chan struct{}, 1) + coldStart <- struct{}{} + + tick := time.NewTicker(opt.interval) + defer tick.Stop() + for { + select { + case <-coldStart: + case <-tick.C: + case <-ctx.Done(): + if errors.Is(ctx.Err(), context.Canceled) { + log.ERROR.Printf("LoopDoFunc is canceled") + } + return ctx.Err() + } + + if err := do(); err != nil { + if errors.Is(err, ErrEndLoop) { + return nil + } + return err + } + } +} + +// retryNext 计算重试时间, 基于Fibonacci +func retryNext(count int) int { + start := 1 + for i := 0; i < count; i++ { + start = retry.FibonacciNext(start) + } + return start +} diff --git a/task/utils_test.go b/task/utils_test.go new file mode 100644 index 0000000..a59956e --- /dev/null +++ b/task/utils_test.go @@ -0,0 +1,47 @@ +/* + * Tencent is pleased to support the open source community by making Blueking Container Service available. + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * Licensed under the MIT License (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * http://opensource.org/licenses/MIT + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language governing permissions and + * limitations under the License. + */ + +package task + +import ( + "fmt" + "testing" +) + +func TestRetryIn(t *testing.T) { + tests := []struct { + count int + expected int + }{ + {-1, 1}, + {0, 1}, + {1, 2}, + {2, 3}, + {3, 5}, + {4, 8}, + {5, 13}, + {6, 21}, + {7, 34}, + {8, 55}, + {9, 89}, + {10, 144}, + } + + for _, tt := range tests { + t.Run(fmt.Sprintf("count=%d", tt.count), func(t *testing.T) { + result := retryNext(tt.count) + if result != tt.expected { + t.Errorf("retryNext(%d) = %d; want %d", tt.count, result, tt.expected) + } + }) + } +}