Add a retry mechanism to solve situations where the device is deployed before the devicemodel.

Signed-off-by: luomengY <2938893385@qq.com>

16 files changed, 1020 insertions, 3 deletions

diff --git a/LICENSES/vendor/github.com/avast/retry-go/LICENSE b/LICENSES/vendor/github.com/avast/retry-go/LICENSE
new file mode 100644
index 000000000..2cccf8ece
--- /dev/null
+++ b/LICENSES/vendor/github.com/avast/retry-go/LICENSE
@@ -0,0 +1,25 @@
+= vendor/github.com/avast/retry-go licensed under: =
+
+MIT License
+
+Copyright (c) 2017 Avast
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+= vendor/github.com/avast/retry-go/LICENSE 323e2d10dd9a6e8b52a202601e19f35c
diff --git a/cloud/pkg/devicecontroller/controller/downstream.go b/cloud/pkg/devicecontroller/controller/downstream.go
index 43793fddd..107787048 100644
--- a/cloud/pkg/devicecontroller/controller/downstream.go
+++ b/cloud/pkg/devicecontroller/controller/downstream.go
@@ -17,10 +17,12 @@ limitations under the License.
 package controller
 
 import (
+	"fmt"
 	"reflect"
 	"sync"
 	"time"
 
+	"github.com/avast/retry-go"
 	"github.com/google/uuid"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/watch"
@@ -308,9 +310,22 @@ func (dc *DownstreamController) sendDeviceModelMsg(device *v1beta1.Device, opera
 	if device == nil || device.Spec.DeviceModelRef == nil {
 		return
 	}
-	edgeDeviceModel, ok := dc.deviceModelManager.DeviceModel.Load(device.Spec.DeviceModelRef.Name)
-	if !ok {
-		klog.Warningf("not found device model for device: %s, operation: %s", device.Name, operation)
+	var edgeDeviceModel any
+	var ok bool
+	err := retry.Do(
+		func() error {
+			edgeDeviceModel, ok = dc.deviceModelManager.DeviceModel.Load(device.Spec.DeviceModelRef.Name)
+			if !ok {
+				return fmt.Errorf("not found device model for device: %s, operation: %s", device.Name, operation)
+			}
+			return nil
+		},
+		retry.Delay(1*time.Second),
+		retry.Attempts(10),
+		retry.DelayType(retry.FixedDelay),
+	)
+	if err != nil {
+		klog.Warningf(err.Error())
 		return
 	}
 
diff --git a/go.mod b/go.mod
index 7f5daa362..591af8f8b 100644
--- a/go.mod
+++ b/go.mod
@@ -56,6 +56,7 @@ require (
 
 require (
 	github.com/agiledragon/gomonkey v2.0.2+incompatible
+	github.com/avast/retry-go v3.0.0+incompatible
 	github.com/beego/beego v1.12.12
 	github.com/onsi/ginkgo/v2 v2.9.5
 	github.com/pkg/errors v0.9.1
diff --git a/go.sum b/go.sum
index 18c60305f..33ee8f748 100644
--- a/go.sum
+++ b/go.sum
@@ -142,6 +142,8 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkY
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
 github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 h1:4daAzAu0S6Vi7/lbWECcX0j45yZReDZ56BQsrVBOEEY=
 github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg=
+github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0=
+github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY=
 github.com/aws/aws-sdk-go v1.35.24/go.mod h1:tlPOdRjfxPBpNIwqDj61rmsnA85v9jc0Ps9+muhnW+k=
 github.com/beego/beego v1.12.12 h1:ARY1sNVSS23N0mEQIhSqRDTyyDlx95JY0V3GogBbZbQ=
 github.com/beego/beego v1.12.12/go.mod h1:QURFL1HldOcCZAxnc1cZ7wrplsYR5dKPHFjmk6WkLAs=
diff --git a/vendor/github.com/avast/retry-go/.gitignore b/vendor/github.com/avast/retry-go/.gitignore
new file mode 100644
index 000000000..c40eb23f9
--- /dev/null
+++ b/vendor/github.com/avast/retry-go/.gitignore
@@ -0,0 +1,21 @@
+# Binaries for programs and plugins
+*.exe
+*.dll
+*.so
+*.dylib
+
+# Test binary, build with `go test -c`
+*.test
+
+# Output of the go coverage tool, specifically when used with LiteIDE
+*.out
+
+# Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736
+.glide/
+
+# dep
+vendor/
+Gopkg.lock
+
+# cover
+coverage.txt
diff --git a/vendor/github.com/avast/retry-go/.godocdown.tmpl b/vendor/github.com/avast/retry-go/.godocdown.tmpl
new file mode 100644
index 000000000..6873edf8e
--- /dev/null
+++ b/vendor/github.com/avast/retry-go/.godocdown.tmpl
@@ -0,0 +1,37 @@
+# {{ .Name }}
+
+[![Release](https://img.shields.io/github/release/avast/retry-go.svg?style=flat-square)](https://github.com/avast/retry-go/releases/latest)
+[![Software License](https://img.shields.io/badge/license-MIT-brightgreen.svg?style=flat-square)](LICENSE.md)
+[![Travis](https://img.shields.io/travis/avast/retry-go.svg?style=flat-square)](https://travis-ci.org/avast/retry-go)
+[![AppVeyor](https://ci.appveyor.com/api/projects/status/fieg9gon3qlq0a9a?svg=true)](https://ci.appveyor.com/project/JaSei/retry-go)
+[![Go Report Card](https://goreportcard.com/badge/github.com/avast/retry-go?style=flat-square)](https://goreportcard.com/report/github.com/avast/retry-go)
+[![GoDoc](https://godoc.org/github.com/avast/retry-go?status.svg&style=flat-square)](http://godoc.org/github.com/avast/retry-go)
+[![codecov.io](https://codecov.io/github/avast/retry-go/coverage.svg?branch=master)](https://codecov.io/github/avast/retry-go?branch=master)
+[![Sourcegraph](https://sourcegraph.com/github.com/avast/retry-go/-/badge.svg)](https://sourcegraph.com/github.com/avast/retry-go?badge)
+
+{{ .EmitSynopsis }}
+
+{{ .EmitUsage }}
+
+## Contributing
+
+Contributions are very much welcome.
+
+### Makefile
+
+Makefile provides several handy rules, like README.md `generator` , `setup` for prepare build/dev environment, `test`, `cover`, etc...
+
+Try `make help` for more information.
+
+### Before pull request
+
+please try:
+* run tests (`make test`)
+* run linter (`make lint`)
+* if your IDE don't automaticaly do `go fmt`, run `go fmt` (`make fmt`)
+
+### README
+
+README.md are generate from template [.godocdown.tmpl](.godocdown.tmpl) and code documentation via [godocdown](https://github.com/robertkrimen/godocdown).
+
+Never edit README.md direct, because your change will be lost.
diff --git a/vendor/github.com/avast/retry-go/.travis.yml b/vendor/github.com/avast/retry-go/.travis.yml
new file mode 100644
index 000000000..ae3e0b688
--- /dev/null
+++ b/vendor/github.com/avast/retry-go/.travis.yml
@@ -0,0 +1,20 @@
+language: go
+
+go:
+  - 1.8
+  - 1.9
+  - "1.10"
+  - 1.11
+  - 1.12
+  - 1.13
+  - 1.14
+  - 1.15
+
+install:
+  - make setup
+
+script:
+  - make ci
+
+after_success:
+  - bash <(curl -s https://codecov.io/bash)
diff --git a/vendor/github.com/avast/retry-go/Gopkg.toml b/vendor/github.com/avast/retry-go/Gopkg.toml
new file mode 100644
index 000000000..cf8c9eb0e
--- /dev/null
+++ b/vendor/github.com/avast/retry-go/Gopkg.toml
@@ -0,0 +1,3 @@
+[[constraint]]
+  name = "github.com/stretchr/testify"
+  version = "1.1.4"
diff --git a/vendor/github.com/avast/retry-go/LICENSE b/vendor/github.com/avast/retry-go/LICENSE
new file mode 100644
index 000000000..f63fca814
--- /dev/null
+++ b/vendor/github.com/avast/retry-go/LICENSE
@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2017 Avast
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/vendor/github.com/avast/retry-go/Makefile b/vendor/github.com/avast/retry-go/Makefile
new file mode 100644
index 000000000..769816d24
--- /dev/null
+++ b/vendor/github.com/avast/retry-go/Makefile
@@ -0,0 +1,65 @@
+SOURCE_FILES?=$$(go list ./... | grep -v /vendor/)
+TEST_PATTERN?=.
+TEST_OPTIONS?=
+DEP?=$$(which dep)
+VERSION?=$$(cat VERSION)
+LINTER?=$$(which golangci-lint)
+LINTER_VERSION=1.15.0
+
+ifeq ($(OS),Windows_NT)
+	DEP_VERS=dep-windows-amd64
+	LINTER_FILE=golangci-lint-$(LINTER_VERSION)-windows-amd64.zip
+	LINTER_UNPACK= >| app.zip; unzip -j app.zip -d $$GOPATH/bin; rm app.zip
+else ifeq ($(OS), Darwin)
+	LINTER_FILE=golangci-lint-$(LINTER_VERSION)-darwin-amd64.tar.gz
+	LINTER_UNPACK= | tar xzf - -C $$GOPATH/bin --wildcards --strip 1 "**/golangci-lint"
+else
+	DEP_VERS=dep-linux-amd64
+	LINTER_FILE=golangci-lint-$(LINTER_VERSION)-linux-amd64.tar.gz
+	LINTER_UNPACK= | tar xzf - -C $$GOPATH/bin --wildcards --strip 1 "**/golangci-lint"
+endif
+
+setup:
+	go get -u github.com/pierrre/gotestcover
+	go get -u golang.org/x/tools/cmd/cover
+	go get -u github.com/robertkrimen/godocdown/godocdown
+	@if [ "$(LINTER)" = "" ]; then\
+		curl -L https://github.com/golangci/golangci-lint/releases/download/v$(LINTER_VERSION)/$(LINTER_FILE) $(LINTER_UNPACK) ;\
+		chmod +x $$GOPATH/bin/golangci-lint;\
+	fi
+	@if [ "$(DEP)" = "" ]; then\
+		curl -L https://github.com/golang/dep/releases/download/v0.3.1/$(DEP_VERS) >| $$GOPATH/bin/dep;\
+		chmod +x $$GOPATH/bin/dep;\
+	fi
+	dep ensure
+
+generate: ## Generate README.md
+	godocdown >| README.md
+
+test: generate test_and_cover_report lint
+
+test_and_cover_report:
+	gotestcover $(TEST_OPTIONS) -covermode=atomic -coverprofile=coverage.txt $(SOURCE_FILES) -run $(TEST_PATTERN) -timeout=2m
+
+cover: test ## Run all the tests and opens the coverage report
+	go tool cover -html=coverage.txt
+
+fmt: ## gofmt and goimports all go files
+	find . -name '*.go' -not -wholename './vendor/*' | while read -r file; do gofmt -w -s "$$file"; goimports -w "$$file"; done
+
+lint: ## Run all the linters
+	golangci-lint run
+
+ci: test_and_cover_report ## Run all the tests but no linters - use https://golangci.com integration instead
+
+build:
+	go build
+
+release: ## Release new version
+	git tag | grep -q $(VERSION) && echo This version was released! Increase VERSION! || git tag $(VERSION) && git push origin $(VERSION) && git tag v$(VERSION) && git push origin v$(VERSION)
+
+# Absolutely awesome: http://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
+help:
+	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
+
+.DEFAULT_GOAL := build
diff --git a/vendor/github.com/avast/retry-go/README.md b/vendor/github.com/avast/retry-go/README.md
new file mode 100644
index 000000000..80fb73b4b
--- /dev/null
+++ b/vendor/github.com/avast/retry-go/README.md
@@ -0,0 +1,361 @@
+# retry
+
+[![Release](https://img.shields.io/github/release/avast/retry-go.svg?style=flat-square)](https://github.com/avast/retry-go/releases/latest)
+[![Software License](https://img.shields.io/badge/license-MIT-brightgreen.svg?style=flat-square)](LICENSE.md)
+[![Travis](https://img.shields.io/travis/avast/retry-go.svg?style=flat-square)](https://travis-ci.org/avast/retry-go)
+[![AppVeyor](https://ci.appveyor.com/api/projects/status/fieg9gon3qlq0a9a?svg=true)](https://ci.appveyor.com/project/JaSei/retry-go)
+[![Go Report Card](https://goreportcard.com/badge/github.com/avast/retry-go?style=flat-square)](https://goreportcard.com/report/github.com/avast/retry-go)
+[![GoDoc](https://godoc.org/github.com/avast/retry-go?status.svg&style=flat-square)](http://godoc.org/github.com/avast/retry-go)
+[![codecov.io](https://codecov.io/github/avast/retry-go/coverage.svg?branch=master)](https://codecov.io/github/avast/retry-go?branch=master)
+[![Sourcegraph](https://sourcegraph.com/github.com/avast/retry-go/-/badge.svg)](https://sourcegraph.com/github.com/avast/retry-go?badge)
+
+Simple library for retry mechanism
+
+slightly inspired by
+[Try::Tiny::Retry](https://metacpan.org/pod/Try::Tiny::Retry)
+
+
+### SYNOPSIS
+
+http get with retry:
+
+    url := "http://example.com"
+    var body []byte
+
+    err := retry.Do(
+    	func() error {
+    		resp, err := http.Get(url)
+    		if err != nil {
+    			return err
+    		}
+    		defer resp.Body.Close()
+    		body, err = ioutil.ReadAll(resp.Body)
+    		if err != nil {
+    			return err
+    		}
+
+    		return nil
+    	},
+    )
+
+    fmt.Println(body)
+
+[next examples](https://github.com/avast/retry-go/tree/master/examples)
+
+
+### SEE ALSO
+
+* [giantswarm/retry-go](https://github.com/giantswarm/retry-go) - slightly
+complicated interface.
+
+* [sethgrid/pester](https://github.com/sethgrid/pester) - only http retry for
+http calls with retries and backoff
+
+* [cenkalti/backoff](https://github.com/cenkalti/backoff) - Go port of the
+exponential backoff algorithm from Google's HTTP Client Library for Java. Really
+complicated interface.
+
+* [rafaeljesus/retry-go](https://github.com/rafaeljesus/retry-go) - looks good,
+slightly similar as this package, don't have 'simple' `Retry` method
+
+* [matryer/try](https://github.com/matryer/try) - very popular package,
+nonintuitive interface (for me)
+
+
+### BREAKING CHANGES
+
+3.0.0
+
+* `DelayTypeFunc` accepts a new parameter `err` - this breaking change affects
+only your custom Delay Functions. This change allow [make delay functions based
+on error](examples/delay_based_on_error_test.go).
+
+1.0.2 -> 2.0.0
+
+* argument of `retry.Delay` is final delay (no multiplication by `retry.Units`
+anymore)
+
+* function `retry.Units` are removed
+
+* [more about this breaking change](https://github.com/avast/retry-go/issues/7)
+
+0.3.0 -> 1.0.0
+
+* `retry.Retry` function are changed to `retry.Do` function
+
+* `retry.RetryCustom` (OnRetry) and `retry.RetryCustomWithOpts` functions are
+now implement via functions produces Options (aka `retry.OnRetry`)
+
+## Usage
+
+```go
+var (
+	DefaultAttempts      = uint(10)
+	DefaultDelay         = 100 * time.Millisecond
+	DefaultMaxJitter     = 100 * time.Millisecond
+	DefaultOnRetry       = func(n uint, err error) {}
+	DefaultRetryIf       = IsRecoverable
+	DefaultDelayType     = CombineDelay(BackOffDelay, RandomDelay)
+	DefaultLastErrorOnly = false
+	DefaultContext       = context.Background()
+)
+```
+
+#### func  BackOffDelay
+
+```go
+func BackOffDelay(n uint, _ error, config *Config) time.Duration
+```
+BackOffDelay is a DelayType which increases delay between consecutive retries
+
+#### func  Do
+
+```go
+func Do(retryableFunc RetryableFunc, opts ...Option) error
+```
+
+#### func  FixedDelay
+
+```go
+func FixedDelay(_ uint, _ error, config *Config) time.Duration
+```
+FixedDelay is a DelayType which keeps delay the same through all iterations
+
+#### func  IsRecoverable
+
+```go
+func IsRecoverable(err error) bool
+```
+IsRecoverable checks if error is an instance of `unrecoverableError`
+
+#### func  RandomDelay
+
+```go
+func RandomDelay(_ uint, _ error, config *Config) time.Duration
+```
+RandomDelay is a DelayType which picks a random delay up to config.maxJitter
+
+#### func  Unrecoverable
+
+```go
+func Unrecoverable(err error) error
+```
+Unrecoverable wraps an error in `unrecoverableError` struct
+
+#### type Config
+
+```go
+type Config struct {
+}
+```
+
+
+#### type DelayTypeFunc
+
+```go
+type DelayTypeFunc func(n uint, err error, config *Config) time.Duration
+```
+
+DelayTypeFunc is called to return the next delay to wait after the retriable
+function fails on `err` after `n` attempts.
+
+#### func  CombineDelay
+
+```go
+func CombineDelay(delays ...DelayTypeFunc) DelayTypeFunc
+```
+CombineDelay is a DelayType the combines all of the specified delays into a new
+DelayTypeFunc
+
+#### type Error
+
+```go
+type Error []error
+```
+
+Error type represents list of errors in retry
+
+#### func (Error) Error
+
+```go
+func (e Error) Error() string
+```
+Error method return string representation of Error It is an implementation of
+error interface
+
+#### func (Error) WrappedErrors
+
+```go
+func (e Error) WrappedErrors() []error
+```
+WrappedErrors returns the list of errors that this Error is wrapping. It is an
+implementation of the `errwrap.Wrapper` interface in package
+[errwrap](https://github.com/hashicorp/errwrap) so that `retry.Error` can be
+used with that library.
+
+#### type OnRetryFunc
+
+```go
+type OnRetryFunc func(n uint, err error)
+```
+
+Function signature of OnRetry function n = count of attempts
+
+#### type Option
+
+```go
+type Option func(*Config)
+```
+
+Option represents an option for retry.
+
+#### func  Attempts
+
+```go
+func Attempts(attempts uint) Option
+```
+Attempts set count of retry default is 10
+
+#### func  Context
+
+```go
+func Context(ctx context.Context) Option
+```
+Context allow to set context of retry default are Background context
+
+example of immediately cancellation (maybe it isn't the best example, but it
+describes behavior enough; I hope)
+
+    ctx, cancel := context.WithCancel(context.Background())
+    cancel()
+
+    retry.Do(
+    	func() error {
+    		...
+    	},
+    	retry.Context(ctx),
+    )
+
+#### func  Delay
+
+```go
+func Delay(delay time.Duration) Option
+```
+Delay set delay between retry default is 100ms
+
+#### func  DelayType
+
+```go
+func DelayType(delayType DelayTypeFunc) Option
+```
+DelayType set type of the delay between retries default is BackOff
+
+#### func  LastErrorOnly
+
+```go
+func LastErrorOnly(lastErrorOnly bool) Option
+```
+return the direct last error that came from the retried function default is
+false (return wrapped errors with everything)
+
+#### func  MaxDelay
+
+```go
+func MaxDelay(maxDelay time.Duration) Option
+```
+MaxDelay set maximum delay between retry does not apply by default
+
+#### func  MaxJitter
+
+```go
+func MaxJitter(maxJitter time.Duration) Option
+```
+MaxJitter sets the maximum random Jitter between retries for RandomDelay
+
+#### func  OnRetry
+
+```go
+func OnRetry(onRetry OnRetryFunc) Option
+```
+OnRetry function callback are called each retry
+
+log each retry example:
+
+    retry.Do(
+    	func() error {
+    		return errors.New("some error")
+    	},
+    	retry.OnRetry(func(n uint, err error) {
+    		log.Printf("#%d: %s\n", n, err)
+    	}),
+    )
+
+#### func  RetryIf
+
+```go
+func RetryIf(retryIf RetryIfFunc) Option
+```
+RetryIf controls whether a retry should be attempted after an error (assuming
+there are any retry attempts remaining)
+
+skip retry if special error example:
+
+    retry.Do(
+    	func() error {
+    		return errors.New("special error")
+    	},
+    	retry.RetryIf(func(err error) bool {
+    		if err.Error() == "special error" {
+    			return false
+    		}
+    		return true
+    	})
+    )
+
+By default RetryIf stops execution if the error is wrapped using
+`retry.Unrecoverable`, so above example may also be shortened to:
+
+    retry.Do(
+    	func() error {
+    		return retry.Unrecoverable(errors.New("special error"))
+    	}
+    )
+
+#### type RetryIfFunc
+
+```go
+type RetryIfFunc func(error) bool
+```
+
+Function signature of retry if function
+
+#### type RetryableFunc
+
+```go
+type RetryableFunc func() error
+```
+
+Function signature of retryable function
+
+## Contributing
+
+Contributions are very much welcome.
+
+### Makefile
+
+Makefile provides several handy rules, like README.md `generator` , `setup` for prepare build/dev environment, `test`, `cover`, etc...
+
+Try `make help` for more information.
+
+### Before pull request
+
+please try:
+* run tests (`make test`)
+* run linter (`make lint`)
+* if your IDE don't automaticaly do `go fmt`, run `go fmt` (`make fmt`)
+
+### README
+
+README.md are generate from template [.godocdown.tmpl](.godocdown.tmpl) and code documentation via [godocdown](https://github.com/robertkrimen/godocdown).
+
+Never edit README.md direct, because your change will be lost.
diff --git a/vendor/github.com/avast/retry-go/VERSION b/vendor/github.com/avast/retry-go/VERSION
new file mode 100644
index 000000000..4a36342fc
--- /dev/null
+++ b/vendor/github.com/avast/retry-go/VERSION
@@ -0,0 +1 @@
+3.0.0
diff --git a/vendor/github.com/avast/retry-go/appveyor.yml b/vendor/github.com/avast/retry-go/appveyor.yml
new file mode 100644
index 000000000..dc5234ac8
--- /dev/null
+++ b/vendor/github.com/avast/retry-go/appveyor.yml
@@ -0,0 +1,19 @@
+version: "{build}"
+
+clone_folder: c:\Users\appveyor\go\src\github.com\avast\retry-go
+
+#os: Windows Server 2012 R2
+platform: x64
+
+install:
+  - copy c:\MinGW\bin\mingw32-make.exe c:\MinGW\bin\make.exe
+  - set GOPATH=C:\Users\appveyor\go
+  - set PATH=%PATH%;c:\MinGW\bin
+  - set PATH=%PATH%;%GOPATH%\bin;c:\go\bin
+  - set GOBIN=%GOPATH%\bin
+  - go version
+  - go env
+  - make setup
+
+build_script:
+  - make ci
diff --git a/vendor/github.com/avast/retry-go/options.go b/vendor/github.com/avast/retry-go/options.go
new file mode 100644
index 000000000..a6c57207c
--- /dev/null
+++ b/vendor/github.com/avast/retry-go/options.go
@@ -0,0 +1,198 @@
+package retry
+
+import (
+	"context"
+	"math"
+	"math/rand"
+	"time"
+)
+
+// Function signature of retry if function
+type RetryIfFunc func(error) bool
+
+// Function signature of OnRetry function
+// n = count of attempts
+type OnRetryFunc func(n uint, err error)
+
+// DelayTypeFunc is called to return the next delay to wait after the retriable function fails on `err` after `n` attempts.
+type DelayTypeFunc func(n uint, err error, config *Config) time.Duration
+
+type Config struct {
+	attempts      uint
+	delay         time.Duration
+	maxDelay      time.Duration
+	maxJitter     time.Duration
+	onRetry       OnRetryFunc
+	retryIf       RetryIfFunc
+	delayType     DelayTypeFunc
+	lastErrorOnly bool
+	context       context.Context
+
+	maxBackOffN uint
+}
+
+// Option represents an option for retry.
+type Option func(*Config)
+
+// return the direct last error that came from the retried function
+// default is false (return wrapped errors with everything)
+func LastErrorOnly(lastErrorOnly bool) Option {
+	return func(c *Config) {
+		c.lastErrorOnly = lastErrorOnly
+	}
+}
+
+// Attempts set count of retry
+// default is 10
+func Attempts(attempts uint) Option {
+	return func(c *Config) {
+		c.attempts = attempts
+	}
+}
+
+// Delay set delay between retry
+// default is 100ms
+func Delay(delay time.Duration) Option {
+	return func(c *Config) {
+		c.delay = delay
+	}
+}
+
+// MaxDelay set maximum delay between retry
+// does not apply by default
+func MaxDelay(maxDelay time.Duration) Option {
+	return func(c *Config) {
+		c.maxDelay = maxDelay
+	}
+}
+
+// MaxJitter sets the maximum random Jitter between retries for RandomDelay
+func MaxJitter(maxJitter time.Duration) Option {
+	return func(c *Config) {
+		c.maxJitter = maxJitter
+	}
+}
+
+// DelayType set type of the delay between retries
+// default is BackOff
+func DelayType(delayType DelayTypeFunc) Option {
+	return func(c *Config) {
+		c.delayType = delayType
+	}
+}
+
+// BackOffDelay is a DelayType which increases delay between consecutive retries
+func BackOffDelay(n uint, _ error, config *Config) time.Duration {
+	// 1 << 63 would overflow signed int64 (time.Duration), thus 62.
+	const max uint = 62
+
+	if config.maxBackOffN == 0 {
+		if config.delay <= 0 {
+			config.delay = 1
+		}
+
+		config.maxBackOffN = max - uint(math.Floor(math.Log2(float64(config.delay))))
+	}
+
+	if n > config.maxBackOffN {
+		n = config.maxBackOffN
+	}
+
+	return config.delay << n
+}
+
+// FixedDelay is a DelayType which keeps delay the same through all iterations
+func FixedDelay(_ uint, _ error, config *Config) time.Duration {
+	return config.delay
+}
+
+// RandomDelay is a DelayType which picks a random delay up to config.maxJitter
+func RandomDelay(_ uint, _ error, config *Config) time.Duration {
+	return time.Duration(rand.Int63n(int64(config.maxJitter)))
+}
+
+// CombineDelay is a DelayType the combines all of the specified delays into a new DelayTypeFunc
+func CombineDelay(delays ...DelayTypeFunc) DelayTypeFunc {
+	const maxInt64 = uint64(math.MaxInt64)
+
+	return func(n uint, err error, config *Config) time.Duration {
+		var total uint64
+		for _, delay := range delays {
+			total += uint64(delay(n, err, config))
+			if total > maxInt64 {
+				total = maxInt64
+			}
+		}
+
+		return time.Duration(total)
+	}
+}
+
+// OnRetry function callback are called each retry
+//
+// log each retry example:
+//
+//	retry.Do(
+//		func() error {
+//			return errors.New("some error")
+//		},
+//		retry.OnRetry(func(n uint, err error) {
+//			log.Printf("#%d: %s\n", n, err)
+//		}),
+//	)
+func OnRetry(onRetry OnRetryFunc) Option {
+	return func(c *Config) {
+		c.onRetry = onRetry
+	}
+}
+
+// RetryIf controls whether a retry should be attempted after an error
+// (assuming there are any retry attempts remaining)
+//
+// skip retry if special error example:
+//
+//	retry.Do(
+//		func() error {
+//			return errors.New("special error")
+//		},
+//		retry.RetryIf(func(err error) bool {
+//			if err.Error() == "special error" {
+//				return false
+//			}
+//			return true
+//		})
+//	)
+//
+// By default RetryIf stops execution if the error is wrapped using `retry.Unrecoverable`,
+// so above example may also be shortened to:
+//
+//	retry.Do(
+//		func() error {
+//			return retry.Unrecoverable(errors.New("special error"))
+//		}
+//	)
+func RetryIf(retryIf RetryIfFunc) Option {
+	return func(c *Config) {
+		c.retryIf = retryIf
+	}
+}
+
+// Context allow to set context of retry
+// default are Background context
+//
+// example of immediately cancellation (maybe it isn't the best example, but it describes behavior enough; I hope)
+//
+//	ctx, cancel := context.WithCancel(context.Background())
+//	cancel()
+//
+//	retry.Do(
+//		func() error {
+//			...
+//		},
+//		retry.Context(ctx),
+//	)
+func Context(ctx context.Context) Option {
+	return func(c *Config) {
+		c.context = ctx
+	}
+}
diff --git a/vendor/github.com/avast/retry-go/retry.go b/vendor/github.com/avast/retry-go/retry.go
new file mode 100644
index 000000000..af2d92641
--- /dev/null
+++ b/vendor/github.com/avast/retry-go/retry.go
@@ -0,0 +1,225 @@
+/*
+Simple library for retry mechanism
+
+slightly inspired by [Try::Tiny::Retry](https://metacpan.org/pod/Try::Tiny::Retry)
+
+SYNOPSIS
+
+http get with retry:
+
+	url := "http://example.com"
+	var body []byte
+
+	err := retry.Do(
+		func() error {
+			resp, err := http.Get(url)
+			if err != nil {
+				return err
+			}
+			defer resp.Body.Close()
+			body, err = ioutil.ReadAll(resp.Body)
+			if err != nil {
+				return err
+			}
+
+			return nil
+		},
+	)
+
+	fmt.Println(body)
+
+[next examples](https://github.com/avast/retry-go/tree/master/examples)
+
+
+SEE ALSO
+
+* [giantswarm/retry-go](https://github.com/giantswarm/retry-go) - slightly complicated interface.
+
+* [sethgrid/pester](https://github.com/sethgrid/pester) - only http retry for http calls with retries and backoff
+
+* [cenkalti/backoff](https://github.com/cenkalti/backoff) - Go port of the exponential backoff algorithm from Google's HTTP Client Library for Java. Really complicated interface.
+
+* [rafaeljesus/retry-go](https://github.com/rafaeljesus/retry-go) - looks good, slightly similar as this package, don't have 'simple' `Retry` method
+
+* [matryer/try](https://github.com/matryer/try) - very popular package, nonintuitive interface (for me)
+
+
+BREAKING CHANGES
+
+3.0.0
+
+* `DelayTypeFunc` accepts a new parameter `err` - this breaking change affects only your custom Delay Functions. This change allow [make delay functions based on error](examples/delay_based_on_error_test.go).
+
+
+1.0.2 -> 2.0.0
+
+* argument of `retry.Delay` is final delay (no multiplication by `retry.Units` anymore)
+
+* function `retry.Units` are removed
+
+* [more about this breaking change](https://github.com/avast/retry-go/issues/7)
+
+
+0.3.0 -> 1.0.0
+
+* `retry.Retry` function are changed to `retry.Do` function
+
+* `retry.RetryCustom` (OnRetry) and `retry.RetryCustomWithOpts` functions are now implement via functions produces Options (aka `retry.OnRetry`)
+
+
+*/
+package retry
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+)
+
+// Function signature of retryable function
+type RetryableFunc func() error
+
+var (
+	DefaultAttempts      = uint(10)
+	DefaultDelay         = 100 * time.Millisecond
+	DefaultMaxJitter     = 100 * time.Millisecond
+	DefaultOnRetry       = func(n uint, err error) {}
+	DefaultRetryIf       = IsRecoverable
+	DefaultDelayType     = CombineDelay(BackOffDelay, RandomDelay)
+	DefaultLastErrorOnly = false
+	DefaultContext       = context.Background()
+)
+
+func Do(retryableFunc RetryableFunc, opts ...Option) error {
+	var n uint
+
+	//default
+	config := &Config{
+		attempts:      DefaultAttempts,
+		delay:         DefaultDelay,
+		maxJitter:     DefaultMaxJitter,
+		onRetry:       DefaultOnRetry,
+		retryIf:       DefaultRetryIf,
+		delayType:     DefaultDelayType,
+		lastErrorOnly: DefaultLastErrorOnly,
+		context:       DefaultContext,
+	}
+
+	//apply opts
+	for _, opt := range opts {
+		opt(config)
+	}
+
+	if err := config.context.Err(); err != nil {
+		return err
+	}
+
+	var errorLog Error
+	if !config.lastErrorOnly {
+		errorLog = make(Error, config.attempts)
+	} else {
+		errorLog = make(Error, 1)
+	}
+
+	lastErrIndex := n
+	for n < config.attempts {
+		err := retryableFunc()
+
+		if err != nil {
+			errorLog[lastErrIndex] = unpackUnrecoverable(err)
+
+			if !config.retryIf(err) {
+				break
+			}
+
+			config.onRetry(n, err)
+
+			// if this is last attempt - don't wait
+			if n == config.attempts-1 {
+				break
+			}
+
+			delayTime := config.delayType(n, err, config)
+			if config.maxDelay > 0 && delayTime > config.maxDelay {
+				delayTime = config.maxDelay
+			}
+
+			select {
+			case <-time.After(delayTime):
+			case <-config.context.Done():
+				return config.context.Err()
+			}
+
+		} else {
+			return nil
+		}
+
+		n++
+		if !config.lastErrorOnly {
+			lastErrIndex = n
+		}
+	}
+
+	if config.lastErrorOnly {
+		return errorLog[lastErrIndex]
+	}
+	return errorLog
+}
+
+// Error type represents list of errors in retry
+type Error []error
+
+// Error method return string representation of Error
+// It is an implementation of error interface
+func (e Error) Error() string {
+	logWithNumber := make([]string, lenWithoutNil(e))
+	for i, l := range e {
+		if l != nil {
+			logWithNumber[i] = fmt.Sprintf("#%d: %s", i+1, l.Error())
+		}
+	}
+
+	return fmt.Sprintf("All attempts fail:\n%s", strings.Join(logWithNumber, "\n"))
+}
+
+func lenWithoutNil(e Error) (count int) {
+	for _, v := range e {
+		if v != nil {
+			count++
+		}
+	}
+
+	return
+}
+
+// WrappedErrors returns the list of errors that this Error is wrapping.
+// It is an implementation of the `errwrap.Wrapper` interface
+// in package [errwrap](https://github.com/hashicorp/errwrap) so that
+// `retry.Error` can be used with that library.
+func (e Error) WrappedErrors() []error {
+	return e
+}
+
+type unrecoverableError struct {
+	error
+}
+
+// Unrecoverable wraps an error in `unrecoverableError` struct
+func Unrecoverable(err error) error {
+	return unrecoverableError{err}
+}
+
+// IsRecoverable checks if error is an instance of `unrecoverableError`
+func IsRecoverable(err error) bool {
+	_, isUnrecoverable := err.(unrecoverableError)
+	return !isUnrecoverable
+}
+
+func unpackUnrecoverable(err error) error {
+	if unrecoverable, isUnrecoverable := err.(unrecoverableError); isUnrecoverable {
+		return unrecoverable.error
+	}
+
+	return err
+}
diff --git a/vendor/modules.txt b/vendor/modules.txt
index a15b131e9..8835959cd 100644
--- a/vendor/modules.txt
+++ b/vendor/modules.txt
@@ -115,6 +115,9 @@ github.com/armon/circbuf
 # github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535
 ## explicit; go 1.12
 github.com/asaskevich/govalidator
+# github.com/avast/retry-go v3.0.0+incompatible
+## explicit
+github.com/avast/retry-go
 # github.com/beego/beego v1.12.12
 ## explicit; go 1.13
 github.com/beego/beego/orm