Skip to content

Commit

Permalink
Merge branch 'dev' into overflow-conversion-utilities
Browse files Browse the repository at this point in the history
  • Loading branch information
omerzi authored Sep 8, 2024
2 parents c198558 + d342a81 commit a95bde9
Show file tree
Hide file tree
Showing 18 changed files with 281 additions and 117 deletions.
27 changes: 8 additions & 19 deletions .github/workflows/analysis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,30 +13,19 @@ jobs:
- name: Checkout Source
uses: actions/checkout@v4

- name: Install Go
uses: actions/setup-go@v5
with:
go-version: 1.22.x

- name: Static Code Analysis
uses: golangci/golangci-lint-action@v3
with:
args: |
--timeout 5m --out-${NO_FUTURE}format colored-line-number --enable errcheck,gosimple,govet,ineffassign,staticcheck,typecheck,unused,gocritic,asasalint,asciicheck,errchkjson,exportloopref,makezero,nilerr,unparam,unconvert,wastedassign,usestdlibvars
- name: Setup Go with cache
uses: jfrog/.github/actions/install-go-with-cache@main

- name: Run golangci lint
uses: jfrog/.github/actions/golangci-lint@main
Go-Sec:
runs-on: ubuntu-latest
steps:
- name: Checkout Source
uses: actions/checkout@v4

- name: Install Go
uses: actions/setup-go@v5
with:
go-version: 1.22.x
- name: Setup Go with cache
uses: jfrog/.github/actions/install-go-with-cache@main

- name: Run Gosec Security Scanner
uses: securego/gosec@master
with:
args: -exclude G204,G304,G404,G401,G505 -tests -exclude-dir \.*test\.* ./...
- name: Run Go-Sec scanner
uses: jfrog/.github/actions/gosec-scanner@main
26 changes: 5 additions & 21 deletions .github/workflows/cla.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,26 +10,10 @@ jobs:
CLAssistant:
runs-on: ubuntu-latest
steps:
- uses: actions-ecosystem/action-regex-match@v2
id: sign-or-recheck
- name: Run CLA Check
uses: jfrog/.github/actions/cla@main
with:
text: ${{ github.event.comment.body }}
regex: '\s*(I have read the CLA Document and I hereby sign the CLA)|(recheck)\s*'

- name: "CLA Assistant"
if: ${{ steps.sign-or-recheck.outputs.match != '' || github.event_name == 'pull_request_target' }}
# Alpha Release
uses: cla-assistant/[email protected]
env:
# Generated and maintained by GitHub
event_comment_body: ${{ github.event.comment.body }}
event_name: ${{ github.event_name }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
# JFrog's organization secret
PERSONAL_ACCESS_TOKEN : ${{ secrets.CLA_SIGN_TOKEN }}
with:
path-to-signatures: 'signed_clas.json'
path-to-document: 'https://jfrog.com/cla/'
remote-organization-name: 'jfrog'
remote-repository-name: 'jfrog-signed-clas'
# branch should not be protected
branch: 'master'
allowlist: bot*
CLA_SIGN_TOKEN: ${{ secrets.CLA_SIGN_TOKEN }}
3 changes: 3 additions & 0 deletions .github/workflows/frogbot-scan-pull-request.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,9 @@ jobs:
# "frogbot" GitHub environment can approve the pull request to be scanned.
environment: frogbot
steps:
- name: Setup Go with cache
uses: jfrog/.github/actions/install-go-with-cache@main

- uses: jfrog/frogbot@v2
env:
JFROG_CLI_LOG_LEVEL: "DEBUG"
Expand Down
3 changes: 3 additions & 0 deletions .github/workflows/frogbot-scan-repository.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,9 @@ jobs:
# The repository scanning will be triggered periodically on the following branches.
branch: [ "master" ]
steps:
- name: Setup Go with cache
uses: jfrog/.github/actions/install-go-with-cache@main

- uses: jfrog/frogbot@v2
env:
JFROG_CLI_LOG_LEVEL: "DEBUG"
Expand Down
15 changes: 2 additions & 13 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,19 +17,8 @@ jobs:
- name: Checkout Source
uses: actions/checkout@v4

- name: Go Cache
uses: actions/cache@v4
with:
path: ~/go/pkg/mod
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-
- name: Setup Go
uses: actions/setup-go@v5
with:
go-version: 1.22.x
cache: false
- name: Setup Go with cache
uses: jfrog/.github/actions/install-go-with-cache@main

- name: Tests
run: go test -v -race -covermode atomic -coverprofile=covprofile ./...
3 changes: 2 additions & 1 deletion crypto/checksum.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,8 @@ const (

var algorithmFunc = map[Algorithm]func() hash.Hash{
// Go native crypto algorithms:
MD5: md5.New,
MD5: md5.New,
//#nosec G401 -- Sha1 is supported by Artifactory.
SHA1: sha1.New,
// sha256-simd algorithm:
SHA256: sha256.New,
Expand Down
4 changes: 3 additions & 1 deletion fanout/reader_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package fanout

import (
"bytes"
//#nosec G505 -- Sha1 is supported by Artifactory.
"crypto/sha1"
"crypto/sha256"
"encoding/hex"
Expand Down Expand Up @@ -66,6 +67,7 @@ func TestFanoutRead(t *testing.T) {
}

func TestFanoutProgressiveRead(t *testing.T) {
//#nosec G401 -- Sha1 is supported by Artifactory.
hash1 := sha1.New()
proc1 := func(p []byte) (err error) {
if _, err := hash1.Write(p); err != nil {
Expand Down Expand Up @@ -104,7 +106,7 @@ func TestFanoutProgressiveRead(t *testing.T) {

func TestFanoutProgressiveReadError(t *testing.T) {
const errmsg = "ERRSHA1"

//#nosec G401 -- Sha1 is supported by Artifactory.
hash1 := sha1.New()
proc1 := func(p []byte) (err error) {
return errors.New(errmsg)
Expand Down
4 changes: 0 additions & 4 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ require (
github.com/jfrog/archiver/v3 v3.6.0
github.com/minio/sha256-simd v1.0.1
github.com/pkg/errors v0.9.1
github.com/schollz/progressbar/v3 v3.14.2
github.com/stretchr/testify v1.9.0
)

Expand All @@ -19,16 +18,13 @@ require (
github.com/klauspost/compress v1.17.4 // indirect
github.com/klauspost/cpuid/v2 v2.2.3 // indirect
github.com/klauspost/pgzip v1.2.6 // indirect
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
github.com/nwaples/rardecode v1.1.3 // indirect
github.com/pierrec/lz4/v4 v4.1.21 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/rivo/uniseg v0.4.7 // indirect
github.com/ulikunitz/xz v0.5.11 // indirect
github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect
github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 // indirect
golang.org/x/sync v0.6.0
golang.org/x/sys v0.17.0 // indirect
golang.org/x/term v0.17.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
14 changes: 0 additions & 14 deletions go.sum
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M=
github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q=
Expand All @@ -12,7 +11,6 @@ github.com/gookit/color v1.5.4 h1:FZmqs7XOyGgCAxmWyPslpiok1k05wmY3SJTytgvYFs0=
github.com/gookit/color v1.5.4/go.mod h1:pZJOeOS8DM43rXbp4AZo1n9zCU2qjpcRko0b6/QJi9w=
github.com/jfrog/archiver/v3 v3.6.0 h1:OVZ50vudkIQmKMgA8mmFF9S0gA47lcag22N13iV3F1w=
github.com/jfrog/archiver/v3 v3.6.0/go.mod h1:fCAof46C3rAXgZurS8kNRNdSVMKBbZs+bNNhPYxLldI=
github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw=
github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4=
github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
Expand All @@ -21,11 +19,8 @@ github.com/klauspost/cpuid/v2 v2.2.3 h1:sxCkb+qR91z4vsqw4vGGZlDgPz3G7gjaLyK3V8y7
github.com/klauspost/cpuid/v2 v2.2.3/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY=
github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU=
github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM=
github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8=
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
github.com/nwaples/rardecode v1.1.3 h1:cWCaZwfM5H7nAD6PyEdcVnczzV8i/JtotnyW/dD9lEc=
github.com/nwaples/rardecode v1.1.3/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0=
github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ=
Expand All @@ -34,12 +29,6 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
github.com/schollz/progressbar/v3 v3.14.2 h1:EducH6uNLIWsr560zSV1KrTeUb/wZGAHqyMFIEa99ks=
github.com/schollz/progressbar/v3 v3.14.2/go.mod h1:aQAZQnhF4JGFtRJiw/eobaXpsqpVQAftEQ+hLGXaRc4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8=
Expand All @@ -52,11 +41,8 @@ github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778/go.mod h1:2MuV+tbUrU1z
golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ=
golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U=
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
Expand Down
12 changes: 0 additions & 12 deletions http/filestream/filestream.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ import (
"os"

ioutils "github.com/jfrog/gofrog/io"
"github.com/schollz/progressbar/v3"
)

const (
Expand Down Expand Up @@ -69,11 +68,6 @@ type FileInfo struct {
}

func WriteFilesToStream(multipartWriter *multipart.Writer, filesList []*FileInfo) (err error) {
return WriteFilesToStreamWithProgressBar(multipartWriter, filesList, nil)
}

// Deprecated: Use WriteFilesToStream instead
func WriteFilesToStreamWithProgressBar(multipartWriter *multipart.Writer, filesList []*FileInfo, bar *progressbar.ProgressBar) (err error) {
// Close finishes the multipart message and writes the trailing
// boundary end line to the output, thereby marking the EOF.
defer ioutils.Close(multipartWriter, &err)
Expand All @@ -82,12 +76,6 @@ func WriteFilesToStreamWithProgressBar(multipartWriter *multipart.Writer, filesL
// Returning the error from writeFile with a possible error from the writeErr function
return errors.Join(err, writeErr(multipartWriter, file, err))
}
if bar != nil {
err = bar.Add(1)
if err != nil {
return fmt.Errorf("failed to update progress bar: %w", err)
}
}
}

return nil
Expand Down
107 changes: 107 additions & 0 deletions http/retryexecutor/retryexecutor.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
package retryexecutor

import (
"context"
"errors"
"fmt"
"github.com/jfrog/gofrog/log"
"time"
)

type ExecutionHandlerFunc func() (shouldRetry bool, err error)

type RetryExecutor struct {
// The context
Context context.Context

// The amount of retries to perform.
MaxRetries int

// Number of milliseconds to sleep between retries.
RetriesIntervalMilliSecs int

// Message to display when retrying.
ErrorMessage string

// Prefix to print at the beginning of each log.
LogMsgPrefix string

// ExecutionHandler is the operation to run with retries.
ExecutionHandler ExecutionHandlerFunc
}

func (runner *RetryExecutor) Execute() error {
var err error
var shouldRetry bool
for i := 0; i <= runner.MaxRetries; i++ {
// Run ExecutionHandler
shouldRetry, err = runner.ExecutionHandler()

// If we should not retry, return.
if !shouldRetry {
return err
}
if cancelledErr := runner.checkCancelled(); cancelledErr != nil {
return cancelledErr
}

// Print retry log message
runner.LogRetry(i, err)

// Going to sleep for RetryInterval milliseconds
if runner.RetriesIntervalMilliSecs > 0 && i < runner.MaxRetries {
time.Sleep(time.Millisecond * time.Duration(runner.RetriesIntervalMilliSecs))
}
}
// If the error is not nil, return it and log the timeout message. Otherwise, generate new error.
if err != nil {
log.Info(runner.getTimeoutErrorMsg())
return err
}
return TimeoutError{runner.getTimeoutErrorMsg()}
}

// Error of this type will be returned if the executor reaches timeout and no other error is returned by the execution handler.
type TimeoutError struct {
errMsg string
}

func (retryErr TimeoutError) Error() string {
return retryErr.errMsg
}

func (runner *RetryExecutor) getTimeoutErrorMsg() string {
prefix := ""
if runner.LogMsgPrefix != "" {
prefix = runner.LogMsgPrefix + " "
}
return fmt.Sprintf("%sexecutor timeout after %v attempts with %v milliseconds wait intervals", prefix, runner.MaxRetries, runner.RetriesIntervalMilliSecs)
}

func (runner *RetryExecutor) LogRetry(attemptNumber int, err error) {
message := fmt.Sprintf("%s(Attempt %v)", runner.LogMsgPrefix, attemptNumber+1)
if runner.ErrorMessage != "" {
message = fmt.Sprintf("%s - %s", message, runner.ErrorMessage)
}
if err != nil {
message = fmt.Sprintf("%s: %s", message, err.Error())
}

if err != nil || runner.ErrorMessage != "" {
log.Warn(message)
} else {
log.Debug(message)
}
}

func (runner *RetryExecutor) checkCancelled() error {
if runner.Context == nil {
return nil
}
contextErr := runner.Context.Err()
if errors.Is(contextErr, context.Canceled) {
log.Info("Retry executor was cancelled")
return contextErr
}
return nil
}
Loading

0 comments on commit a95bde9

Please sign in to comment.