From 31150aa8da380dafeb76339313900a4b6c8fbddb Mon Sep 17 00:00:00 2001 From: Ryan Diers Date: Thu, 22 Jun 2023 09:46:57 -0700 Subject: [PATCH] integrate bindetector into upload via fileutils helpers --- internal/cli/BUILD.bazel | 1 + internal/cli/upload.go | 7 +++++- internal/fileutils/BUILD.bazel | 15 +++++++++-- internal/fileutils/fileutils.go | 37 ++++++++++++++++++++++++++++ internal/fileutils/fileutils_test.go | 15 +++++++++++ internal/stores/azure.go | 1 + internal/stores/gcs.go | 1 + internal/stores/s3.go | 11 ++++++--- 8 files changed, 81 insertions(+), 7 deletions(-) create mode 100644 internal/fileutils/fileutils_test.go diff --git a/internal/cli/BUILD.bazel b/internal/cli/BUILD.bazel index 8c976184..cc586bbd 100644 --- a/internal/cli/BUILD.bazel +++ b/internal/cli/BUILD.bazel @@ -14,6 +14,7 @@ go_library( visibility = ["//:__subpackages__"], deps = [ "//internal/config", + "//internal/fileutils", "//internal/metadata", "//internal/program", "//internal/stores", diff --git a/internal/cli/upload.go b/internal/cli/upload.go index 25bfc56a..aab7b253 100644 --- a/internal/cli/upload.go +++ b/internal/cli/upload.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" + "github.com/discentem/cavorite/internal/fileutils" "github.com/discentem/cavorite/internal/program" "github.com/google/logger" "github.com/spf13/afero" @@ -33,6 +34,7 @@ func uploadCmd() *cobra.Command { RunE: uploadFn, } + // uploadCmd.LocalFlags().Bool("store_type", "", "Storage backend to use") return uploadCmd } @@ -75,7 +77,10 @@ func uploadFn(cmd *cobra.Command, objects []string) error { } logger.Infof("Uploading to: %s", s.GetOptions().BackendAddress) - logger.Infof("Uploading file: %s", objects) + objects, err = fileutils.GetBinariesWalkPath(fsys, objects) + if err != nil { + return fmt.Errorf("walking binaries path error: %w", err) + } if err := s.Upload(cmd.Context(), objects...); err != nil { return err } diff --git a/internal/fileutils/BUILD.bazel b/internal/fileutils/BUILD.bazel index ebaab3b6..a67bf4c8 100644 --- a/internal/fileutils/BUILD.bazel +++ b/internal/fileutils/BUILD.bazel @@ -1,9 +1,20 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library") +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "fileutils", srcs = ["fileutils.go"], importpath = "github.com/discentem/cavorite/internal/fileutils", visibility = ["//:__subpackages__"], - deps = ["@com_github_spf13_afero//:afero"], + deps = [ + "//internal/bindetector", + "@com_github_google_logger//:logger", + "@com_github_spf13_afero//:afero", + ], +) + +go_test( + name = "fileutils_test", + srcs = ["fileutils_test.go"], + embed = [":fileutils"], + deps = ["@com_github_stretchr_testify//assert"], ) diff --git a/internal/fileutils/fileutils.go b/internal/fileutils/fileutils.go index 96f458b5..fa2814b8 100644 --- a/internal/fileutils/fileutils.go +++ b/internal/fileutils/fileutils.go @@ -3,7 +3,10 @@ package fileutils import ( "fmt" "io" + "io/fs" + "github.com/discentem/cavorite/internal/bindetector" + "github.com/google/logger" "github.com/spf13/afero" ) @@ -23,3 +26,37 @@ func BytesFromAferoFile(f afero.File) ([]byte, error) { } return b, nil } + +func GetBinariesWalkPath(fsys afero.Fs, objects []string) ([]string, error) { + afs := afero.Afero{Fs: fsys} + + var paths []string + logger.V(2).Infof("walking objects: %s", objects) + for _, object := range objects { + walkErr := afs.Walk(object, func(p string, info fs.FileInfo, err error) error { + logger.V(2).Infof("walkfunc() p: %s", p) + isDir, err := afs.IsDir(p) + if err != nil { + logger.Errorf("walkfunc() fsys.IsDir error: %v", err) + return err + } + logger.V(2).Infof("walkfunc() isDir: %t", isDir) + + if isDir { + return nil + } + + if bindetector.IsBinary(p) { + logger.V(2).Infof("walkfunc() - bindetector().IsBinary appending path: %s", p) + paths = append(paths, p) + } + + return nil + }) + if walkErr != nil { + return []string{}, walkErr + } + } + + return paths, nil +} diff --git a/internal/fileutils/fileutils_test.go b/internal/fileutils/fileutils_test.go new file mode 100644 index 00000000..6982dcea --- /dev/null +++ b/internal/fileutils/fileutils_test.go @@ -0,0 +1,15 @@ +package fileutils + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGetBinariesWalkPath(t *testing.T) { + // expectedBinariesFromPath := []string{ + // "abc.pkg", + // "zxf.pkg", + // } + assert.True(t, true) +} diff --git a/internal/stores/azure.go b/internal/stores/azure.go index 6474cbf7..660b46db 100644 --- a/internal/stores/azure.go +++ b/internal/stores/azure.go @@ -49,6 +49,7 @@ func (s *AzureBlobStore) Upload(ctx context.Context, objects ...string) error { return err } containerName := path.Base(s.Options.BackendAddress) + logger.Infof(`Uploading "%s" to Azure`, o) _, err = s.containerClient.UploadStream( ctx, containerName, diff --git a/internal/stores/gcs.go b/internal/stores/gcs.go index 055839a9..91c875d7 100644 --- a/internal/stores/gcs.go +++ b/internal/stores/gcs.go @@ -107,6 +107,7 @@ func (s *GCSStore) Upload(ctx context.Context, objects ...string) error { // return multiple errors return multErr } + logger.Infof(`Uploading "%s" to GCS`, o) _, err = io.Copy(wc, f) if err != nil { multErr = multierror.Append(multErr, fmt.Errorf("io.Copy() error: %w", err)) diff --git a/internal/stores/s3.go b/internal/stores/s3.go index 8de585ac..6eda2931 100644 --- a/internal/stores/s3.go +++ b/internal/stores/s3.go @@ -160,22 +160,25 @@ func (s *S3Store) Upload(ctx context.Context, objects ...string) error { logger.Errorf("error encountered parsing backend address: %v", err) return err } - obj := s3.PutObjectInput{ + + putObjInput := s3.PutObjectInput{ Bucket: aws.String(s3BucketName), Key: aws.String(o), Body: f, } - out, err := s.s3Uploader.Upload(ctx, &obj) + logger.Infof(`Uploading "%s" to S3`, o) + uploadOutput, err := s.s3Uploader.Upload(ctx, &putObjInput) if err != nil { if err := cleanupFn(); err != nil { return fmt.Errorf("cleanup() failed after Upload failure: %w", err) } - logger.Error(out) + logger.Error(uploadOutput) return err } if err := f.Close(); err != nil { return err } + logger.Infof(`Upload Complete for "%s" to S3`, o) } return nil } @@ -240,7 +243,7 @@ func (s *S3Store) Retrieve(ctx context.Context, objects ...string) error { func (s *S3Store) getBucketName() (string, error) { var bucketName string - logger.Infof("s3store getBucketName: backend address: %s", s.Options.BackendAddress) + logger.V(2).Infof("s3store getBucketName: backend address: %s", s.Options.BackendAddress) switch { case strings.HasPrefix(s.Options.BackendAddress, "s3://"): s3BucketUrl, err := url.Parse(s.Options.BackendAddress)