From 8710394dc9d34b82cf20d966653cd8a8dd185f72 Mon Sep 17 00:00:00 2001 From: Devansh Jain Date: Mon, 6 Mar 2023 23:44:16 +0530 Subject: [PATCH 1/3] add azure driver --- metadata/pkg/drivers/azure/azure.go | 168 ++++++++++++++++++++++++++ metadata/pkg/drivers/azure/factory.go | 53 ++++++++ metadata/pkg/drivers/init.go | 1 + 3 files changed, 222 insertions(+) create mode 100644 metadata/pkg/drivers/azure/azure.go create mode 100644 metadata/pkg/drivers/azure/factory.go diff --git a/metadata/pkg/drivers/azure/azure.go b/metadata/pkg/drivers/azure/azure.go new file mode 100644 index 000000000..060ee5e9e --- /dev/null +++ b/metadata/pkg/drivers/azure/azure.go @@ -0,0 +1,168 @@ +// Copyright 2023 The SODA Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package azure + +import ( + "context" + "sync" + + log "github.com/sirupsen/logrus" + + "github.com/Azure/azure-storage-blob-go/azblob" + backendpb "github.com/opensds/multi-cloud/backend/proto" + "github.com/opensds/multi-cloud/metadata/pkg/db" + "github.com/opensds/multi-cloud/metadata/pkg/model" + pb "github.com/opensds/multi-cloud/metadata/proto" +) + +type AzureAdapter struct { + backend *backendpb.BackendDetail + serviceURL azblob.ServiceURL +} + +var ObjectList = func(containerURL azblob.ContainerURL, bucketName string) ([]*model.MetaObject, int64, error) { + + var numObjects int = 0 + var totSize int64 = 0 + objectArray := []*model.MetaObject{} + + blobResponse, err := containerURL.ListBlobsFlatSegment(context.Background(), azblob.Marker{}, azblob.ListBlobsSegmentOptions{}) + + if err != nil { + log.Fatal(err) + } + + for _, blob := range blobResponse.Segment.BlobItems { + numObjects += 1 + blobURL := containerURL.NewBlobURL(blob.Name) + obj := &model.MetaObject{} + objectArray = append(objectArray, obj) + obj.ObjectName = blob.Name + obj.LastModifiedDate = &blob.Properties.LastModified + obj.Size = *blob.Properties.ContentLength + totSize += obj.Size + obj.StorageClass = string(blob.Properties.AccessTier) + + if blob.Properties.DestinationSnapshot != nil { + obj.RedirectLocation = *blob.Properties.DestinationSnapshot + } + + obj.ReplicationStatus = string(blob.Properties.CopyStatus) + + if blob.Properties.ContentType != nil { + obj.ObjectType = *blob.Properties.ContentType + } + props, err := blobURL.GetProperties(context.Background(), azblob.BlobAccessConditions{}) + if err != nil { + log.Error("get properties failed", err) + } + + obj.Metadata = props.NewMetadata() + } + + return objectArray, totSize, nil + +} + +func GetBucketMeta(idx int, container azblob.ContainerItem, serviceURL azblob.ServiceURL, bucketArray []*model.MetaBucket, wg *sync.WaitGroup) { + + defer wg.Done() + + buck := &model.MetaBucket{} + bucketArray[idx] = buck + buck.Name = container.Name + buck.CreationDate = &container.Properties.LastModified + + containerURL := serviceURL.NewContainerURL(buck.Name) + objectArray, totalSize, err := ObjectList(containerURL, buck.Name) + + if err != nil { + return + } + + buck.Objects = objectArray + buck.NumberOfObjects = len(objectArray) + buck.TotalSize = totalSize + + bucketArray[idx] = buck + + props, err := containerURL.GetProperties(context.Background(), azblob.LeaseAccessConditions{}) + + if err != nil { + log.Errorf("failed to get bucket tags. failed wth error: %v", err) + } else { + buck.BucketTags = props.NewMetadata() + } + + acl, err := containerURL.GetAccessPolicy(context.Background(), azblob.LeaseAccessConditions{}) + + if err != nil { + log.Errorf("unable to get bucket Acl. failed with error: %v", err) + } else { + access := []*model.Access{} + + for _, item := range acl.Items { + acc := &model.Access{} + acc.ID = item.ID + acc.Permission = item.AccessPolicy.Permission + access = append(access, acc) + } + buck.BucketAcl = access + } +} + +func BucketList(serviceURL azblob.ServiceURL) ([]*model.MetaBucket, error) { + response, err := serviceURL.ListContainersSegment(context.Background(), azblob.Marker{}, azblob.ListContainersSegmentOptions{}) + + if err != nil { + log.Errorf("unable to list buckets. failed with err: %v", err) + } + + bucketArray := make([]*model.MetaBucket, len(response.ContainerItems)) + + wg := sync.WaitGroup{} + for idx, container := range response.ContainerItems { + wg.Add(1) + go GetBucketMeta(idx, container, serviceURL, bucketArray, &wg) + } + wg.Wait() + + return bucketArray, nil +} + +func (ad *AzureAdapter) SyncMetadata(ctx context.Context, in *pb.SyncMetadataRequest) error { + + buckArr, err := BucketList(ad.serviceURL) + if err != nil { + log.Errorf("metadata collection for backend id: %v failed with error: %v", ad.backend.Id, err) + return err + } + + metaBackend := model.MetaBackend{} + metaBackend.Id = ad.backend.Id + metaBackend.BackendName = ad.backend.Name + metaBackend.Type = ad.backend.Type + metaBackend.Region = ad.backend.Region + metaBackend.Buckets = buckArr + metaBackend.NumberOfBuckets = int32(len(buckArr)) + newContext := context.TODO() + err = db.DbAdapter.CreateMetadata(newContext, metaBackend) + + return err +} + +func (ad *AzureAdapter) DownloadObject() { + +} diff --git a/metadata/pkg/drivers/azure/factory.go b/metadata/pkg/drivers/azure/factory.go new file mode 100644 index 000000000..1f1be45bf --- /dev/null +++ b/metadata/pkg/drivers/azure/factory.go @@ -0,0 +1,53 @@ +// Copyright 2023 The SODA Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package azure + +import ( + "net/url" + + "github.com/Azure/azure-storage-blob-go/azblob" + "github.com/opensds/multi-cloud/backend/pkg/utils/constants" + backendpb "github.com/opensds/multi-cloud/backend/proto" + driver "github.com/opensds/multi-cloud/metadata/pkg/drivers/cloudfactory" +) + +type AzureBlobDriverFactory struct { +} + +func (factory *AzureBlobDriverFactory) CreateDriver(backend *backendpb.BackendDetail) (driver.CloudDriver, error) { + endpoint := backend.Endpoint + AccessKeyID := backend.Access + AccessKeySecret := backend.Security + + credential, err := azblob.NewSharedKeyCredential(AccessKeyID, AccessKeySecret) + + if err != nil { + return nil, err + } + + p := azblob.NewPipeline(credential, azblob.PipelineOptions{}) + + u, _ := url.Parse(endpoint) + + serviceURL := azblob.NewServiceURL(*u, p) + + adap := &AzureAdapter{backend: backend, serviceURL: serviceURL} + + return adap, nil +} + +func init() { + driver.RegisterDriverFactory(constants.BackendTypeAzure, &AzureBlobDriverFactory{}) +} diff --git a/metadata/pkg/drivers/init.go b/metadata/pkg/drivers/init.go index fb8350d49..4a6d92715 100644 --- a/metadata/pkg/drivers/init.go +++ b/metadata/pkg/drivers/init.go @@ -2,5 +2,6 @@ package drivers import ( _ "github.com/opensds/multi-cloud/metadata/pkg/drivers/aws" + _ "github.com/opensds/multi-cloud/metadata/pkg/drivers/azure" _ "github.com/opensds/multi-cloud/metadata/pkg/drivers/gcp" ) From 2cb121c2dfccc026dda291bb3295762ce4ed72d5 Mon Sep 17 00:00:00 2001 From: Devansh Jain Date: Thu, 9 Mar 2023 11:36:25 +0530 Subject: [PATCH 2/3] add additional object metadata --- metadata/pkg/drivers/azure/azure.go | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/metadata/pkg/drivers/azure/azure.go b/metadata/pkg/drivers/azure/azure.go index 060ee5e9e..b14e23788 100644 --- a/metadata/pkg/drivers/azure/azure.go +++ b/metadata/pkg/drivers/azure/azure.go @@ -41,7 +41,7 @@ var ObjectList = func(containerURL azblob.ContainerURL, bucketName string) ([]*m blobResponse, err := containerURL.ListBlobsFlatSegment(context.Background(), azblob.Marker{}, azblob.ListBlobsSegmentOptions{}) if err != nil { - log.Fatal(err) + log.Errorf("unable to list objects. failed with error: %v", err) } for _, blob := range blobResponse.Segment.BlobItems { @@ -59,14 +59,33 @@ var ObjectList = func(containerURL azblob.ContainerURL, bucketName string) ([]*m obj.RedirectLocation = *blob.Properties.DestinationSnapshot } + if blob.VersionID != nil { + obj.VersionId = *blob.VersionID + } + + tagset := map[string]string{} + + if blob.BlobTags != nil { + for _, tag := range blob.BlobTags.BlobTagSet { + tagset[tag.Key] = tag.Value + } + obj.ObjectTags = tagset + } + + obj.ExpiresDate = blob.Properties.ExpiresOn + + if blob.Properties.EncryptionScope != nil { + obj.ServerSideEncryption = *blob.Properties.EncryptionScope + } + obj.ReplicationStatus = string(blob.Properties.CopyStatus) if blob.Properties.ContentType != nil { obj.ObjectType = *blob.Properties.ContentType } - props, err := blobURL.GetProperties(context.Background(), azblob.BlobAccessConditions{}) + props, err := blobURL.GetProperties(context.Background(), azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{}) if err != nil { - log.Error("get properties failed", err) + log.Error("unable to get properties for object %v. failed with error: %v", obj.ObjectName, err) } obj.Metadata = props.NewMetadata() @@ -101,7 +120,7 @@ func GetBucketMeta(idx int, container azblob.ContainerItem, serviceURL azblob.Se props, err := containerURL.GetProperties(context.Background(), azblob.LeaseAccessConditions{}) if err != nil { - log.Errorf("failed to get bucket tags. failed wth error: %v", err) + log.Errorf("unable to get bucket tags. failed wth error: %v", err) } else { buck.BucketTags = props.NewMetadata() } @@ -116,7 +135,7 @@ func GetBucketMeta(idx int, container azblob.ContainerItem, serviceURL azblob.Se for _, item := range acl.Items { acc := &model.Access{} acc.ID = item.ID - acc.Permission = item.AccessPolicy.Permission + acc.Permission = *item.AccessPolicy.Permission access = append(access, acc) } buck.BucketAcl = access From a3fdc7e93972c0587d6ca75df4caffa0e4d36d90 Mon Sep 17 00:00:00 2001 From: Devansh Jain Date: Thu, 13 Apr 2023 14:47:09 +0530 Subject: [PATCH 3/3] update azure sdk version in go.mod --- go.mod | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/go.mod b/go.mod index b7fb4968f..1d7a0c66c 100644 --- a/go.mod +++ b/go.mod @@ -3,9 +3,9 @@ module github.com/opensds/multi-cloud go 1.17 require ( - github.com/Azure/azure-pipeline-go v0.2.2 + github.com/Azure/azure-pipeline-go v0.2.3 github.com/Azure/azure-sdk-for-go v32.4.0+incompatible - github.com/Azure/azure-storage-blob-go v0.8.0 + github.com/Azure/azure-storage-blob-go v0.15.0 github.com/Azure/azure-storage-file-go v0.7.0 github.com/BurntSushi/toml v0.3.1 github.com/Shopify/sarama v1.19.0 @@ -27,6 +27,7 @@ require ( github.com/huaweicloud/huaweicloud-sdk-go-v3 v0.0.16-beta github.com/journeymidnight/radoshttpd v0.0.7 github.com/journeymidnight/yig v2.0.0+incompatible + github.com/mailru/easyjson v0.7.7 github.com/micro/go-log v0.1.0 github.com/micro/go-micro/v2 v2.9.1 github.com/micro/go-plugins/registry/kubernetes/v2 v2.9.1 @@ -44,7 +45,7 @@ require ( golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 google.golang.org/api v0.14.0 google.golang.org/protobuf v1.22.0 - gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f + gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c gopkg.in/natefinch/lumberjack.v2 v2.0.0 k8s.io/apimachinery v0.18.5 ) @@ -52,11 +53,12 @@ require ( require ( cloud.google.com/go v0.50.0 // indirect contrib.go.opencensus.io/exporter/ocagent v0.4.12 // indirect - github.com/Azure/go-autorest/autorest v0.5.0 // indirect - github.com/Azure/go-autorest/autorest/adal v0.2.0 // indirect - github.com/Azure/go-autorest/autorest/date v0.1.0 // indirect - github.com/Azure/go-autorest/logger v0.1.0 // indirect - github.com/Azure/go-autorest/tracing v0.1.0 // indirect + github.com/Azure/go-autorest v14.2.0+incompatible // indirect + github.com/Azure/go-autorest/autorest v0.11.28 // indirect + github.com/Azure/go-autorest/autorest/adal v0.9.18 // indirect + github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect + github.com/Azure/go-autorest/logger v0.2.1 // indirect + github.com/Azure/go-autorest/tracing v0.6.0 // indirect github.com/bitly/go-simplejson v0.5.0 // indirect github.com/census-instrumentation/opencensus-proto v0.2.1 // indirect github.com/coreos/etcd v3.3.18+incompatible // indirect @@ -69,14 +71,16 @@ require ( github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 // indirect github.com/eapache/queue v1.1.0 // indirect github.com/emirpasic/gods v1.12.0 // indirect + github.com/form3tech-oss/jwt-go v3.2.2+incompatible // indirect github.com/ghodss/yaml v1.0.0 // indirect github.com/go-git/gcfg v1.5.0 // indirect github.com/go-git/go-billy/v5 v5.0.0 // indirect github.com/go-git/go-git/v5 v5.1.0 // indirect github.com/go-log/log v0.1.0 // indirect github.com/gogo/protobuf v1.3.1 // indirect + github.com/golang-jwt/jwt/v4 v4.2.0 // indirect github.com/golang/snappy v0.0.1 // indirect - github.com/google/uuid v1.1.1 // indirect + github.com/google/uuid v1.2.0 // indirect github.com/googleapis/gax-go/v2 v2.0.5 // indirect github.com/grpc-ecosystem/grpc-gateway v1.9.5 // indirect github.com/hashicorp/errwrap v1.0.0 // indirect @@ -92,13 +96,15 @@ require ( github.com/imdario/mergo v0.3.9 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/jmespath/go-jmespath v0.3.0 // indirect + github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.10 // indirect github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd // indirect github.com/klauspost/compress v1.13.6 // indirect github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect + github.com/kr/pretty v0.2.1 // indirect github.com/kr/text v0.2.0 // indirect github.com/magiconair/properties v1.8.1 // indirect - github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d // indirect + github.com/mattn/go-ieproxy v0.0.1 // indirect github.com/micro/cli/v2 v2.1.2 // indirect github.com/miekg/dns v1.1.27 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect @@ -159,4 +165,4 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect honnef.co/go/tools v0.0.1-2019.2.3 // indirect k8s.io/klog v1.0.0 // indirect -) +) \ No newline at end of file