support azure driver for metadata management

go.mod

@@ -3,9 +3,9 @@ module github.com/opensds/multi-cloud
 go 1.17
 
 require (
-	github.com/Azure/azure-pipeline-go v0.2.2
+	github.com/Azure/azure-pipeline-go v0.2.3
 	github.com/Azure/azure-sdk-for-go v32.4.0+incompatible
-	github.com/Azure/azure-storage-blob-go v0.8.0
+	github.com/Azure/azure-storage-blob-go v0.15.0
 	github.com/Azure/azure-storage-file-go v0.7.0
 	github.com/BurntSushi/toml v0.3.1
 	github.com/Shopify/sarama v1.19.0
@@ -27,6 +27,7 @@ require (
 	github.com/huaweicloud/huaweicloud-sdk-go-v3 v0.0.16-beta
 	github.com/journeymidnight/radoshttpd v0.0.7
 	github.com/journeymidnight/yig v2.0.0+incompatible
+	github.com/mailru/easyjson v0.7.7
 	github.com/micro/go-log v0.1.0
 	github.com/micro/go-micro/v2 v2.9.1
 	github.com/micro/go-plugins/registry/kubernetes/v2 v2.9.1
@@ -44,19 +45,20 @@ require (
 	golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
 	google.golang.org/api v0.14.0
 	google.golang.org/protobuf v1.22.0
-	gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f
+	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0
 	k8s.io/apimachinery v0.18.5
 )
 
 require (
 	cloud.google.com/go v0.50.0 // indirect
 	contrib.go.opencensus.io/exporter/ocagent v0.4.12 // indirect
-	github.com/Azure/go-autorest/autorest v0.5.0 // indirect
-	github.com/Azure/go-autorest/autorest/adal v0.2.0 // indirect
-	github.com/Azure/go-autorest/autorest/date v0.1.0 // indirect
-	github.com/Azure/go-autorest/logger v0.1.0 // indirect
-	github.com/Azure/go-autorest/tracing v0.1.0 // indirect
+	github.com/Azure/go-autorest v14.2.0+incompatible // indirect
+	github.com/Azure/go-autorest/autorest v0.11.28 // indirect
+	github.com/Azure/go-autorest/autorest/adal v0.9.18 // indirect
+	github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
+	github.com/Azure/go-autorest/logger v0.2.1 // indirect
+	github.com/Azure/go-autorest/tracing v0.6.0 // indirect
 	github.com/bitly/go-simplejson v0.5.0 // indirect
 	github.com/census-instrumentation/opencensus-proto v0.2.1 // indirect
 	github.com/coreos/etcd v3.3.18+incompatible // indirect
@@ -69,14 +71,16 @@ require (
 	github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 // indirect
 	github.com/eapache/queue v1.1.0 // indirect
 	github.com/emirpasic/gods v1.12.0 // indirect
+	github.com/form3tech-oss/jwt-go v3.2.2+incompatible // indirect
 	github.com/ghodss/yaml v1.0.0 // indirect
 	github.com/go-git/gcfg v1.5.0 // indirect
 	github.com/go-git/go-billy/v5 v5.0.0 // indirect
 	github.com/go-git/go-git/v5 v5.1.0 // indirect
 	github.com/go-log/log v0.1.0 // indirect
 	github.com/gogo/protobuf v1.3.1 // indirect
+	github.com/golang-jwt/jwt/v4 v4.2.0 // indirect
 	github.com/golang/snappy v0.0.1 // indirect
-	github.com/google/uuid v1.1.1 // indirect
+	github.com/google/uuid v1.2.0 // indirect
 	github.com/googleapis/gax-go/v2 v2.0.5 // indirect
 	github.com/grpc-ecosystem/grpc-gateway v1.9.5 // indirect
 	github.com/hashicorp/errwrap v1.0.0 // indirect
@@ -92,13 +96,15 @@ require (
 	github.com/imdario/mergo v0.3.9 // indirect
 	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
 	github.com/jmespath/go-jmespath v0.3.0 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.10 // indirect
 	github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd // indirect
 	github.com/klauspost/compress v1.13.6 // indirect
 	github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect
+	github.com/kr/pretty v0.2.1 // indirect
 	github.com/kr/text v0.2.0 // indirect
 	github.com/magiconair/properties v1.8.1 // indirect
-	github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d // indirect
+	github.com/mattn/go-ieproxy v0.0.1 // indirect
 	github.com/micro/cli/v2 v2.1.2 // indirect
 	github.com/miekg/dns v1.1.27 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
@@ -159,4 +165,4 @@ require (
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	honnef.co/go/tools v0.0.1-2019.2.3 // indirect
 	k8s.io/klog v1.0.0 // indirect
-)
+)

metadata/pkg/drivers/azure/azure.go

@@ -0,0 +1,187 @@
+// Copyright 2023 The SODA Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package azure
+
+import (
+	"context"
+	"sync"
+
+	log "github.com/sirupsen/logrus"
+
+	"github.com/Azure/azure-storage-blob-go/azblob"
+	backendpb "github.com/opensds/multi-cloud/backend/proto"
+	"github.com/opensds/multi-cloud/metadata/pkg/db"
+	"github.com/opensds/multi-cloud/metadata/pkg/model"
+	pb "github.com/opensds/multi-cloud/metadata/proto"
+)
+
+type AzureAdapter struct {
+	backend    *backendpb.BackendDetail
+	serviceURL azblob.ServiceURL
+}
+
+var ObjectList = func(containerURL azblob.ContainerURL, bucketName string) ([]*model.MetaObject, int64, error) {
+
+	var numObjects int = 0
+	var totSize int64 = 0
+	objectArray := []*model.MetaObject{}
+
+	blobResponse, err := containerURL.ListBlobsFlatSegment(context.Background(), azblob.Marker{}, azblob.ListBlobsSegmentOptions{})
+
+	if err != nil {
+		log.Errorf("unable to list objects. failed with error: %v", err)
+	}
+
+	for _, blob := range blobResponse.Segment.BlobItems {
+		numObjects += 1
+		blobURL := containerURL.NewBlobURL(blob.Name)
+		obj := &model.MetaObject{}
+		objectArray = append(objectArray, obj)
+		obj.ObjectName = blob.Name
+		obj.LastModifiedDate = &blob.Properties.LastModified
+		obj.Size = *blob.Properties.ContentLength
+		totSize += obj.Size
+		obj.StorageClass = string(blob.Properties.AccessTier)
+
+		if blob.Properties.DestinationSnapshot != nil {
+			obj.RedirectLocation = *blob.Properties.DestinationSnapshot
+		}
+
+		if blob.VersionID != nil {
+			obj.VersionId = *blob.VersionID
+		}
+
+		tagset := map[string]string{}
+
+		if blob.BlobTags != nil {
+			for _, tag := range blob.BlobTags.BlobTagSet {
+				tagset[tag.Key] = tag.Value
+			}
+			obj.ObjectTags = tagset
+		}
+
+		obj.ExpiresDate = blob.Properties.ExpiresOn
+
+		if blob.Properties.EncryptionScope != nil {
+			obj.ServerSideEncryption = *blob.Properties.EncryptionScope
+		}
+
+		obj.ReplicationStatus = string(blob.Properties.CopyStatus)
+
+		if blob.Properties.ContentType != nil {
+			obj.ObjectType = *blob.Properties.ContentType
+		}
+		props, err := blobURL.GetProperties(context.Background(), azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{})
+		if err != nil {
+			log.Error("unable to get properties for object %v. failed with error: %v", obj.ObjectName, err)
+		}
+
+		obj.Metadata = props.NewMetadata()
+	}
+
+	return objectArray, totSize, nil
+
+}
+
+func GetBucketMeta(idx int, container azblob.ContainerItem, serviceURL azblob.ServiceURL, bucketArray []*model.MetaBucket, wg *sync.WaitGroup) {
+
+	defer wg.Done()
+
+	buck := &model.MetaBucket{}
+	bucketArray[idx] = buck
+	buck.Name = container.Name
+	buck.CreationDate = &container.Properties.LastModified
+
+	containerURL := serviceURL.NewContainerURL(buck.Name)
+	objectArray, totalSize, err := ObjectList(containerURL, buck.Name)
+
+	if err != nil {
+		return
+	}
+
+	buck.Objects = objectArray
+	buck.NumberOfObjects = len(objectArray)
+	buck.TotalSize = totalSize
+
+	bucketArray[idx] = buck
+
+	props, err := containerURL.GetProperties(context.Background(), azblob.LeaseAccessConditions{})
+
+	if err != nil {
+		log.Errorf("unable to get bucket tags. failed wth error: %v", err)
+	} else {
+		buck.BucketTags = props.NewMetadata()
+	}
+
+	acl, err := containerURL.GetAccessPolicy(context.Background(), azblob.LeaseAccessConditions{})
+
+	if err != nil {
+		log.Errorf("unable to get bucket Acl. failed with error: %v", err)
+	} else {
+		access := []*model.Access{}
+
+		for _, item := range acl.Items {
+			acc := &model.Access{}
+			acc.ID = item.ID
+			acc.Permission = *item.AccessPolicy.Permission
+			access = append(access, acc)
+		}
+		buck.BucketAcl = access
+	}
+}
+
+func BucketList(serviceURL azblob.ServiceURL) ([]*model.MetaBucket, error) {
+	response, err := serviceURL.ListContainersSegment(context.Background(), azblob.Marker{}, azblob.ListContainersSegmentOptions{})
+
+	if err != nil {
+		log.Errorf("unable to list buckets. failed with err: %v", err)
+	}
+
+	bucketArray := make([]*model.MetaBucket, len(response.ContainerItems))
+
+	wg := sync.WaitGroup{}
+	for idx, container := range response.ContainerItems {
+		wg.Add(1)
+		go GetBucketMeta(idx, container, serviceURL, bucketArray, &wg)
+	}
+	wg.Wait()
+
+	return bucketArray, nil
+}
+
+func (ad *AzureAdapter) SyncMetadata(ctx context.Context, in *pb.SyncMetadataRequest) error {
+
+	buckArr, err := BucketList(ad.serviceURL)
+	if err != nil {
+		log.Errorf("metadata collection for backend id: %v failed with error: %v", ad.backend.Id, err)
+		return err
+	}
+
+	metaBackend := model.MetaBackend{}
+	metaBackend.Id = ad.backend.Id
+	metaBackend.BackendName = ad.backend.Name
+	metaBackend.Type = ad.backend.Type
+	metaBackend.Region = ad.backend.Region
+	metaBackend.Buckets = buckArr
+	metaBackend.NumberOfBuckets = int32(len(buckArr))
+	newContext := context.TODO()
+	err = db.DbAdapter.CreateMetadata(newContext, metaBackend)
+
+	return err
+}
+
+func (ad *AzureAdapter) DownloadObject() {
+
+}

metadata/pkg/drivers/azure/factory.go

@@ -0,0 +1,53 @@
+// Copyright 2023 The SODA Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package azure
+
+import (
+	"net/url"
+
+	"github.com/Azure/azure-storage-blob-go/azblob"
+	"github.com/opensds/multi-cloud/backend/pkg/utils/constants"
+	backendpb "github.com/opensds/multi-cloud/backend/proto"
+	driver "github.com/opensds/multi-cloud/metadata/pkg/drivers/cloudfactory"
+)
+
+type AzureBlobDriverFactory struct {
+}
+
+func (factory *AzureBlobDriverFactory) CreateDriver(backend *backendpb.BackendDetail) (driver.CloudDriver, error) {
+	endpoint := backend.Endpoint
+	AccessKeyID := backend.Access
+	AccessKeySecret := backend.Security
+
+	credential, err := azblob.NewSharedKeyCredential(AccessKeyID, AccessKeySecret)
+
+	if err != nil {
+		return nil, err
+	}
+
+	p := azblob.NewPipeline(credential, azblob.PipelineOptions{})
+
+	u, _ := url.Parse(endpoint)
+
+	serviceURL := azblob.NewServiceURL(*u, p)
+
+	adap := &AzureAdapter{backend: backend, serviceURL: serviceURL}
+
+	return adap, nil
+}
+
+func init() {
+	driver.RegisterDriverFactory(constants.BackendTypeAzure, &AzureBlobDriverFactory{})
+}

metadata/pkg/drivers/init.go

@@ -2,5 +2,6 @@ package drivers
 
 import (
 	_ "github.com/opensds/multi-cloud/metadata/pkg/drivers/aws"
+	_ "github.com/opensds/multi-cloud/metadata/pkg/drivers/azure"
 	_ "github.com/opensds/multi-cloud/metadata/pkg/drivers/gcp"
 )