Skip to content

Commit 9220ae7

Browse files
johhajohha
committed
rename config
1 parent ed9964c commit 9220ae7

File tree

10 files changed

+60
-35
lines changed

10 files changed

+60
-35
lines changed

‎lib/cloud_controller/blobstore/client_provider.rb‎

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -14,10 +14,8 @@ def self.provide(options:, directory_key:, root_dir: nil, resource_type: nil)
1414
if options[:blobstore_type].blank? || (options[:blobstore_type] == 'fog')
1515
provide_fog(options, directory_key, root_dir)
1616
elsif options[:blobstore_type] == 'storage-cli'
17+
# storage-cli is an experimental feature and not yet fully implemented. !!! DO NOT USE IN PRODUCTION !!!
1718
provide_storage_cli(options, directory_key, root_dir)
18-
elsif options[:blobstore_type] == 'storage-cli-fork'
19-
# Just for testing not yet merged features in bosh-azure-cli
20-
provide_storage_cli(options, directory_key, root_dir, fork: true)
2119
else
2220
provide_webdav(options, directory_key, root_dir)
2321
end
@@ -72,13 +70,14 @@ def provide_webdav(options, directory_key, root_dir)
7270
Client.new(SafeDeleteClient.new(retryable_client, root_dir))
7371
end
7472

75-
def provide_storage_cli(options, directory_key, root_dir, fork: false)
76-
client = StorageCliClient.build(fog_connection: options.fetch(:fog_connection),
73+
def provide_storage_cli(options, directory_key, root_dir)
74+
raise BlobstoreError.new('connection_config for storage-cli is not provided') unless options[:connection_config]
75+
76+
client = StorageCliClient.build(connection_config: options.fetch(:connection_config),
7777
directory_key: directory_key,
7878
root_dir: root_dir,
7979
min_size: options[:minimum_size],
80-
max_size: options[:maximum_size],
81-
fork: fork)
80+
max_size: options[:maximum_size])
8281

8382
logger = Steno.logger('cc.blobstore.storage_cli_client')
8483
errors = [StandardError]

‎lib/cloud_controller/blobstore/storage_cli/azure_storage_cli_client.rb‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,20 @@
11
module CloudController
22
module Blobstore
33
class AzureStorageCliClient < StorageCliClient
4-
StorageCliClient.register('AzureRM', CloudController::Blobstore::AzureStorageCliClient)
5-
64
def cli_path
75
ENV['AZURE_STORAGE_CLI_PATH'] || '/var/vcap/packages/azure-storage-cli/bin/azure-storage-cli'
86
end
97

10-
def build_config(fog_connection)
8+
def build_config(connection_config)
119
{
12-
account_name: fog_connection[:azure_storage_account_name],
13-
account_key: fog_connection[:azure_storage_access_key],
10+
account_name: connection_config[:azure_storage_account_name],
11+
account_key: connection_config[:azure_storage_access_key],
1412
container_name: @directory_key,
15-
environment: fog_connection[:environment]
13+
environment: connection_config[:environment]
1614
}.compact
1715
end
16+
17+
CloudController::Blobstore::StorageCliClient.register('AzureRM', AzureStorageCliClient)
1818
end
1919
end
2020
end

‎lib/cloud_controller/blobstore/storage_cli/storage_cli_client.rb‎

Lines changed: 16 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
require 'open3'
22
require 'tempfile'
3+
require 'tmpdir'
34
require 'fileutils'
45
require 'cloud_controller/blobstore/base_client'
56
require 'cloud_controller/blobstore/storage_cli/storage_cli_blob'
@@ -16,27 +17,26 @@ def register(provider, klass)
1617
@registry[provider] = klass
1718
end
1819

19-
def build(fog_connection:, directory_key:, root_dir:, min_size: nil, max_size: nil, fork: false)
20-
provider = fog_connection[:provider]
21-
raise 'Missing fog_connection[:provider]' if provider.nil?
20+
def build(connection_config:, directory_key:, root_dir:, min_size: nil, max_size: nil)
21+
provider = connection_config[:provider]
22+
raise 'Missing connection_config[:provider]' if provider.nil?
2223

2324
impl_class = @registry[provider]
2425
raise "No storage CLI client registered for provider #{provider}" unless impl_class
2526

26-
impl_class.new(fog_connection:, directory_key:, root_dir:, min_size:, max_size:, fork:)
27+
impl_class.new(connection_config:, directory_key:, root_dir:, min_size:, max_size:, fork:)
2728
end
2829
end
2930

30-
def initialize(fog_connection:, directory_key:, root_dir:, min_size: nil, max_size: nil, fork: false)
31+
def initialize(connection_config:, directory_key:, root_dir:, min_size: nil, max_size: nil)
3132
@cli_path = cli_path
3233
@directory_key = directory_key
3334
@root_dir = root_dir
3435
@min_size = min_size || 0
3536
@max_size = max_size
36-
@fork = fork
37-
38-
config = build_config(fog_connection)
37+
config = build_config(connection_config)
3938
@config_file = write_config_file(config)
39+
@fork = connection_config.fetch(:fork, false)
4040
end
4141

4242
def local?
@@ -142,9 +142,9 @@ def files_for(prefix, _ignored_directory_prefixes=[])
142142
end
143143

144144
def ensure_bucket_exists
145-
pass unless @fork
145+
return unless @fork
146146

147-
run_cli('ensure-bucket-exists', @directory_key)
147+
run_cli('ensure-bucket-exists')
148148
end
149149

150150
private
@@ -188,23 +188,27 @@ def cli_path
188188
raise NotImplementedError
189189
end
190190

191-
def build_config(fog_connection)
191+
def build_config(connection_config)
192192
raise NotImplementedError
193193
end
194194

195195
def write_config_file(config)
196+
# TODO: Consider to move the config generation into capi-release
196197
config_dir = File.join(tmpdir, 'blobstore-configs')
197198
FileUtils.mkdir_p(config_dir)
198199

199200
config_file_path = File.join(config_dir, "#{@directory_key}.json")
200201
File.open(config_file_path, 'w', 0o600) do |f|
201-
f.write(Oj.dump(config))
202+
f.write(Oj.dump(config.transform_keys(&:to_s)))
202203
end
203204
config_file_path
204205
end
205206

206207
def tmpdir
207208
VCAP::CloudController::Config.config.get(:directories, :tmpdir)
209+
rescue StandardError
210+
# Fallback to a temporary directory if the config is not set (e.g. for cc-deployment-updater
211+
Dir.mktmpdir('cc_blobstore')
208212
end
209213

210214
def logger

‎lib/cloud_controller/config_schemas/api_schema.rb‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -198,13 +198,15 @@ class ApiSchema < VCAP::Config
198198
minimum_size: Integer,
199199
resource_directory_key: String,
200200
fog_connection: Hash,
201+
optional(:connection_config) => Hash,
201202
fog_aws_storage_options: Hash,
202203
fog_gcp_storage_options: Hash
203204
},
204205

205206
buildpacks: {
206207
buildpack_directory_key: String,
207208
fog_connection: Hash,
209+
optional(:connection_config) => Hash,
208210
fog_aws_storage_options: Hash,
209211
fog_gcp_storage_options: Hash
210212
},
@@ -214,6 +216,7 @@ class ApiSchema < VCAP::Config
214216
max_valid_packages_stored: Integer,
215217
app_package_directory_key: String,
216218
fog_connection: Hash,
219+
optional(:connection_config) => Hash,
217220
fog_aws_storage_options: Hash,
218221
fog_gcp_storage_options: Hash
219222
},
@@ -222,6 +225,7 @@ class ApiSchema < VCAP::Config
222225
droplet_directory_key: String,
223226
max_staged_droplets_stored: Integer,
224227
fog_connection: Hash,
228+
optional(:connection_config) => Hash,
225229
fog_aws_storage_options: Hash,
226230
fog_gcp_storage_options: Hash
227231
},

‎lib/cloud_controller/config_schemas/clock_schema.rb‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -109,13 +109,15 @@ class ClockSchema < VCAP::Config
109109
minimum_size: Integer,
110110
resource_directory_key: String,
111111
fog_connection: Hash,
112+
optional(:connection_config) => Hash,
112113
fog_aws_storage_options: Hash,
113114
fog_gcp_storage_options: Hash
114115
},
115116

116117
buildpacks: {
117118
buildpack_directory_key: String,
118119
fog_connection: Hash,
120+
optional(:connection_config) => Hash,
119121
fog_aws_storage_options: Hash,
120122
fog_gcp_storage_options: Hash
121123
},
@@ -124,13 +126,15 @@ class ClockSchema < VCAP::Config
124126
max_package_size: Integer,
125127
app_package_directory_key: String,
126128
fog_connection: Hash,
129+
optional(:connection_config) => Hash,
127130
fog_aws_storage_options: Hash,
128131
fog_gcp_storage_options: Hash
129132
},
130133

131134
droplets: {
132135
droplet_directory_key: String,
133136
fog_connection: Hash,
137+
optional(:connection_config) => Hash,
134138
fog_aws_storage_options: Hash,
135139
fog_gcp_storage_options: Hash
136140
},

‎lib/cloud_controller/config_schemas/deployment_updater_schema.rb‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -109,13 +109,15 @@ class DeploymentUpdaterSchema < VCAP::Config
109109
minimum_size: Integer,
110110
resource_directory_key: String,
111111
fog_connection: Hash,
112+
optional(:connection_config) => Hash,
112113
fog_aws_storage_options: Hash,
113114
fog_gcp_storage_options: Hash
114115
},
115116

116117
buildpacks: {
117118
buildpack_directory_key: String,
118119
fog_connection: Hash,
120+
optional(:connection_config) => Hash,
119121
fog_aws_storage_options: Hash,
120122
fog_gcp_storage_options: Hash
121123
},
@@ -124,13 +126,15 @@ class DeploymentUpdaterSchema < VCAP::Config
124126
max_package_size: Integer,
125127
app_package_directory_key: String,
126128
fog_connection: Hash,
129+
optional(:connection_config) => Hash,
127130
fog_aws_storage_options: Hash,
128131
fog_gcp_storage_options: Hash
129132
},
130133

131134
droplets: {
132135
droplet_directory_key: String,
133136
fog_connection: Hash,
137+
optional(:connection_config) => Hash,
134138
fog_aws_storage_options: Hash,
135139
fog_gcp_storage_options: Hash
136140
},

‎lib/cloud_controller/config_schemas/worker_schema.rb‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,13 +100,15 @@ class WorkerSchema < VCAP::Config
100100
minimum_size: Integer,
101101
resource_directory_key: String,
102102
fog_connection: Hash,
103+
optional(:connection_config) => Hash,
103104
fog_aws_storage_options: Hash,
104105
fog_gcp_storage_options: Hash
105106
},
106107

107108
buildpacks: {
108109
buildpack_directory_key: String,
109110
fog_connection: Hash,
111+
optional(:connection_config) => Hash,
110112
fog_aws_storage_options: Hash,
111113
fog_gcp_storage_options: Hash
112114
},
@@ -116,13 +118,15 @@ class WorkerSchema < VCAP::Config
116118
max_valid_packages_stored: Integer,
117119
app_package_directory_key: String,
118120
fog_connection: Hash,
121+
optional(:connection_config) => Hash,
119122
fog_aws_storage_options: Hash,
120123
fog_gcp_storage_options: Hash
121124
},
122125

123126
droplets: {
124127
droplet_directory_key: String,
125128
fog_connection: Hash,
129+
optional(:connection_config) => Hash,
126130
fog_aws_storage_options: Hash,
127131
fog_gcp_storage_options: Hash
128132
},

‎spec/unit/lib/cloud_controller/blobstore/client_provider_spec.rb‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -133,13 +133,18 @@ module Blobstore
133133
let(:storage_cli_client_mock) { class_double(CloudController::Blobstore::StorageCliClient) }
134134

135135
before do
136-
options.merge!(fog_connection: {}, minimum_size: 100, maximum_size: 1000)
136+
options.merge!(connection_config: {}, minimum_size: 100, maximum_size: 1000)
137137
end
138138

139139
it 'provides a storage-cli client' do
140140
allow(StorageCliClient).to receive(:build).and_return(storage_cli_client_mock)
141141
ClientProvider.provide(options:, directory_key:, root_dir:)
142-
expect(StorageCliClient).to have_received(:build).with(fog_connection: {}, directory_key: directory_key, root_dir: root_dir, min_size: 100, max_size: 1000, fork: false)
142+
expect(StorageCliClient).to have_received(:build).with(connection_config: {}, directory_key: directory_key, root_dir: root_dir, min_size: 100, max_size: 1000)
143+
end
144+
145+
it 'raises an error if connection_config is not provided' do
146+
options.delete(:connection_config)
147+
expect { ClientProvider.provide(options:, directory_key:, root_dir:) }.to raise_error(BlobstoreError, 'connection_config for storage-cli is not provided')
143148
end
144149
end
145150
end

‎spec/unit/lib/cloud_controller/blobstore/storage_cli/azure_storage_cli_client_spec.rb‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,15 +6,16 @@
66
module CloudController
77
module Blobstore
88
RSpec.describe AzureStorageCliClient do
9-
subject(:client) { AzureStorageCliClient.new(fog_connection: fog_connection, directory_key: directory_key, root_dir: 'bommel', fork: true) }
9+
subject(:client) { AzureStorageCliClient.new(connection_config: connection_config, directory_key: directory_key, root_dir: 'bommel') }
1010
let(:directory_key) { 'my-bucket' }
11-
let(:fog_connection) do
11+
let(:connection_config) do
1212
{
1313
azure_storage_access_key: 'some-access-key',
1414
azure_storage_account_name: 'some-account-name',
1515
container_name: directory_key,
1616
environment: 'AzureCloud',
17-
provider: 'AzureRM'
17+
provider: 'AzureRM',
18+
fork: true
1819
}
1920
end
2021
let(:downloaded_file) do
@@ -39,7 +40,7 @@ module Blobstore
3940
allow(client).to receive(:run_cli).with('delete', anything).and_return([nil, instance_double(Process::Status, exitstatus: 0)])
4041
allow(client).to receive(:run_cli).with('delete-recursive', anything).and_return([nil, instance_double(Process::Status, exitstatus: 0)])
4142
allow(client).to receive(:run_cli).with('list', anything).and_return(["aa/bb/blob1\ncc/dd/blob2\n", instance_double(Process::Status, exitstatus: 0)])
42-
allow(client).to receive(:run_cli).with('ensure-bucket-exists', anything).and_return([nil, instance_double(Process::Status, exitstatus: 0)])
43+
allow(client).to receive(:run_cli).with('ensure-bucket-exists').and_return([nil, instance_double(Process::Status, exitstatus: 0)])
4344
allow(client).to receive(:run_cli).with('properties', anything).and_return(['{"dummy": "json"}', instance_double(Process::Status, exitstatus: 0)])
4445
allow(client).to receive(:run_cli).with('sign', anything, 'get', '3600s').and_return(['some-url', instance_double(Process::Status, exitstatus: 0)])
4546
end

‎spec/unit/lib/cloud_controller/blobstore/storage_cli/storage_cli_client_spec.rb‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -12,20 +12,20 @@ def initialize(*); end
1212
before { StorageCliClient.register('DummyProvider', DummyClient) }
1313

1414
it 'builds the correct client' do
15-
client_from_registry = StorageCliClient.build(fog_connection: { provider: 'DummyProvider' }, directory_key: 'dummy-key', root_dir: 'dummy-root')
15+
client_from_registry = StorageCliClient.build(connection_config: { provider: 'DummyProvider' }, directory_key: 'dummy-key', root_dir: 'dummy-root')
1616
expect(client_from_registry).to be_a(DummyClient)
1717
end
1818

1919
it 'raises an error for an unregistered provider' do
2020
expect do
21-
StorageCliClient.build(fog_connection: { provider: 'UnknownProvider' }, directory_key: 'dummy-key', root_dir: 'dummy-root')
21+
StorageCliClient.build(connection_config: { provider: 'UnknownProvider' }, directory_key: 'dummy-key', root_dir: 'dummy-root')
2222
end.to raise_error(RuntimeError, 'No storage CLI client registered for provider UnknownProvider')
2323
end
2424

2525
it 'raises an error if provider is missing' do
2626
expect do
27-
StorageCliClient.build(fog_connection: {}, directory_key: 'dummy-key', root_dir: 'dummy-root')
28-
end.to raise_error(RuntimeError, 'Missing fog_connection[:provider]')
27+
StorageCliClient.build(connection_config: {}, directory_key: 'dummy-key', root_dir: 'dummy-root')
28+
end.to raise_error(RuntimeError, 'Missing connection_config[:provider]')
2929
end
3030
end
3131
end

0 commit comments

Comments
 (0)