Skip to content

Commit 23e5dcf

Browse files
deliahuvishalbollu
deliahu
authored and
vishalbollu
committed
Check that cluster image versions match CLI version (#862)
(cherry picked from commit ec0e7e0)
1 parent a6fc9a9 commit 23e5dcf

File tree

9 files changed

+155
-39
lines changed

9 files changed

+155
-39
lines changed

‎cli/cmd/lib_manager.go

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ import (
2727
"syscall"
2828
"time"
2929

30+
"github.com/cortexlabs/cortex/pkg/consts"
3031
"github.com/cortexlabs/cortex/pkg/lib/errors"
3132
"github.com/cortexlabs/cortex/pkg/lib/exit"
3233
"github.com/cortexlabs/cortex/pkg/lib/files"
@@ -111,6 +112,11 @@ func pullManager(managerImage string) error {
111112
}
112113

113114
func runManager(containerConfig *container.Config) (string, *int, error) {
115+
containerConfig.Env = append(containerConfig.Env, "CORTEX_CLI_VERSION="+consts.CortexVersion)
116+
117+
// Add a slight delay before running the command to ensure logs don't start until after the container is attached
118+
containerConfig.Cmd[0] = "sleep 0.1 && /root/check_cortex_version.sh && " + containerConfig.Cmd[0]
119+
114120
docker, err := getDockerClient()
115121
if err != nil {
116122
return "", nil, err
@@ -161,7 +167,6 @@ func runManager(containerConfig *container.Config) (string, *int, error) {
161167
return "", nil, wrapDockerError(err)
162168
}
163169

164-
// There is a slight delay between the container starting at attaching to it, hence the sleep in Cmd
165170
logsOutput, err := docker.ContainerAttach(context.Background(), containerInfo.ID, dockertypes.ContainerAttachOptions{
166171
Stream: true,
167172
Stdout: true,
@@ -216,7 +221,7 @@ func runManagerUpdateCommand(entrypoint string, clusterConfig *clusterconfig.Con
216221
containerConfig := &container.Config{
217222
Image: clusterConfig.ImageManager,
218223
Entrypoint: []string{"/bin/bash", "-c"},
219-
Cmd: []string{fmt.Sprintf("sleep 0.1 && eval $(python /root/cluster_config_env.py %s) && %s", mountedConfigPath, entrypoint)},
224+
Cmd: []string{fmt.Sprintf("eval $(python /root/cluster_config_env.py %s) && %s", mountedConfigPath, entrypoint)},
220225
Tty: true,
221226
AttachStdout: true,
222227
AttachStderr: true,
@@ -246,7 +251,7 @@ func runManagerAccessCommand(entrypoint string, accessConfig clusterconfig.Acces
246251
containerConfig := &container.Config{
247252
Image: accessConfig.ImageManager,
248253
Entrypoint: []string{"/bin/bash", "-c"},
249-
Cmd: []string{"sleep 0.1 && " + entrypoint},
254+
Cmd: []string{entrypoint},
250255
Tty: true,
251256
AttachStdout: true,
252257
AttachStderr: true,

‎manager/check_cortex_version.sh

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
#!/bin/bash
2+
3+
# Copyright 2020 Cortex Labs, Inc.
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
set -e
18+
19+
CORTEX_VERSION=master
20+
21+
if [ "$CORTEX_VERSION" != "$CORTEX_CLI_VERSION" ]; then
22+
echo "error: your CLI version ($CORTEX_CLI_VERSION) doesn't match your Cortex manager image version ($CORTEX_VERSION); please update your CLI by following the instructions at https://www.cortex.dev/install, or update your Cortex manager image by modifying the value for \`image_manager\` in your cluster configuration file (e.g. cluster.yaml) and running \`cortex cluster update --config cluster.yaml\` (update other image paths in cluster.yaml as well if necessary)"
23+
exit 1
24+
fi

‎pkg/lib/configreader/validators.go

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,6 @@ func init() {
3737

3838
func GetFilePathValidator(baseDir string) func(string) (string, error) {
3939
return func(val string) (string, error) {
40-
val = files.RelToAbsPath(val, baseDir)
4140
if err := files.CheckFile(val); err != nil {
4241
return "", err
4342
}

‎pkg/lib/files/errors.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -175,7 +175,7 @@ func ErrorDirDoesNotExist(path string) error {
175175
func ErrorNotAFile(path string) error {
176176
return errors.WithStack(Error{
177177
Kind: ErrNotAFile,
178-
message: fmt.Sprintf("%s: not a file path", path),
178+
message: fmt.Sprintf("%s: no such file", path),
179179
})
180180
}
181181

‎pkg/lib/files/files.go

Lines changed: 41 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -138,7 +138,8 @@ func WriteFile(data []byte, path string) error {
138138
return nil
139139
}
140140

141-
// Returns original path if there was an error
141+
// e.g. ~/path -> /home/ubuntu/path
142+
// returns original path if there was an error
142143
func EscapeTilde(path string) (string, error) {
143144
if !(path == "~" || strings.HasPrefix(path, "~/")) {
144145
return path, nil
@@ -149,6 +150,9 @@ func EscapeTilde(path string) (string, error) {
149150
if err != nil {
150151
return path, err
151152
}
153+
if homeDir == "" || homeDir == "/" {
154+
return path, nil
155+
}
152156
_homeDir = homeDir
153157
}
154158

@@ -160,6 +164,29 @@ func EscapeTilde(path string) (string, error) {
160164
return filepath.Join(_homeDir, path[2:]), nil
161165
}
162166

167+
// e.g. /home/ubuntu/path -> ~/path
168+
func ReplacePathWithTilde(absPath string) string {
169+
if !strings.HasPrefix(absPath, "/") {
170+
return absPath
171+
}
172+
173+
if _homeDir == "" {
174+
homeDir, err := homedir.Dir()
175+
if err != nil || homeDir == "" || homeDir == "/" {
176+
return absPath
177+
}
178+
_homeDir = homeDir
179+
}
180+
181+
trimmedHomeDir := strings.TrimSuffix(s.EnsurePrefix(_homeDir, "/"), "/")
182+
183+
if strings.Index(absPath, trimmedHomeDir) == 0 {
184+
return "~" + absPath[len(trimmedHomeDir):]
185+
}
186+
187+
return absPath
188+
}
189+
163190
func TrimDirPrefix(fullPath string, dirPath string) string {
164191
if !strings.HasSuffix(dirPath, "/") {
165192
dirPath = dirPath + "/"
@@ -175,12 +202,23 @@ func RelToAbsPath(relativePath string, baseDir string) string {
175202
}
176203

177204
func UserRelToAbsPath(relativePath string) string {
178-
cwd, _ := os.Getwd()
205+
cwd, err := os.Getwd()
206+
if err != nil {
207+
return relativePath
208+
}
179209
return RelToAbsPath(relativePath, cwd)
180210
}
181211

182212
func PathRelativeToCWD(absPath string) string {
183-
cwd, _ := os.Getwd()
213+
if !strings.HasPrefix(absPath, "/") {
214+
return absPath
215+
}
216+
217+
cwd, err := os.Getwd()
218+
if err != nil {
219+
return absPath
220+
}
221+
184222
cwd = s.EnsureSuffix(cwd, "/")
185223
return strings.TrimPrefix(absPath, cwd)
186224
}

‎pkg/operator/endpoints/errors.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,7 @@ func (e Error) Error() string {
103103
func ErrorAPIVersionMismatch(operatorVersion string, clientVersion string) error {
104104
return errors.WithStack(Error{
105105
Kind: ErrAPIVersionMismatch,
106-
message: fmt.Sprintf("API version mismatch (Cluster: %s; Client: %s)", operatorVersion, clientVersion),
106+
message: fmt.Sprintf("your CLI version (%s) doesn't match your Cortex operator version (%s); please update your cluster by following the instructions at https://www.cortex.dev/cluster-management/update, or update your CLI by following the instructions at https://www.cortex.dev/install", clientVersion, operatorVersion),
107107
})
108108
}
109109

‎pkg/types/clusterconfig/clusterconfig.go

Lines changed: 65 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -227,121 +227,141 @@ var UserValidation = &cr.StructValidation{
227227
{
228228
StructField: "ImagePythonServe",
229229
StringValidation: &cr.StringValidation{
230-
Default: "cortexlabs/python-serve:" + consts.CortexVersion,
230+
Default: "cortexlabs/python-serve:" + consts.CortexVersion,
231+
Validator: validateImageVersion,
231232
},
232233
},
233234
{
234235
StructField: "ImagePythonServeGPU",
235236
StringValidation: &cr.StringValidation{
236-
Default: "cortexlabs/python-serve-gpu:" + consts.CortexVersion,
237+
Default: "cortexlabs/python-serve-gpu:" + consts.CortexVersion,
238+
Validator: validateImageVersion,
237239
},
238240
},
239241
{
240242
StructField: "ImageTFServe",
241243
StringValidation: &cr.StringValidation{
242-
Default: "cortexlabs/tf-serve:" + consts.CortexVersion,
244+
Default: "cortexlabs/tf-serve:" + consts.CortexVersion,
245+
Validator: validateImageVersion,
243246
},
244247
},
245248
{
246249
StructField: "ImageTFServeGPU",
247250
StringValidation: &cr.StringValidation{
248-
Default: "cortexlabs/tf-serve-gpu:" + consts.CortexVersion,
251+
Default: "cortexlabs/tf-serve-gpu:" + consts.CortexVersion,
252+
Validator: validateImageVersion,
249253
},
250254
},
251255
{
252256
StructField: "ImageTFAPI",
253257
StringValidation: &cr.StringValidation{
254-
Default: "cortexlabs/tf-api:" + consts.CortexVersion,
258+
Default: "cortexlabs/tf-api:" + consts.CortexVersion,
259+
Validator: validateImageVersion,
255260
},
256261
},
257262
{
258263
StructField: "ImageONNXServe",
259264
StringValidation: &cr.StringValidation{
260-
Default: "cortexlabs/onnx-serve:" + consts.CortexVersion,
265+
Default: "cortexlabs/onnx-serve:" + consts.CortexVersion,
266+
Validator: validateImageVersion,
261267
},
262268
},
263269
{
264270
StructField: "ImageONNXServeGPU",
265271
StringValidation: &cr.StringValidation{
266-
Default: "cortexlabs/onnx-serve-gpu:" + consts.CortexVersion,
272+
Default: "cortexlabs/onnx-serve-gpu:" + consts.CortexVersion,
273+
Validator: validateImageVersion,
267274
},
268275
},
269276
{
270277
StructField: "ImageOperator",
271278
StringValidation: &cr.StringValidation{
272-
Default: "cortexlabs/operator:" + consts.CortexVersion,
279+
Default: "cortexlabs/operator:" + consts.CortexVersion,
280+
Validator: validateImageVersion,
273281
},
274282
},
275283
{
276284
StructField: "ImageManager",
277285
StringValidation: &cr.StringValidation{
278-
Default: "cortexlabs/manager:" + consts.CortexVersion,
286+
Default: "cortexlabs/manager:" + consts.CortexVersion,
287+
Validator: validateImageVersion,
279288
},
280289
},
281290
{
282291
StructField: "ImageDownloader",
283292
StringValidation: &cr.StringValidation{
284-
Default: "cortexlabs/downloader:" + consts.CortexVersion,
293+
Default: "cortexlabs/downloader:" + consts.CortexVersion,
294+
Validator: validateImageVersion,
285295
},
286296
},
287297
{
288298
StructField: "ImageRequestMonitor",
289299
StringValidation: &cr.StringValidation{
290-
Default: "cortexlabs/request-monitor:" + consts.CortexVersion,
300+
Default: "cortexlabs/request-monitor:" + consts.CortexVersion,
301+
Validator: validateImageVersion,
291302
},
292303
},
293304
{
294305
StructField: "ImageClusterAutoscaler",
295306
StringValidation: &cr.StringValidation{
296-
Default: "cortexlabs/cluster-autoscaler:" + consts.CortexVersion,
307+
Default: "cortexlabs/cluster-autoscaler:" + consts.CortexVersion,
308+
Validator: validateImageVersion,
297309
},
298310
},
299311
{
300312
StructField: "ImageMetricsServer",
301313
StringValidation: &cr.StringValidation{
302-
Default: "cortexlabs/metrics-server:" + consts.CortexVersion,
314+
Default: "cortexlabs/metrics-server:" + consts.CortexVersion,
315+
Validator: validateImageVersion,
303316
},
304317
},
305318
{
306319
StructField: "ImageNvidia",
307320
StringValidation: &cr.StringValidation{
308-
Default: "cortexlabs/nvidia:" + consts.CortexVersion,
321+
Default: "cortexlabs/nvidia:" + consts.CortexVersion,
322+
Validator: validateImageVersion,
309323
},
310324
},
311325
{
312326
StructField: "ImageFluentd",
313327
StringValidation: &cr.StringValidation{
314-
Default: "cortexlabs/fluentd:" + consts.CortexVersion,
328+
Default: "cortexlabs/fluentd:" + consts.CortexVersion,
329+
Validator: validateImageVersion,
315330
},
316331
},
317332
{
318333
StructField: "ImageStatsd",
319334
StringValidation: &cr.StringValidation{
320-
Default: "cortexlabs/statsd:" + consts.CortexVersion,
335+
Default: "cortexlabs/statsd:" + consts.CortexVersion,
336+
Validator: validateImageVersion,
321337
},
322338
},
323339
{
324340
StructField: "ImageIstioProxy",
325341
StringValidation: &cr.StringValidation{
326-
Default: "cortexlabs/istio-proxy:" + consts.CortexVersion,
342+
Default: "cortexlabs/istio-proxy:" + consts.CortexVersion,
343+
Validator: validateImageVersion,
327344
},
328345
},
329346
{
330347
StructField: "ImageIstioPilot",
331348
StringValidation: &cr.StringValidation{
332-
Default: "cortexlabs/istio-pilot:" + consts.CortexVersion,
349+
Default: "cortexlabs/istio-pilot:" + consts.CortexVersion,
350+
Validator: validateImageVersion,
333351
},
334352
},
335353
{
336354
StructField: "ImageIstioCitadel",
337355
StringValidation: &cr.StringValidation{
338-
Default: "cortexlabs/istio-citadel:" + consts.CortexVersion,
356+
Default: "cortexlabs/istio-citadel:" + consts.CortexVersion,
357+
Validator: validateImageVersion,
339358
},
340359
},
341360
{
342361
StructField: "ImageIstioGalley",
343362
StringValidation: &cr.StringValidation{
344-
Default: "cortexlabs/istio-galley:" + consts.CortexVersion,
363+
Default: "cortexlabs/istio-galley:" + consts.CortexVersion,
364+
Validator: validateImageVersion,
345365
},
346366
},
347367
// Extra keys that exist in the cluster config file
@@ -364,6 +384,29 @@ var UserValidation = &cr.StructValidation{
364384
},
365385
}
366386

387+
func validateImageVersion(image string) (string, error) {
388+
if !strings.HasPrefix(image, "cortexlabs/") && !strings.HasPrefix(image, "cortexlabsdev/") {
389+
return image, nil
390+
}
391+
392+
var tag string
393+
394+
if colonIndex := strings.LastIndex(image, ":"); colonIndex != -1 {
395+
tag = image[colonIndex+1:]
396+
}
397+
398+
// in docker, missing tag implies "latest"
399+
if tag == "" {
400+
tag = "latest"
401+
}
402+
403+
if tag != consts.CortexVersion {
404+
return "", ErrorImageVersionMismatch(image, tag)
405+
}
406+
407+
return image, nil
408+
}
409+
367410
var Validation = &cr.StructValidation{
368411
StructFieldValidations: append(UserValidation.StructFieldValidations,
369412
&cr.StructFieldValidation{
@@ -395,7 +438,8 @@ var AccessValidation = &cr.StructValidation{
395438
{
396439
StructField: "ImageManager",
397440
StringValidation: &cr.StringValidation{
398-
Default: "cortexlabs/manager:" + consts.CortexVersion,
441+
Default: "cortexlabs/manager:" + consts.CortexVersion,
442+
Validator: validateImageVersion,
399443
},
400444
},
401445
},

0 commit comments

Comments
 (0)