fix(job): validate project ID early during job submit

cmd/job/job.go

@@ -59,10 +59,12 @@ var JobCmd = &cobra.Command{
 		if location == "" {
 			return fmt.Errorf("location is required; please specify it using the --location flag or set a default value using 'gcluster job config set location <value>'")
 		}
-
 		if projectID == "" {
 			return fmt.Errorf("project ID is required; please specify it using the --project flag or set a default value using 'gcluster job config set project <value>'")
 		}
+		if err := ensureBasicPrerequisites(cmd, projectID); err != nil {
+			return err
+		}
 
 		resolvedLoc, err := orc.Initialize(clusterName, location, projectID)
 		if err != nil {

cmd/job/prereq.go

@@ -122,6 +122,8 @@ func ensureGCloudAuthenticated() error {
 	return nil
 }
 
+var getADCSetupCommandFunc = getADCSetupCommand
+
 // getADCSetupCommand checks if Application Default Credentials are valid and returns the setup command if not.
 func getADCSetupCommand() string {
 	creds, err := google.FindDefaultCredentials(context.Background(), "https://www.googleapis.com/auth/cloud-platform")
@@ -224,19 +226,28 @@ func isDockerCredsConfigured(region string) bool {
 	return config.CredHelpers[pkgDevReg] == "gcloud"
 }
 
-// EnsurePrerequisites checks all necessary gcloud and kubectl prerequisites.
-func ensurePrerequisites(cmd *cobra.Command, projectID *string, location string) error {
+// ensureProjectExists checks if the project exists and is accessible.
+func ensureProjectExists(projectID string) error {
+	result := shell.ExecuteCommand("gcloud", "projects", "describe", projectID)
+	if result.ExitCode != 0 {
+		return fmt.Errorf("failed to validate project: %s", strings.TrimSpace(result.Stderr))
+	}
+	return nil
+}
+
+// ensureBasicPrerequisites checks for gcloud, auth, project existence, and kubectl.
+func ensureBasicPrerequisites(cmd *cobra.Command, projectID string) error {
 	if dryRunManifest != "" {
 		return nil
 	}
 
 	state := store.Load()
 
-	if !isStateStale(state, *projectID) {
-		logging.Info("Skipping checks; prerequisites are fresh (project: %s, checked: %v ago).", state.LastCheckedProjectID, time.Since(state.LastCheckedTimestamp).Round(time.Second))
+	if !isStateStale(state, projectID) {
+		logging.Info("Skipping basic checks; prerequisites are fresh (project: %s, checked: %v ago).", state.LastCheckedProjectID, time.Since(state.LastCheckedTimestamp).Round(time.Second))
 		return nil
 	}
-	logging.Info("Prerequisites state is stale or project ID changed, performing fresh check.")
+
 	state = PrereqState{}
 
 	var missing []missingPrereq
@@ -245,55 +256,124 @@ func ensurePrerequisites(cmd *cobra.Command, projectID *string, location string)
 	if err := ensureGCloudSDKInstalled(); err != nil {
 		return err
 	}
-	state.GCloudSDKInstalled = true
 
 	// Check GCloud Auth
+	gcloudAuthOK := false
 	if err := ensureGCloudAuthenticated(); err != nil {
 		missing = append(missing, missingPrereq{name: "Google Cloud Authentication", commands: []string{"gcloud auth login"}})
 	} else {
-		state.GCloudAuthenticated = true
+		gcloudAuthOK = true
 	}
 
 	// Check ADC
-	if adcCmd := getADCSetupCommand(); adcCmd != "" {
+	adcCmd := getADCSetupCommandFunc()
+	if adcCmd != "" {
 		missing = append(missing, missingPrereq{name: "Application Default Credentials (ADC)", commands: []string{adcCmd}})
-	} else {
-		state.ADCConfigured = true
 	}
 
 	checkK8sDependencies(&state, &missing)
 
-	// Check Docker creds
+	// Run project validation if auth is OK, regardless of other missing checks
+	var projectErr error
+	if gcloudAuthOK && projectID != "" {
+		projectErr = ensureProjectExists(projectID)
+	}
+
+	// Now decide what to return
+	if projectErr != nil {
+		if len(missing) > 0 {
+			printMissingPrereqs(cmd, missing)
+		}
+		return fmt.Errorf("project %q is invalid or inaccessible: %w", projectID, projectErr)
+	}
+
+	if len(missing) > 0 {
+		printMissingPrereqs(cmd, missing)
+		return fmt.Errorf("some basic prerequisites are missing")
+	}
+
+	// All basic checks passed! Save state.
+	state.GCloudSDKInstalled = true
+	state.GCloudAuthenticated = true
+	state.ADCConfigured = (adcCmd == "")
+	// state.KubectlInstalled and state.GKEGCloudAuthPluginInstalled are already set inside checkK8sDependencies
+
+	state.LastCheckedTimestamp = time.Now()
+	state.LastCheckedProjectID = projectID
+	store.Save(state)
+
+	return nil
+}
+
+// hasPassedBasicPrerequisites checks if all basic prerequisite checks are marked as passed.
+func hasPassedBasicPrerequisites(state PrereqState) bool {
+	return state.GCloudSDKInstalled &&
+		state.GCloudAuthenticated &&
+		state.ADCConfigured &&
+		state.KubectlInstalled &&
+		state.GKEGCloudAuthPluginInstalled
+}
+
+func checkArtifactRegistryAPI(projectID string, state *PrereqState, missing *[]missingPrereq) {
+	if projectID == "" {
+		return
+	}
+	apiResult := shell.ExecuteCommand("gcloud", "services", "list", "--filter=NAME:artifactregistry.googleapis.com", "--format=value(STATE)", "--project", projectID)
+	if strings.TrimSpace(apiResult.Stdout) != "ENABLED" {
+		*missing = append(*missing, missingPrereq{
+			name:     "Artifact Registry API",
+			commands: []string{fmt.Sprintf("gcloud services enable artifactregistry.googleapis.com --project %s --quiet", projectID)},
+		})
+	} else {
+		state.ArtifactRegistryAPIEnabled = true
+	}
+}
+
+func checkDockerCredentials(location string, state *PrereqState, missing *[]missingPrereq) {
 	region := shell.ExtractRegion(location)
 	if !isDockerCredsConfigured(region) {
 		cmds := []string{"gcloud auth configure-docker gcr.io --quiet"}
 		if region != "" {
 			cmds = append(cmds, fmt.Sprintf("gcloud auth configure-docker %s-docker.pkg.dev --quiet", region))
 		}
-		missing = append(missing, missingPrereq{
+		*missing = append(*missing, missingPrereq{
 			name:     "Docker Credentials",
 			commands: cmds,
 		})
 	} else {
 		state.DockerCredsConfigured = true
 	}
+}
 
-	// Check Artifact Registry API
-	if *projectID != "" {
-		apiResult := shell.ExecuteCommand("gcloud", "services", "list", "--filter=NAME:artifactregistry.googleapis.com", "--format=value(STATE)", "--project", *projectID)
-		if strings.TrimSpace(apiResult.Stdout) != "ENABLED" {
-			missing = append(missing, missingPrereq{
-				name:     "Artifact Registry API",
-				commands: []string{fmt.Sprintf("gcloud services enable artifactregistry.googleapis.com --project %s --quiet", *projectID)},
-			})
-		} else {
-			state.ArtifactRegistryAPIEnabled = true
+// EnsurePrerequisites checks all necessary gcloud and kubectl prerequisites.
+func ensurePrerequisites(cmd *cobra.Command, projectID *string, location string) error {
+	if dryRunManifest != "" {
+		return nil
+	}
+
+	state := store.Load()
+
+	if !isStateStale(state, *projectID) && state.DockerCredsConfigured && state.ArtifactRegistryAPIEnabled {
+		logging.Info("Skipping checks; prerequisites are fresh (project: %s, checked: %v ago).", state.LastCheckedProjectID, time.Since(state.LastCheckedTimestamp).Round(time.Second))
+		return nil
+	}
+
+	// Safety check: if basic checks haven't run or are not recorded as passed, run them now.
+	if !hasPassedBasicPrerequisites(state) {
+		if err := ensureBasicPrerequisites(cmd, *projectID); err != nil {
+			return err
 		}
+		state = store.Load() // Reload the state updated by ensureBasicPrerequisites
 	}
 
+	var missing []missingPrereq
+
+	checkArtifactRegistryAPI(*projectID, &state, &missing)
+	checkDockerCredentials(location, &state, &missing)
+
 	if len(missing) > 0 {
 		printMissingPrereqs(cmd, missing)
-		return fmt.Errorf("job could not be submitted because some prerequisites are missing.")
+		return fmt.Errorf("job could not be submitted because some prerequisites are missing")
 	}
 
 	state.LastCheckedTimestamp = time.Now()

cmd/job/prereq_test.go

@@ -99,7 +99,16 @@ func TestStateFilePath(t *testing.T) {
 	}
 }
 
+func useFileStore(t *testing.T) {
+	oldStore := store
+	store = &FilePrereqStore{}
+	t.Cleanup(func() {
+		store = oldStore
+	})
+}
+
 func TestLoadPrereqState_Success(t *testing.T) {
+	useFileStore(t)
 	tempDir, err := os.MkdirTemp("", "prereq-load-test")
 	if err != nil {
 		t.Fatal(err)
@@ -136,6 +145,7 @@ func TestLoadPrereqState_Success(t *testing.T) {
 }
 
 func TestSavePrereqState_Success(t *testing.T) {
+	useFileStore(t)
 	tempDir, err := os.MkdirTemp("", "prereq-save-test")
 	if err != nil {
 		t.Fatal(err)
@@ -177,6 +187,7 @@ func TestSavePrereqState_Success(t *testing.T) {
 }
 
 func TestLoadPrereqState_CorruptedFile(t *testing.T) {
+	useFileStore(t)
 	tempDir, err := os.MkdirTemp("", "prereq-corrupt-test")
 	if err != nil {
 		t.Fatal(err)
@@ -204,6 +215,7 @@ func TestLoadPrereqState_CorruptedFile(t *testing.T) {
 }
 
 func TestSavePrereqState_WriteError(t *testing.T) {
+	useFileStore(t)
 	tempDir, err := os.MkdirTemp("", "prereq-write-error-test")
 	if err != nil {
 		t.Fatal(err)
@@ -422,9 +434,17 @@ func TestEnsurePrerequisites_DockerCreds(t *testing.T) {
 		return shell.CommandResult{ExitCode: 0}
 	}
 
+	origGetADCSetupCommand := getADCSetupCommandFunc
+	defer func() { getADCSetupCommandFunc = origGetADCSetupCommand }()
+	getADCSetupCommandFunc = func() string { return "" }
+
 	origStore := store
 	defer func() { store = origStore }()
-	store = &mockPrereqStore{}
+	store = &MockPrereqStore{
+		State: PrereqState{
+			LastCheckedTimestamp: time.Now().Add(-48 * time.Hour), // Stale
+		},
+	}
 
 	cmd := &cobra.Command{}
 	projectID := "test-project"
@@ -454,3 +474,92 @@ func (m *mockPrereqStore) Load() PrereqState {
 }
 
 func (m *mockPrereqStore) Save(state PrereqState) {}
+
+func TestEnsureBasicPrerequisites_InvalidProject(t *testing.T) {
+	tempDir := t.TempDir()
+	t.Setenv("HOME", tempDir)
+
+	origExecuteCommand := shell.ExecuteCommand
+	defer func() { shell.ExecuteCommand = origExecuteCommand }()
+
+	shell.ExecuteCommand = func(name string, args ...string) shell.CommandResult {
+		cmdStr := name + " " + strings.Join(args, " ")
+		switch {
+		case strings.HasPrefix(cmdStr, "gcloud auth list"):
+			return shell.CommandResult{ExitCode: 0, Stdout: "user@example.com"}
+		case cmdStr == "gcloud projects describe invalid-project":
+			return shell.CommandResult{ExitCode: 1, Stderr: "Project not found"}
+		default:
+			return shell.CommandResult{ExitCode: 0}
+		}
+	}
+
+	origStore := store
+	defer func() { store = origStore }()
+	store = &mockPrereqStore{}
+
+	origGetADCSetupCommand := getADCSetupCommandFunc
+	defer func() { getADCSetupCommandFunc = origGetADCSetupCommand }()
+	getADCSetupCommandFunc = func() string { return "" }
+
+	cmd := &cobra.Command{}
+	projectID := "invalid-project"
+
+	err := ensureBasicPrerequisites(cmd, projectID)
+	if err == nil {
+		t.Fatal("expected error because project is invalid, got nil")
+	}
+
+	expectedErrorMsg := "project \"invalid-project\" is invalid or inaccessible"
+	if !strings.Contains(err.Error(), expectedErrorMsg) {
+		t.Errorf("expected error to contain %q, but got: %v", expectedErrorMsg, err)
+	}
+}
+
+func TestEnsureBasicPrerequisites_SaveState(t *testing.T) {
+	useFileStore(t)
+	tempDir := t.TempDir()
+	t.Setenv("HOME", tempDir)
+
+	origExecuteCommand := shell.ExecuteCommand
+	defer func() { shell.ExecuteCommand = origExecuteCommand }()
+
+	shell.ExecuteCommand = func(name string, args ...string) shell.CommandResult {
+		return shell.CommandResult{ExitCode: 0, Stdout: "user@example.com"}
+	}
+
+	origGetADCSetupCommand := getADCSetupCommandFunc
+	defer func() { getADCSetupCommandFunc = origGetADCSetupCommand }()
+	getADCSetupCommandFunc = func() string { return "" }
+
+	cmd := &cobra.Command{}
+	projectID := "test-project"
+
+	err := ensureBasicPrerequisites(cmd, projectID)
+	if err != nil {
+		t.Fatalf("expected nil error, got: %v", err)
+	}
+
+	state := store.Load()
+	if !state.GCloudSDKInstalled {
+		t.Error("expected GCloudSDKInstalled to be true")
+	}
+	if !state.GCloudAuthenticated {
+		t.Error("expected GCloudAuthenticated to be true")
+	}
+	if !state.ADCConfigured {
+		t.Error("expected ADCConfigured to be true")
+	}
+	if !state.KubectlInstalled {
+		t.Error("expected KubectlInstalled to be true")
+	}
+	if !state.GKEGCloudAuthPluginInstalled {
+		t.Error("expected GKEGCloudAuthPluginInstalled to be true")
+	}
+	if state.LastCheckedProjectID != "test-project" {
+		t.Errorf("expected LastCheckedProjectID to be 'test-project', got: %s", state.LastCheckedProjectID)
+	}
+	if time.Since(state.LastCheckedTimestamp) > 5*time.Second {
+		t.Errorf("expected LastCheckedTimestamp to be recent, got: %v", state.LastCheckedTimestamp)
+	}
+}

cmd/job/submit.go

@@ -109,7 +109,6 @@ and JobSet/Kueue specific configurations like workload name, queue, nodes, and r
 		if err := validatePathwaysFlags(); err != nil {
 			return err
 		}
-
 		if err := ensurePrerequisites(cmd, &projectID, location); err != nil {
 			return err
 		}