fix(job): validate project ID early during job submit

cmd/job/job.go

@@ -64,6 +64,12 @@ var JobCmd = &cobra.Command{
 			return fmt.Errorf("project ID is required; please specify it using the --project flag or set a default value using 'gcluster job config set project <value>'")
 		}
 
+		if cmd.Name() == "submit" {
+			if err := ensurePrerequisites(cmd, &projectID, location); err != nil {
+				return err
+			}
+		}
+
 		resolvedLoc, err := orc.Initialize(clusterName, location, projectID)
 		if err != nil {
 			return err

cmd/job/prereq.go

@@ -224,6 +224,42 @@ func isDockerCredsConfigured(region string) bool {
 	return config.CredHelpers[pkgDevReg] == "gcloud"
 }
 
+// ensureProjectExists checks if the project exists and is accessible.
+func ensureProjectExists(projectID string) error {
+	result := shell.ExecuteCommand("gcloud", "projects", "describe", projectID)
+	if result.ExitCode != 0 {
+		return fmt.Errorf("failed to validate project: %s", strings.TrimSpace(result.Stderr))
+	}
+	return nil
+}
+
+// checkProjectPrereqs validates the project exists and checks if Artifact Registry API is enabled.
+func checkProjectPrereqs(projectID string, gcloudAuthOK bool, state *PrereqState, missing *[]missingPrereq) {
+	if !gcloudAuthOK || projectID == "" {
+		return
+	}
+
+	if err := ensureProjectExists(projectID); err != nil {
+		*missing = append(*missing, missingPrereq{
+			name:     fmt.Sprintf("Project ID validation for %q", projectID),
+			commands: []string{fmt.Sprintf("# Error: %v", err)},
+		})
+		return
+	}
+	state.GCloudProjectConfigured = true
+
+	// Check Artifact Registry API
+	apiResult := shell.ExecuteCommand("gcloud", "services", "list", "--filter=NAME:artifactregistry.googleapis.com", "--format=value(STATE)", "--project", projectID)
+	if strings.TrimSpace(apiResult.Stdout) != "ENABLED" {
+		*missing = append(*missing, missingPrereq{
+			name:     "Artifact Registry API",
+			commands: []string{fmt.Sprintf("gcloud services enable artifactregistry.googleapis.com --project %s --quiet", projectID)},
+		})
+	} else {
+		state.ArtifactRegistryAPIEnabled = true
+	}
+}
+
 // EnsurePrerequisites checks all necessary gcloud and kubectl prerequisites.
 func ensurePrerequisites(cmd *cobra.Command, projectID *string, location string) error {
 	if dryRunManifest != "" {
@@ -278,18 +314,7 @@ func ensurePrerequisites(cmd *cobra.Command, projectID *string, location string)
 		state.DockerCredsConfigured = true
 	}
 
-	// Check Artifact Registry API
-	if *projectID != "" {
-		apiResult := shell.ExecuteCommand("gcloud", "services", "list", "--filter=NAME:artifactregistry.googleapis.com", "--format=value(STATE)", "--project", *projectID)
-		if strings.TrimSpace(apiResult.Stdout) != "ENABLED" {
-			missing = append(missing, missingPrereq{
-				name:     "Artifact Registry API",
-				commands: []string{fmt.Sprintf("gcloud services enable artifactregistry.googleapis.com --project %s --quiet", *projectID)},
-			})
-		} else {
-			state.ArtifactRegistryAPIEnabled = true
-		}
-	}
+	checkProjectPrereqs(*projectID, state.GCloudAuthenticated, &state, &missing)
 
 	if len(missing) > 0 {
 		printMissingPrereqs(cmd, missing)

cmd/job/prereq_test.go

@@ -454,3 +454,53 @@ func (m *mockPrereqStore) Load() PrereqState {
 }
 
 func (m *mockPrereqStore) Save(state PrereqState) {}
+
+func TestEnsurePrerequisites_InvalidProject(t *testing.T) {
+	tempDir := t.TempDir()
+	t.Setenv("HOME", tempDir)
+
+	origExecuteCommand := shell.ExecuteCommand
+	defer func() { shell.ExecuteCommand = origExecuteCommand }()
+
+	servicesListCalled := false
+	shell.ExecuteCommand = func(name string, args ...string) shell.CommandResult {
+		cmdStr := name + " " + strings.Join(args, " ")
+		switch {
+		case strings.HasPrefix(cmdStr, "gcloud auth list"):
+			return shell.CommandResult{ExitCode: 0, Stdout: "user@example.com"}
+		case cmdStr == "gcloud projects describe invalid-project":
+			return shell.CommandResult{ExitCode: 1, Stderr: "Project not found"}
+		case strings.HasPrefix(cmdStr, "gcloud services list"):
+			servicesListCalled = true
+			return shell.CommandResult{ExitCode: 0}
+		default:
+			return shell.CommandResult{ExitCode: 0}
+		}
+	}
+
+	origStore := store
+	defer func() { store = origStore }()
+	store = &mockPrereqStore{}
+
+	cmd := &cobra.Command{}
+	projectID := "invalid-project"
+	location := "us-central1-a"
+
+	var buf bytes.Buffer
+	cmd.SetOut(&buf)
+
+	err := ensurePrerequisites(cmd, &projectID, location)
+	if err == nil {
+		t.Error("expected error because project is invalid, got nil")
+	}
+
+	output := buf.String()
+	expectedErrorMsg := "Project ID validation for \"invalid-project\""
+	if !strings.Contains(output, expectedErrorMsg) {
+		t.Errorf("expected output to contain %q, but got:\n%s", expectedErrorMsg, output)
+	}
+
+	if servicesListCalled {
+		t.Error("expected gcloud services list to NOT be called, but it was")
+	}
+}

cmd/job/submit.go

@@ -110,10 +110,6 @@ and JobSet/Kueue specific configurations like workload name, queue, nodes, and r
 			return err
 		}
 
-		if err := ensurePrerequisites(cmd, &projectID, location); err != nil {
-			return err
-		}
-
 		if err := validateGKENAPFlags(); err != nil {
 			return err
 		}

cmd/job/submit_test.go

@@ -332,7 +332,11 @@ type MockPrereqStore struct {
 }
 
 func (m *MockPrereqStore) Load() PrereqState {
-	return m.State
+	state := m.State
+	if state.LastCheckedProjectID == "" {
+		state.LastCheckedProjectID = "test-project"
+	}
+	return state
 }
 
 func (m *MockPrereqStore) Save(state PrereqState) {