feat!: Use artifactory catalog

cmd/topo/projects.go

@@ -27,13 +27,11 @@ var projectsCmd = &cobra.Command{
 
 		var projects []catalog.Project
 		var err error
-		source := getSource(cmd)
-		switch source {
-		case builtinProjects:
-			projects, err = catalog.ListBuiltinProjects()
-		default:
-			projects, err = catalog.ListProjectsFromURL(ctx, source)
+		source, err := cmd.Flags().GetString(sourceFlag)
+		if err != nil {
+			panic(fmt.Sprintf("internal error: %s flag not registered: %v", sourceFlag, err))
 		}
+		projects, err = catalog.ListProjectsFromURL(ctx, source)
 		if err != nil {
 			return err
 		}
@@ -56,23 +54,6 @@ var projectsCmd = &cobra.Command{
 func init() {
 	addTargetFlag(projectsCmd)
 	addTimeoutFlag(projectsCmd, defaultTimeout)
-	if experimentalFeaturesEnabled() {
-		projectsCmd.Flags().StringP(sourceFlag, "s", "", "where to source projects' data from")
-	}
+	projectsCmd.Flags().StringP(sourceFlag, "s", catalog.DefaultCatalogURL, "where to source projects' data from")
 	rootCmd.AddCommand(projectsCmd)
 }
-
-const builtinProjects = "builtin"
-
-func getSource(cmd *cobra.Command) string {
-	if experimentalFeaturesEnabled() {
-		flagValue, err := cmd.Flags().GetString(sourceFlag)
-		if err != nil {
-			panic(fmt.Sprintf("internal error: %s flag not registered: %v", sourceFlag, err))
-		}
-		if flagValue != "" {
-			return flagValue
-		}
-	}
-	return builtinProjects
-}

docs/development/DEVELOPMENT.md

@@ -87,3 +87,19 @@ docker compose up
 ```
 
 The documentation preview is available at `http://localhost:3000` and automatically reloads when files change.
+
+## Updating the catalog schema
+
+The catalog schema version is declared by the `go:generate` directive in `internal/catalog/catalog.go` and copied into `internal/catalog/catalog_schema_generated.go`. Its major component also selects the catalog version used by Topo.
+
+Generating the Go types requires Node.js 20 or newer, `npx`, and access to the internet. From the repository root, run:
+
+```sh
+go generate ./internal/catalog
+```
+
+The generator downloads the configured version's `catalog.schema.json`, generates the catalog Go types with the pinned Quicktype version, and rewrites `internal/catalog/catalog_schema_generated.go`. To use another schema release, update the directive in `internal/catalog/catalog.go` and rerun the command.
+
+Commit the newly generated catalog schema types file and raise a PR to update the main branch.
+
+Available catalog versions and schemas are published in the [Topo Project Catalog Artifactory repository](https://artifacts.tools.arm.com/devx-topo-project-catalog/).

internal/catalog/catalog.go

@@ -1,41 +1,27 @@
 package catalog
 
+//go:generate go run ../../scripts/generate_catalog_types v2.0.0
+
 import (
-	"bytes"
 	"context"
-	_ "embed"
 	"encoding/json"
 	"fmt"
-	"io"
-	"net/http"
-	"net/url"
 	"os"
 	"strings"
 
-	"github.com/santhosh-tekuri/jsonschema/v6"
+	"github.com/arm/topo/internal/fetch"
 )
 
-//go:embed data/catalog.json
-var catalogJSON []byte
-
-//go:embed data/catalog.schema.json
-var catalogSchemaJSON []byte
-
-type catalogDocument struct {
-	Schema   string    `json:"$schema,omitempty"`
-	Projects []Project `json:"projects"`
-}
+type Project = ProjectElement
 
-type Project struct {
-	Name        string   `json:"name"`
-	Description string   `json:"description"`
-	Features    []string `json:"features"`
-	URL         string   `json:"url"`
-	Ref         string   `json:"ref"`
-}
+var (
+	defaultURL        = "https://artifacts.tools.arm.com/devx-topo-project-catalog/" + majorVersion(CatalogSchemaVersion) + "/catalog/"
+	DefaultCatalogURL = defaultURL + "catalog.json"
+)
 
-func ListBuiltinProjects() ([]Project, error) {
-	return parseProjects(catalogJSON)
+func majorVersion(version string) string {
+	major, _, _ := strings.Cut(version, ".")
+	return major
 }
 
 func ListProjectsFromURL(ctx context.Context, url string) ([]Project, error) {
@@ -47,39 +33,32 @@ func ListProjectsFromURL(ctx context.Context, url string) ([]Project, error) {
 }
 
 func parseProjects(b []byte) ([]Project, error) {
-	if err := validateAgainstSchema(b); err != nil {
-		return nil, fmt.Errorf("failed schema validation: %w", err)
+	catalogVersion, versionErr := unmarshalCatalogVersion(b)
+	catalogVersionMajor := majorVersion(catalogVersion)
+	SchemaVersionMajor := majorVersion(CatalogSchemaVersion)
+	if catalogVersionMajor != SchemaVersionMajor {
+		return nil, fmt.Errorf(
+			"failed to parse catalog: requested catalog version %q is incompatible with supported schema version %q: %w",
+			catalogVersion,
+			CatalogSchemaVersion,
+			versionErr,
+		)
 	}
-
-	var catalog catalogDocument
-	if err := json.Unmarshal(b, &catalog); err != nil {
-		return nil, fmt.Errorf("failed to unmarshal projects: %w", err)
+	catalog, err := UnmarshalCatalogDocument(b)
+	if err != nil {
+		return nil, fmt.Errorf("failed to unmarshal catalog: %w", err)
 	}
-
 	return catalog.Projects, nil
 }
 
-func validateAgainstSchema(b []byte) error {
-	const projectsSchemaURL = "https://raw.githubusercontent.com/arm/topo/main/internal/catalog/data/catalog.schema.json"
-
-	compiler := jsonschema.NewCompiler()
-	schemaDoc, err := jsonschema.UnmarshalJSON(bytes.NewReader(catalogSchemaJSON))
-	if err != nil {
-		return fmt.Errorf("failed to unmarshal schema: %w", err)
-	}
-	if err := compiler.AddResource(projectsSchemaURL, schemaDoc); err != nil {
-		return fmt.Errorf("failed to add schema resource: %w", err)
+func unmarshalCatalogVersion(b []byte) (string, error) {
+	var header struct {
+		Version string `json:"version"`
 	}
-	schema, err := compiler.Compile(projectsSchemaURL)
-	if err != nil {
-		return fmt.Errorf("failed to compile schema: %w", err)
-	}
-
-	jsonDoc, err := jsonschema.UnmarshalJSON(bytes.NewReader(b))
-	if err != nil {
-		return fmt.Errorf("failed to unmarshal projects: %w", err)
+	if err := json.Unmarshal(b, &header); err != nil {
+		return "", err
 	}
-	return schema.Validate(jsonDoc)
+	return header.Version, nil
 }
 
 func fetchProjectsJSON(ctx context.Context, url string) ([]byte, error) {
@@ -92,42 +71,9 @@ func fetchProjectsJSON(ctx context.Context, url string) ([]byte, error) {
 		return data, nil
 	}
 
-	data, err := httpGet(ctx, url)
+	data, err := fetch.Get(ctx, url)
 	if err != nil {
 		return nil, fmt.Errorf("failed to fetch project: %w", err)
 	}
 	return data, nil
 }
-
-func httpGet(ctx context.Context, rawURL string) ([]byte, error) {
-	parsedURL, err := url.Parse(rawURL)
-	if err != nil {
-		return nil, err
-	}
-
-	if parsedURL.Scheme != "http" && parsedURL.Scheme != "https" {
-		return nil, fmt.Errorf("unsupported URL scheme: %s", parsedURL.Scheme)
-	}
-
-	req, err := http.NewRequestWithContext(
-		ctx,
-		http.MethodGet,
-		parsedURL.String(),
-		nil,
-	)
-	if err != nil {
-		return nil, err
-	}
-
-	resp, err := http.DefaultClient.Do(req) // #nosec G704 -- URL is explicitly provided by the CLI user and scheme-validated above.
-	if err != nil {
-		return nil, err
-	}
-	defer resp.Body.Close() // nolint:errcheck
-
-	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
-		return nil, fmt.Errorf("request failed: %s", resp.Status)
-	}
-
-	return io.ReadAll(resp.Body)
-}

internal/catalog/catalog_test.go

@@ -44,17 +44,6 @@ func TestListProjectsFromURL(t *testing.T) {
 		assert.Equal(t, projects, got)
 	})
 
-	t.Run("errors when payload doesn't validate against schema", func(t *testing.T) {
-		path := filepath.Join(t.TempDir(), "file.json")
-		projects := []catalog.Project{{Name: "aloha"}}
-		testutil.RequireWriteFile(t, path, string(asJSON(projects)))
-
-		url := fmt.Sprintf("file://%s", path)
-		_, err := catalog.ListProjectsFromURL(context.Background(), url)
-
-		require.Error(t, err)
-	})
-
 	t.Run("errors when request fails", func(t *testing.T) {
 		server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 			http.NotFound(w, r)
@@ -76,15 +65,37 @@ func TestListProjectsFromURL(t *testing.T) {
 		_, err := catalog.ListProjectsFromURL(context.Background(), url)
 
 		require.Error(t, err)
-		assert.ErrorContains(t, err, "failed to unmarshal projects")
+		assert.ErrorContains(t, err, "failed to parse catalog")
+		assert.ErrorContains(t, err, `requested catalog version "" is incompatible`)
+	})
+
+	t.Run("reports catalog and schema versions when incompatible catalog fails to unmarshal", func(t *testing.T) {
+		path := filepath.Join(t.TempDir(), "file.json")
+		catalogVersion := "v0.0.0"
+		if catalogVersion == catalog.CatalogSchemaVersion {
+			catalogVersion = "v999.0.0"
+		}
+		testutil.RequireWriteFile(t, path, fmt.Sprintf(`{"projects":"invalid","version":%q}`, catalogVersion))
+
+		url := fmt.Sprintf("file://%s", path)
+		_, err := catalog.ListProjectsFromURL(context.Background(), url)
+
+		require.Error(t, err)
+		assert.ErrorContains(t, err, fmt.Sprintf(
+			`requested catalog version %q is incompatible with supported schema version %q`,
+			catalogVersion,
+			catalog.CatalogSchemaVersion,
+		))
 	})
 }
 
 func asJSON(projects []catalog.Project) []byte {
 	data, err := json.Marshal(struct {
 		Projects []catalog.Project `json:"projects"`
+		Version  string            `json:"version"`
 	}{
 		Projects: projects,
+		Version:  catalog.CatalogSchemaVersion,
 	})
 	if err != nil {
 		panic(err)

internal/fetch/fetch.go

@@ -0,0 +1,42 @@
+package fetch
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+)
+
+func Get(ctx context.Context, rawURL string) ([]byte, error) {
+	parsedURL, err := url.Parse(rawURL)
+	if err != nil {
+		return nil, fmt.Errorf("parsing URL failed: %w", err)
+	}
+	if parsedURL.Scheme != "http" && parsedURL.Scheme != "https" {
+		return nil, fmt.Errorf("unsupported URL scheme: %s", parsedURL.Scheme)
+	}
+
+	request, err := http.NewRequestWithContext(ctx, http.MethodGet, parsedURL.String(), nil)
+	if err != nil {
+		return nil, fmt.Errorf("creating request failed: %w", err)
+	}
+
+	// #nosec G704 -- callers explicitly provide the URL and its scheme is validated above.
+	response, err := http.DefaultClient.Do(request)
+	if err != nil {
+		return nil, fmt.Errorf("sending request failed: %w", err)
+	}
+
+	if response.StatusCode != http.StatusOK {
+		statusErr := fmt.Errorf("request failed: HTTP %d (%s)", response.StatusCode, response.Status)
+		return nil, errors.Join(statusErr, response.Body.Close())
+	}
+
+	data, readErr := io.ReadAll(response.Body)
+	if err := errors.Join(readErr, response.Body.Close()); err != nil {
+		return nil, fmt.Errorf("failed to read response: %w", err)
+	}
+	return data, nil
+}